Warning: The following options to parse-opts are unrecognized: :valid? 12:27:02.140 [main] INFO jepsen.cli - Test options: {:hashdump false, :ssh-private-key "./resources/vagrantkey", :enable-memcached-debug-log-level false, :package {:type :deb, :package #object[java.io.File 0x4b508371 "./couchbase-server-enterprise_6.5.1-6007-ubuntu16.04_amd64.deb"]}, :bucket-type :couchbase, :persist-to 0, :collect-core-files false, :manipulate-disks false, :concurrency 4, :autoreprovision-maxnodes 1, :skip-teardown false, :dcp-set-read false, :replicate-to 0, :cluster-run false, :server-groups-enabled false, :cas true, :ssh {:username "vagrant", :password "root", :strict-host-key-checking false, :private-key-path "./resources/vagrantkey"}, :rate 0, :doc-padding-size 3072, :durability (0 100 0 0), :nodes ["172.28.128.125" "172.28.128.126" "172.28.128.127" "172.28.128.128"], :failover-type :graceful, :test-count 1, :install-path "/opt/couchbase", :doc-count 30, :process-suspend-time 10, :perf-graphs false, :replicas 2, :autofailover false, :cbcollect true, :connect-timeout 10, :autoreprovision true, :disrupt-count 1, :collect-data-files false, :mixed-txns false, :kv-timeout #object[java.time.Duration 0x20027c44 "PT2S"], :server-group-count 1, :time-limit 60, :transactions false, :use-json-docs true, :eviction-policy "valueOnly", :workload "failover", :recovery-type :delta, :enable-tcp-capture false, :target-server-groups false} INFO [2019-12-17 12:27:02,265] jepsen test runner - jepsen.core Running test: {:hashdump false, :cycles 1, :node-count 4, :ssh-private-key "./resources/vagrantkey", :enable-memcached-debug-log-level false, :package {:type :deb, :package #object[java.io.File 0x4b508371 "./couchbase-server-enterprise_6.5.1-6007-ubuntu16.04_amd64.deb"]}, :bucket-type :couchbase, :persist-to 0, :collect-core-files false, :manipulate-disks false, :concurrency 90, :pool-size 6, :autoreprovision-maxnodes 1, :random-server-group nil, :txn-doc-count 3, :db #object[couchbase.core$couchbase_remote$reify__1269 0x60ed0b9d "couchbase.core$couchbase_remote$reify__1269@60ed0b9d"], :autofailover-maxcount 3, :skip-teardown false, :name "Couchbase", :sg-enabled false, :dcp-set-read false, :start-time #object[org.joda.time.DateTime 0x3ae2702a "2019-12-17T12:27:02.000Z"], :net #object[jepsen.net$reify__5211 0x6e948f1c "jepsen.net$reify__5211@6e948f1c"], :client {:cluster nil, :bucket nil, :collection nil, :env nil, :txn nil}, :replicate-to 0, :barrier #object[java.util.concurrent.CyclicBarrier 0x6f289728 "java.util.concurrent.CyclicBarrier@6f289728"], :cluster-run false, :server-groups-enabled false, :cas true, :ssh {:username "vagrant", :password "root", :strict-host-key-checking false, :private-key-path "./resources/vagrantkey"}, :rate 0, :autofailover-timeout 6, :doc-padding-size 3072, :checker #object[jepsen.checker$compose$reify__4751 0x23bd047f "jepsen.checker$compose$reify__4751@23bd047f"], :client-generator #object[jepsen.independent$concurrent_generator$reify__2879 0x7c226095 "jepsen.independent$concurrent_generator$reify__2879@7c226095"], :db-intialized #, :nemesis #object[couchbase.nemesis$couchbase$reify__1064 0x79b63325 "couchbase.nemesis$couchbase$reify__1064@79b63325"], :durability (0 100 0 0), :active-histories #, :control-atom #, :nodes ["172.28.128.125" "172.28.128.126" "172.28.128.127" "172.28.128.128"], :failover-type :graceful, :test-count 1, :install-path "/opt/couchbase", :doc-count 30, :process-suspend-time 10, :perf-graphs false, :replicas 2, :autofailover false, :cbcollect true, :connect-timeout 10, :autoreprovision true, :txn-bucket-count 1, :disrupt-count 1, :generator (gen/concat [(gen/on #{:nemesis} (gen/seqone ((gen/start-timeout 1800000) (gen/delayfn #object[clojure.core$constantly$fn__5672 0x1a6dc5ea "clojure.core$constantly$fn__5672@1a6dc5ea"] (gen/gvoid)) {:type :info, :f :failover, :failover-type :graceful, :targeter #object[couchbase.nemesis$basic_nodes_targeter 0x61957d9c "couchbase.nemesis$basic_nodes_targeter@61957d9c"], :target-count 1} (gen/delayfn #object[clojure.core$constantly$fn__5672 0x29c59b6e "clojure.core$constantly$fn__5672@29c59b6e"] (gen/gvoid)) {:type :info, :f :recover, :recovery-type :delta} (gen/delayfn #object[clojure.core$constantly$fn__5672 0x483b7dc4 "clojure.core$constantly$fn__5672@483b7dc4"] (gen/gvoid)) (gen/delayfn #object[clojure.core$constantly$fn__5672 0x6b8bdcc6 "clojure.core$constantly$fn__5672@6b8bdcc6"] (gen/gvoid)) (gen/set-atom-stop)))) (gen/on #object[clojure.core$complement$fn__5669 0x24a3d414 "clojure.core$complement$fn__5669@24a3d414"] (gen/while-atom-continue #object[jepsen.independent$concurrent_generator$reify__2879 0x37e55819 "jepsen.independent$concurrent_generator$reify__2879@37e55819"]))]), :collect-data-files false, :doc-threads 3, :mixed-txns false, :kv-timeout #object[java.time.Duration 0x20027c44 "PT2S"], :os #object[jepsen.os$reify__2349 0x54defd69 "jepsen.os$reify__2349@54defd69"], :server-group-count nil, :time-limit 60, :transactions false, :use-json-docs true, :eviction-policy "valueOnly", :workload "failover", :recovery-type :delta, :max-txn-size 3, :enable-tcp-capture false, :target-server-groups false, :model {:value :nil}} INFO [2019-12-17 12:27:02,820] jepsen test runner - jepsen.db Tearing down DB INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.128 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.125 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.127 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:02,822] jepsen node 172.28.128.126 - couchbase.util Tearing down couchbase node INFO [2019-12-17 12:27:05,866] jepsen node 172.28.128.128 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:06,638] jepsen node 172.28.128.127 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:06,859] jepsen node 172.28.128.125 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:07,105] jepsen node 172.28.128.126 - couchbase.util Teardown Complete INFO [2019-12-17 12:27:07,108] jepsen test runner - jepsen.db Setting up DB INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.128 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.125 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.126 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.127 - couchbase.util Setting up Couchbase Server INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.128 - couchbase.util Installing package INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.125 - couchbase.util Installing package INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.126 - couchbase.util Installing package INFO [2019-12-17 12:27:07,109] jepsen node 172.28.128.127 - couchbase.util Installing package INFO [2019-12-17 12:27:17,814] jepsen node 172.28.128.125 - couchbase.util Package installed INFO [2019-12-17 12:27:17,819] jepsen node 172.28.128.127 - couchbase.util Package installed INFO [2019-12-17 12:27:17,819] jepsen node 172.28.128.126 - couchbase.util Package installed INFO [2019-12-17 12:27:17,866] jepsen node 172.28.128.125 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,866] jepsen node 172.28.128.127 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,866] jepsen node 172.28.128.126 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,928] jepsen node 172.28.128.128 - couchbase.util Package installed INFO [2019-12-17 12:27:17,928] jepsen node 172.28.128.128 - couchbase.util Setting up data paths INFO [2019-12-17 12:27:17,974] jepsen node 172.28.128.126 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:17,974] jepsen node 172.28.128.127 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:17,974] jepsen node 172.28.128.125 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:18,037] jepsen node 172.28.128.128 - couchbase.util Changing permissions for /opt/couchbase INFO [2019-12-17 12:27:18,146] jepsen node 172.28.128.128 - couchbase.util Starting daemon INFO [2019-12-17 12:27:18,193] jepsen node 172.28.128.126 - couchbase.util Starting daemon INFO [2019-12-17 12:27:18,193] jepsen node 172.28.128.125 - couchbase.util Starting daemon INFO [2019-12-17 12:27:18,193] jepsen node 172.28.128.127 - couchbase.util Starting daemon INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.128 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.126 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.125 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.128 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.126 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,381] jepsen node 172.28.128.125 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,382] jepsen node 172.28.128.127 - couchbase.util Daemon started INFO [2019-12-17 12:27:24,382] jepsen node 172.28.128.127 - couchbase.util TCP packet capture is not enabled INFO [2019-12-17 12:27:24,382] jepsen test runner - jepsen.db Setting up primary 172.28.128.125 INFO [2019-12-17 12:27:24,383] jepsen node 172.28.128.125 - couchbase.util Creating couchbase cluster from 172.28.128.125 INFO [2019-12-17 12:27:24,471] jepsen node 172.28.128.125 - couchbase.util Adding node 172.28.128.126 to cluster INFO [2019-12-17 12:27:26,350] jepsen node 172.28.128.125 - couchbase.util Adding node 172.28.128.127 to cluster INFO [2019-12-17 12:27:27,960] jepsen node 172.28.128.125 - couchbase.util Adding node 172.28.128.128 to cluster INFO [2019-12-17 12:27:29,497] jepsen node 172.28.128.125 - couchbase.util Rebalance status: {} INFO [2019-12-17 12:27:29,498] jepsen node 172.28.128.125 - couchbase.util Rebalance check count: 0 INFO [2019-12-17 12:27:29,498] jepsen node 172.28.128.125 - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance status: {:status none} INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance check count: 1 INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:30,520] jepsen node 172.28.128.125 - couchbase.util Rebalance complete INFO [2019-12-17 12:27:30,549] jepsen node 172.28.128.125 - couchbase.util Waiting for bucket warmup to complete... INFO [2019-12-17 12:27:30,584] jepsen node 172.28.128.125 - couchbase.util Setup complete INFO [2019-12-17 12:27:30,588] jepsen test runner - jepsen.core Relative time begins now INFO [2019-12-17 12:27:30,598] jepsen nemesis - jepsen.core Setting up nemesis INFO [2019-12-17 12:27:30,599] jepsen worker 0 - jepsen.core Setting up worker 0 INFO [2019-12-17 12:27:30,600] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:30,601] jepsen worker 1 - jepsen.core Setting up worker 1 INFO [2019-12-17 12:27:30,602] jepsen worker 2 - jepsen.core Setting up worker 2 INFO [2019-12-17 12:27:30,603] jepsen worker 3 - jepsen.core Setting up worker 3 INFO [2019-12-17 12:27:30,603] jepsen worker 4 - jepsen.core Setting up worker 4 INFO [2019-12-17 12:27:30,604] jepsen worker 5 - jepsen.core Setting up worker 5 INFO [2019-12-17 12:27:30,605] jepsen worker 6 - jepsen.core Setting up worker 6 INFO [2019-12-17 12:27:30,605] jepsen worker 7 - jepsen.core Setting up worker 7 INFO [2019-12-17 12:27:30,605] jepsen worker 8 - jepsen.core Setting up worker 8 INFO [2019-12-17 12:27:30,606] jepsen worker 9 - jepsen.core Setting up worker 9 INFO [2019-12-17 12:27:30,607] jepsen worker 10 - jepsen.core Setting up worker 10 INFO [2019-12-17 12:27:30,608] jepsen worker 11 - jepsen.core Setting up worker 11 INFO [2019-12-17 12:27:30,609] jepsen worker 12 - jepsen.core Setting up worker 12 INFO [2019-12-17 12:27:30,610] jepsen worker 13 - jepsen.core Setting up worker 13 INFO [2019-12-17 12:27:30,611] jepsen worker 14 - jepsen.core Setting up worker 14 INFO [2019-12-17 12:27:30,614] jepsen worker 15 - jepsen.core Setting up worker 15 INFO [2019-12-17 12:27:30,615] jepsen worker 16 - jepsen.core Setting up worker 16 INFO [2019-12-17 12:27:30,616] jepsen worker 17 - jepsen.core Setting up worker 17 INFO [2019-12-17 12:27:30,616] jepsen worker 18 - jepsen.core Setting up worker 18 INFO [2019-12-17 12:27:30,616] jepsen worker 19 - jepsen.core Setting up worker 19 INFO [2019-12-17 12:27:30,617] jepsen worker 20 - jepsen.core Setting up worker 20 INFO [2019-12-17 12:27:30,617] jepsen worker 21 - jepsen.core Setting up worker 21 INFO [2019-12-17 12:27:30,618] jepsen worker 22 - jepsen.core Setting up worker 22 INFO [2019-12-17 12:27:30,618] jepsen worker 23 - jepsen.core Setting up worker 23 INFO [2019-12-17 12:27:30,619] jepsen worker 24 - jepsen.core Setting up worker 24 INFO [2019-12-17 12:27:30,619] jepsen worker 25 - jepsen.core Setting up worker 25 INFO [2019-12-17 12:27:30,619] jepsen worker 26 - jepsen.core Setting up worker 26 INFO [2019-12-17 12:27:30,622] jepsen worker 28 - jepsen.core Setting up worker 28 INFO [2019-12-17 12:27:30,622] jepsen worker 29 - jepsen.core Setting up worker 29 INFO [2019-12-17 12:27:30,623] jepsen worker 30 - jepsen.core Setting up worker 30 INFO [2019-12-17 12:27:30,623] jepsen worker 27 - jepsen.core Setting up worker 27 INFO [2019-12-17 12:27:30,623] jepsen worker 31 - jepsen.core Setting up worker 31 INFO [2019-12-17 12:27:30,623] jepsen worker 32 - jepsen.core Setting up worker 32 INFO [2019-12-17 12:27:30,624] jepsen worker 33 - jepsen.core Setting up worker 33 INFO [2019-12-17 12:27:30,625] jepsen worker 34 - jepsen.core Setting up worker 34 INFO [2019-12-17 12:27:30,625] jepsen worker 35 - jepsen.core Setting up worker 35 INFO [2019-12-17 12:27:30,626] jepsen worker 36 - jepsen.core Setting up worker 36 INFO [2019-12-17 12:27:30,626] jepsen worker 37 - jepsen.core Setting up worker 37 INFO [2019-12-17 12:27:30,627] jepsen worker 38 - jepsen.core Setting up worker 38 INFO [2019-12-17 12:27:30,627] jepsen worker 39 - jepsen.core Setting up worker 39 INFO [2019-12-17 12:27:30,627] jepsen worker 40 - jepsen.core Setting up worker 40 INFO [2019-12-17 12:27:30,628] jepsen worker 41 - jepsen.core Setting up worker 41 INFO [2019-12-17 12:27:30,629] jepsen worker 42 - jepsen.core Setting up worker 42 INFO [2019-12-17 12:27:30,630] jepsen worker 43 - jepsen.core Setting up worker 43 INFO [2019-12-17 12:27:30,630] jepsen worker 44 - jepsen.core Setting up worker 44 INFO [2019-12-17 12:27:30,630] jepsen worker 45 - jepsen.core Setting up worker 45 INFO [2019-12-17 12:27:30,631] jepsen worker 46 - jepsen.core Setting up worker 46 INFO [2019-12-17 12:27:30,632] jepsen worker 47 - jepsen.core Setting up worker 47 INFO [2019-12-17 12:27:30,632] jepsen worker 48 - jepsen.core Setting up worker 48 INFO [2019-12-17 12:27:30,633] jepsen worker 49 - jepsen.core Setting up worker 49 INFO [2019-12-17 12:27:30,634] jepsen worker 51 - jepsen.core Setting up worker 51 INFO [2019-12-17 12:27:30,637] jepsen worker 52 - jepsen.core Setting up worker 52 INFO [2019-12-17 12:27:30,637] jepsen worker 53 - jepsen.core Setting up worker 53 INFO [2019-12-17 12:27:30,638] jepsen worker 54 - jepsen.core Setting up worker 54 INFO [2019-12-17 12:27:30,639] jepsen worker 55 - jepsen.core Setting up worker 55 INFO [2019-12-17 12:27:30,640] jepsen worker 50 - jepsen.core Setting up worker 50 INFO [2019-12-17 12:27:30,640] jepsen worker 56 - jepsen.core Setting up worker 56 INFO [2019-12-17 12:27:30,640] jepsen worker 57 - jepsen.core Setting up worker 57 INFO [2019-12-17 12:27:30,641] jepsen worker 58 - jepsen.core Setting up worker 58 INFO [2019-12-17 12:27:30,641] jepsen worker 59 - jepsen.core Setting up worker 59 INFO [2019-12-17 12:27:30,642] jepsen worker 60 - jepsen.core Setting up worker 60 INFO [2019-12-17 12:27:30,642] jepsen worker 61 - jepsen.core Setting up worker 61 INFO [2019-12-17 12:27:30,642] jepsen worker 62 - jepsen.core Setting up worker 62 INFO [2019-12-17 12:27:30,643] jepsen worker 64 - jepsen.core Setting up worker 64 INFO [2019-12-17 12:27:30,644] jepsen worker 65 - jepsen.core Setting up worker 65 INFO [2019-12-17 12:27:30,644] jepsen worker 63 - jepsen.core Setting up worker 63 INFO [2019-12-17 12:27:30,644] jepsen worker 66 - jepsen.core Setting up worker 66 INFO [2019-12-17 12:27:30,644] jepsen worker 67 - jepsen.core Setting up worker 67 INFO [2019-12-17 12:27:30,645] jepsen worker 68 - jepsen.core Setting up worker 68 INFO [2019-12-17 12:27:30,646] jepsen worker 69 - jepsen.core Setting up worker 69 INFO [2019-12-17 12:27:30,647] jepsen worker 70 - jepsen.core Setting up worker 70 INFO [2019-12-17 12:27:30,647] jepsen worker 71 - jepsen.core Setting up worker 71 INFO [2019-12-17 12:27:30,647] jepsen worker 72 - jepsen.core Setting up worker 72 INFO [2019-12-17 12:27:30,648] jepsen worker 73 - jepsen.core Setting up worker 73 INFO [2019-12-17 12:27:30,648] jepsen worker 74 - jepsen.core Setting up worker 74 INFO [2019-12-17 12:27:30,648] jepsen worker 75 - jepsen.core Setting up worker 75 INFO [2019-12-17 12:27:30,648] jepsen worker 76 - jepsen.core Setting up worker 76 INFO [2019-12-17 12:27:30,648] jepsen worker 77 - jepsen.core Setting up worker 77 INFO [2019-12-17 12:27:30,648] jepsen worker 78 - jepsen.core Setting up worker 78 INFO [2019-12-17 12:27:30,649] jepsen worker 79 - jepsen.core Setting up worker 79 INFO [2019-12-17 12:27:30,649] jepsen worker 80 - jepsen.core Setting up worker 80 INFO [2019-12-17 12:27:30,649] jepsen worker 81 - jepsen.core Setting up worker 81 INFO [2019-12-17 12:27:30,649] jepsen worker 82 - jepsen.core Setting up worker 82 INFO [2019-12-17 12:27:30,649] jepsen worker 83 - jepsen.core Setting up worker 83 INFO [2019-12-17 12:27:30,649] jepsen worker 84 - jepsen.core Setting up worker 84 INFO [2019-12-17 12:27:30,650] jepsen worker 85 - jepsen.core Setting up worker 85 INFO [2019-12-17 12:27:30,652] jepsen worker 86 - jepsen.core Setting up worker 86 INFO [2019-12-17 12:27:30,652] jepsen worker 87 - jepsen.core Setting up worker 87 INFO [2019-12-17 12:27:30,656] jepsen worker 88 - jepsen.core Setting up worker 88 INFO [2019-12-17 12:27:30,657] jepsen worker 89 - jepsen.core Setting up worker 89 INFO [2019-12-17 12:27:31,458] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":1} INFO [2019-12-17 12:27:31,469] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:36,790] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:36,791] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:36,791] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":1,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:36,976] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,016] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][173651µs] Opened bucket "default" {"coreId":1} INFO [2019-12-17 12:27:37,086] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":2} INFO [2019-12-17 12:27:37,086] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,087] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,194] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,197] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,197] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":2,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,206] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][95096µs] Opened bucket "default" {"coreId":2} INFO [2019-12-17 12:27:37,306] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":3} INFO [2019-12-17 12:27:37,306] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,307] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,409] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,418] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,419] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":3,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,424] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][90347µs] Opened bucket "default" {"coreId":3} INFO [2019-12-17 12:27:37,520] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":4} INFO [2019-12-17 12:27:37,520] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,623] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,630] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,631] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,631] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":4,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,635] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][88674µs] Opened bucket "default" {"coreId":4} INFO [2019-12-17 12:27:37,734] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":5} INFO [2019-12-17 12:27:37,734] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:37,735] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,840] jepsen worker 0 - couchbase.cbclients Opening new client INFO [2019-12-17 12:27:37,844] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:37,844] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":5,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:37,847] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][93826µs] Opened bucket "default" {"coreId":5} INFO [2019-12-17 12:27:37,954] cb-events - com.couchbase.core [com.couchbase.core][CoreCreatedEvent] {"clientVersion":"3.0.0-beta.1","clientGitHash":"a6fd12eb","coreVersion":"2.0.0-beta.1","coreGitHash":"a6fd12eb","userAgent":"couchbase-java/3.0.0-beta.1 (Mac OS X 10.14.6 x86_64; OpenJDK 64-Bit Server VM 1.8.0_222-b10)","ioEnvironment":{"nativeIoEnabled":true,"eventLoopGroups":["KQueueEventLoopGroup"]},"ioConfig":{"allowedSaslMechs":["PLAIN","CRAM_MD5","SCRAM_SHA1","SCRAM_SHA256","SCRAM_SHA512"],"captureTraffic":[],"mutationTokensEnabled":true,"networkResolution":"auto","dnsSrvEnabled":false,"configPollIntervalMillis":2500,"kvCircuitBreakerConfig":"disabled","queryCircuitBreakerConfig":"disabled","viewCircuitBreakerConfig":"disabled","searchCircuitBreakerConfig":"disabled","analyticsCircuitBreakerConfig":"disabled","managerCircuitBreakerConfig":"disabled"},"compressionConfig":{"enabled":true,"minRatio":0.83,"minSize":32},"securityConfig":{"tlsEnabled":false,"nativeTlsEnabled":true,"hasKey":false,"hasTrustCertificates":false,"trustManagerFactory":null,"keyManagerFactory":null},"timeoutConfig":{"kvMs":2000,"managementMs":75000,"queryMs":75000,"viewMs":75000,"searchMs":75000,"analyticsMs":75000,"connectMs":10000,"disconnectMs":10000},"serviceConfig":{"kv":{"min":1,"max":1,"idleTimeMs":0},"query":{"min":0,"max":12,"idleTimeMs":300000},"view":{"min":0,"max":12,"idleTimeMs":300000},"search":{"min":0,"max":12,"idleTimeMs":300000},"analytics":{"min":0,"max":12,"idleTimeMs":300000}},"loggerConfig":{"customLogger":null,"fallbackToConsole":false,"disableSlf4j":false,"loggerName":"CouchbaseLogger","diagnosticContextEnabled":false},"retryStrategy":"BestEffortRetryStrategy"} {"coreId":6} INFO [2019-12-17 12:27:37,954] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.125"} INFO [2019-12-17 12:27:37,954] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.126"} INFO [2019-12-17 12:27:38,064] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.127"} INFO [2019-12-17 12:27:38,064] cb-events - com.couchbase.node [com.couchbase.node][NodeConnectedEvent] Node connected {"coreId":6,"managerPort":"8091","remote":"172.28.128.128"} INFO [2019-12-17 12:27:38,067] cb-events - com.couchbase.core [com.couchbase.core][BucketOpenedEvent][96689µs] Opened bucket "default" {"coreId":6} INFO [2019-12-17 12:27:43,074] jepsen nemesis - jepsen.core Running nemesis INFO [2019-12-17 12:27:43,074] jepsen worker 0 - jepsen.core Running worker 0 INFO [2019-12-17 12:27:43,074] jepsen worker 1 - jepsen.core Running worker 1 INFO [2019-12-17 12:27:43,075] jepsen worker 2 - jepsen.core Running worker 2 INFO [2019-12-17 12:27:43,075] jepsen worker 3 - jepsen.core Running worker 3 INFO [2019-12-17 12:27:43,075] jepsen worker 4 - jepsen.core Running worker 4 INFO [2019-12-17 12:27:43,075] jepsen worker 5 - jepsen.core Running worker 5 INFO [2019-12-17 12:27:43,075] jepsen worker 6 - jepsen.core Running worker 6 INFO [2019-12-17 12:27:43,075] jepsen worker 7 - jepsen.core Running worker 7 INFO [2019-12-17 12:27:43,075] jepsen worker 11 - jepsen.core Running worker 11 INFO [2019-12-17 12:27:43,076] jepsen worker 9 - jepsen.core Running worker 9 INFO [2019-12-17 12:27:43,076] jepsen worker 12 - jepsen.core Running worker 12 INFO [2019-12-17 12:27:43,076] jepsen worker 13 - jepsen.core Running worker 13 INFO [2019-12-17 12:27:43,076] jepsen worker 10 - jepsen.core Running worker 10 INFO [2019-12-17 12:27:43,076] jepsen worker 14 - jepsen.core Running worker 14 INFO [2019-12-17 12:27:43,076] jepsen worker 15 - jepsen.core Running worker 15 INFO [2019-12-17 12:27:43,076] jepsen worker 16 - jepsen.core Running worker 16 INFO [2019-12-17 12:27:43,076] jepsen worker 8 - jepsen.core Running worker 8 INFO [2019-12-17 12:27:43,076] jepsen worker 17 - jepsen.core Running worker 17 INFO [2019-12-17 12:27:43,076] jepsen worker 18 - jepsen.core Running worker 18 INFO [2019-12-17 12:27:43,076] jepsen worker 19 - jepsen.core Running worker 19 INFO [2019-12-17 12:27:43,076] jepsen worker 20 - jepsen.core Running worker 20 INFO [2019-12-17 12:27:43,076] jepsen worker 21 - jepsen.core Running worker 21 INFO [2019-12-17 12:27:43,077] jepsen worker 22 - jepsen.core Running worker 22 INFO [2019-12-17 12:27:43,077] jepsen worker 23 - jepsen.core Running worker 23 INFO [2019-12-17 12:27:43,077] jepsen worker 25 - jepsen.core Running worker 25 INFO [2019-12-17 12:27:43,077] jepsen worker 26 - jepsen.core Running worker 26 INFO [2019-12-17 12:27:43,077] jepsen worker 27 - jepsen.core Running worker 27 INFO [2019-12-17 12:27:43,077] jepsen worker 28 - jepsen.core Running worker 28 INFO [2019-12-17 12:27:43,077] jepsen worker 29 - jepsen.core Running worker 29 INFO [2019-12-17 12:27:43,077] jepsen worker 30 - jepsen.core Running worker 30 INFO [2019-12-17 12:27:43,077] jepsen worker 31 - jepsen.core Running worker 31 INFO [2019-12-17 12:27:43,077] jepsen worker 24 - jepsen.core Running worker 24 INFO [2019-12-17 12:27:43,078] jepsen worker 32 - jepsen.core Running worker 32 INFO [2019-12-17 12:27:43,078] jepsen worker 33 - jepsen.core Running worker 33 INFO [2019-12-17 12:27:43,078] jepsen worker 34 - jepsen.core Running worker 34 INFO [2019-12-17 12:27:43,078] jepsen worker 35 - jepsen.core Running worker 35 INFO [2019-12-17 12:27:43,078] jepsen worker 36 - jepsen.core Running worker 36 INFO [2019-12-17 12:27:43,078] jepsen worker 37 - jepsen.core Running worker 37 INFO [2019-12-17 12:27:43,079] jepsen worker 38 - jepsen.core Running worker 38 INFO [2019-12-17 12:27:43,079] jepsen worker 39 - jepsen.core Running worker 39 INFO [2019-12-17 12:27:43,079] jepsen worker 40 - jepsen.core Running worker 40 INFO [2019-12-17 12:27:43,079] jepsen worker 41 - jepsen.core Running worker 41 INFO [2019-12-17 12:27:43,079] jepsen worker 42 - jepsen.core Running worker 42 INFO [2019-12-17 12:27:43,079] jepsen worker 43 - jepsen.core Running worker 43 INFO [2019-12-17 12:27:43,079] jepsen worker 44 - jepsen.core Running worker 44 INFO [2019-12-17 12:27:43,079] jepsen worker 45 - jepsen.core Running worker 45 INFO [2019-12-17 12:27:43,079] jepsen worker 46 - jepsen.core Running worker 46 INFO [2019-12-17 12:27:43,079] jepsen worker 47 - jepsen.core Running worker 47 INFO [2019-12-17 12:27:43,080] jepsen worker 48 - jepsen.core Running worker 48 INFO [2019-12-17 12:27:43,080] jepsen worker 51 - jepsen.core Running worker 51 INFO [2019-12-17 12:27:43,080] jepsen worker 52 - jepsen.core Running worker 52 INFO [2019-12-17 12:27:43,080] jepsen worker 53 - jepsen.core Running worker 53 INFO [2019-12-17 12:27:43,080] jepsen worker 54 - jepsen.core Running worker 54 INFO [2019-12-17 12:27:43,080] jepsen worker 49 - jepsen.core Running worker 49 INFO [2019-12-17 12:27:43,080] jepsen worker 55 - jepsen.core Running worker 55 INFO [2019-12-17 12:27:43,080] jepsen worker 56 - jepsen.core Running worker 56 INFO [2019-12-17 12:27:43,081] jepsen worker 57 - jepsen.core Running worker 57 INFO [2019-12-17 12:27:43,081] jepsen worker 58 - jepsen.core Running worker 58 INFO [2019-12-17 12:27:43,081] jepsen worker 59 - jepsen.core Running worker 59 INFO [2019-12-17 12:27:43,081] jepsen worker 60 - jepsen.core Running worker 60 INFO [2019-12-17 12:27:43,082] jepsen worker 61 - jepsen.core Running worker 61 INFO [2019-12-17 12:27:43,082] jepsen worker 62 - jepsen.core Running worker 62 INFO [2019-12-17 12:27:43,082] jepsen worker 64 - jepsen.core Running worker 64 INFO [2019-12-17 12:27:43,082] jepsen worker 50 - jepsen.core Running worker 50 INFO [2019-12-17 12:27:43,082] jepsen worker 65 - jepsen.core Running worker 65 INFO [2019-12-17 12:27:43,082] jepsen worker 66 - jepsen.core Running worker 66 INFO [2019-12-17 12:27:43,082] jepsen worker 67 - jepsen.core Running worker 67 INFO [2019-12-17 12:27:43,082] jepsen worker 68 - jepsen.core Running worker 68 INFO [2019-12-17 12:27:43,082] jepsen worker 69 - jepsen.core Running worker 69 INFO [2019-12-17 12:27:43,082] jepsen worker 70 - jepsen.core Running worker 70 INFO [2019-12-17 12:27:43,082] jepsen worker 63 - jepsen.core Running worker 63 INFO [2019-12-17 12:27:43,082] jepsen worker 71 - jepsen.core Running worker 71 INFO [2019-12-17 12:27:43,082] jepsen worker 72 - jepsen.core Running worker 72 INFO [2019-12-17 12:27:43,082] jepsen worker 73 - jepsen.core Running worker 73 INFO [2019-12-17 12:27:43,082] jepsen worker 74 - jepsen.core Running worker 74 INFO [2019-12-17 12:27:43,082] jepsen worker 75 - jepsen.core Running worker 75 INFO [2019-12-17 12:27:43,082] jepsen worker 76 - jepsen.core Running worker 76 INFO [2019-12-17 12:27:43,083] jepsen worker 77 - jepsen.core Running worker 77 INFO [2019-12-17 12:27:43,083] jepsen worker 78 - jepsen.core Running worker 78 INFO [2019-12-17 12:27:43,083] jepsen worker 79 - jepsen.core Running worker 79 INFO [2019-12-17 12:27:43,083] jepsen worker 80 - jepsen.core Running worker 80 INFO [2019-12-17 12:27:43,083] jepsen worker 81 - jepsen.core Running worker 81 INFO [2019-12-17 12:27:43,083] jepsen worker 82 - jepsen.core Running worker 82 INFO [2019-12-17 12:27:43,083] jepsen worker 83 - jepsen.core Running worker 83 INFO [2019-12-17 12:27:43,083] jepsen worker 84 - jepsen.core Running worker 84 INFO [2019-12-17 12:27:43,083] jepsen worker 85 - jepsen.core Running worker 85 INFO [2019-12-17 12:27:43,083] jepsen worker 86 - jepsen.core Running worker 86 INFO [2019-12-17 12:27:43,083] jepsen worker 87 - jepsen.core Running worker 87 INFO [2019-12-17 12:27:43,083] jepsen worker 88 - jepsen.core Running worker 88 INFO [2019-12-17 12:27:43,083] jepsen worker 89 - jepsen.core Running worker 89 INFO [2019-12-17 12:27:43,102] jepsen worker 14 - jepsen.util 14 :invoke :read [4 nil] INFO [2019-12-17 12:27:43,102] jepsen worker 4 - jepsen.util 4 :invoke :read [1 nil] INFO [2019-12-17 12:27:43,107] jepsen worker 2 - jepsen.util 2 :invoke :cas [0 [44 32]] INFO [2019-12-17 12:27:43,107] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,107] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 4 - jepsen.util 4 :ok :read [1 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 14 - jepsen.util 14 :ok :read [4 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 2 - jepsen.util 2 :fail :cas [0 [44 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,116] jepsen worker 35 - jepsen.util 35 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 24 - jepsen.util 24 :ok :read [8 :nil] INFO [2019-12-17 12:27:43,116] jepsen worker 14 - jepsen.util 14 :invoke :read [4 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 4 - jepsen.util 4 :invoke :read [1 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 2 - jepsen.util 2 :invoke :read [0 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,116] jepsen worker 24 - jepsen.util 24 :invoke :write [8 14] INFO [2019-12-17 12:27:43,117] jepsen worker 6 - jepsen.util 6 :invoke :write [2 15] INFO [2019-12-17 12:27:43,117] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,117] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,117] jepsen worker 36 - jepsen.util 36 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,119] jepsen worker 35 - jepsen.util 35 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,119] jepsen worker 14 - jepsen.util 14 :ok :read [4 :nil] INFO [2019-12-17 12:27:43,119] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,120] jepsen worker 14 - jepsen.util 14 :invoke :write [4 24] INFO [2019-12-17 12:27:43,121] jepsen worker 2 - jepsen.util 2 :ok :read [0 :nil] INFO [2019-12-17 12:27:43,122] jepsen worker 27 - jepsen.util 27 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,122] jepsen worker 2 - jepsen.util 2 :invoke :write [0 34] INFO [2019-12-17 12:27:43,125] jepsen worker 34 - jepsen.util 34 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,128] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,132] jepsen worker 36 - jepsen.util 36 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,133] jepsen worker 35 - jepsen.util 35 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,133] jepsen worker 4 - jepsen.util 4 :ok :read [1 :nil] INFO [2019-12-17 12:27:43,133] jepsen worker 34 - jepsen.util 34 :invoke :cas [11 [31 17]] INFO [2019-12-17 12:27:43,134] jepsen worker 36 - jepsen.util 36 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,134] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [9 41]] INFO [2019-12-17 12:27:43,134] jepsen worker 4 - jepsen.util 4 :invoke :write [1 23] INFO [2019-12-17 12:27:43,138] jepsen worker 27 - jepsen.util 27 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,139] jepsen worker 36 - jepsen.util 36 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,139] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [9 41]] :KeyNotFoundException INFO [2019-12-17 12:27:43,139] jepsen worker 34 - jepsen.util 34 :fail :cas [11 [31 17]] :KeyNotFoundException INFO [2019-12-17 12:27:43,139] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [3 16]] INFO [2019-12-17 12:27:43,139] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [18 40]] INFO [2019-12-17 12:27:43,139] jepsen worker 36 - jepsen.util 36 :invoke :cas [12 [22 33]] INFO [2019-12-17 12:27:43,139] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:43,145] jepsen worker 34 - jepsen.util 34 :ok :read [11 :nil] INFO [2019-12-17 12:27:43,145] jepsen worker 36 - jepsen.util 36 :fail :cas [12 [22 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,145] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [3 16]] :KeyNotFoundException INFO [2019-12-17 12:27:43,147] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [18 40]] :KeyNotFoundException INFO [2019-12-17 12:27:43,147] jepsen worker 34 - jepsen.util 34 :invoke :cas [11 [42 8]] INFO [2019-12-17 12:27:43,147] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,147] jepsen worker 36 - jepsen.util 36 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,147] jepsen worker 35 - jepsen.util 35 :invoke :write [11 22] INFO [2019-12-17 12:27:43,147] jepsen worker 26 - jepsen.util 26 :invoke :read [8 nil] INFO [2019-12-17 12:27:43,147] jepsen worker 1 - jepsen.util 1 :invoke :cas [0 [19 9]] INFO [2019-12-17 12:27:43,150] jepsen worker 26 - jepsen.util 26 :ok :read [8 :nil] INFO [2019-12-17 12:27:43,150] jepsen worker 27 - jepsen.util 27 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,150] jepsen worker 34 - jepsen.util 34 :fail :cas [11 [42 8]] :KeyNotFoundException INFO [2019-12-17 12:27:43,153] jepsen worker 36 - jepsen.util 36 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,153] jepsen worker 1 - jepsen.util 1 :fail :cas [0 [19 9]] :KeyNotFoundException INFO [2019-12-17 12:27:43,153] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [32 33]] INFO [2019-12-17 12:27:43,153] jepsen worker 26 - jepsen.util 26 :invoke :cas [8 [47 16]] INFO [2019-12-17 12:27:43,153] jepsen worker 34 - jepsen.util 34 :invoke :write [11 7] INFO [2019-12-17 12:27:43,153] jepsen worker 36 - jepsen.util 36 :invoke :cas [12 [47 31]] INFO [2019-12-17 12:27:43,153] jepsen worker 1 - jepsen.util 1 :invoke :read [0 nil] INFO [2019-12-17 12:27:43,158] jepsen worker 1 - jepsen.util 1 :ok :read [0 :nil] INFO [2019-12-17 12:27:43,158] jepsen worker 26 - jepsen.util 26 :fail :cas [8 [47 16]] :KeyNotFoundException INFO [2019-12-17 12:27:43,158] jepsen worker 36 - jepsen.util 36 :fail :cas [12 [47 31]] :KeyNotFoundException INFO [2019-12-17 12:27:43,158] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [32 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,159] jepsen worker 1 - jepsen.util 1 :invoke :write [0 14] INFO [2019-12-17 12:27:43,159] jepsen worker 26 - jepsen.util 26 :invoke :cas [8 [30 45]] INFO [2019-12-17 12:27:43,159] jepsen worker 36 - jepsen.util 36 :invoke :write [12 0] INFO [2019-12-17 12:27:43,159] jepsen worker 27 - jepsen.util 27 :invoke :write [9 7] INFO [2019-12-17 12:27:43,161] jepsen worker 26 - jepsen.util 26 :fail :cas [8 [30 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,162] jepsen worker 26 - jepsen.util 26 :invoke :read [8 nil] INFO [2019-12-17 12:27:43,164] jepsen worker 23 - jepsen.util 23 :invoke :write [7 48] INFO [2019-12-17 12:27:43,164] jepsen worker 9 - jepsen.util 9 :invoke :read [3 nil] INFO [2019-12-17 12:27:43,168] jepsen worker 29 - jepsen.util 29 :invoke :cas [9 [28 28]] INFO [2019-12-17 12:27:43,174] jepsen worker 8 - jepsen.util 8 :invoke :read [2 nil] INFO [2019-12-17 12:27:43,177] jepsen worker 26 - jepsen.util 26 :ok :read [8 :nil] INFO [2019-12-17 12:27:43,177] jepsen worker 26 - jepsen.util 26 :invoke :write [8 25] INFO [2019-12-17 12:27:43,184] jepsen worker 9 - jepsen.util 9 :ok :read [3 :nil] INFO [2019-12-17 12:27:43,184] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [26 23]] INFO [2019-12-17 12:27:43,193] jepsen worker 44 - jepsen.util 44 :invoke :write [14 27] INFO [2019-12-17 12:27:43,198] jepsen worker 29 - jepsen.util 29 :fail :cas [9 [28 28]] :KeyNotFoundException INFO [2019-12-17 12:27:43,199] jepsen worker 29 - jepsen.util 29 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,207] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [26 23]] :KeyNotFoundException INFO [2019-12-17 12:27:43,207] jepsen worker 9 - jepsen.util 9 :invoke :read [3 nil] INFO [2019-12-17 12:27:43,210] jepsen worker 8 - jepsen.util 8 :ok :read [2 :nil] INFO [2019-12-17 12:27:43,210] jepsen worker 8 - jepsen.util 8 :invoke :cas [2 [18 36]] INFO [2019-12-17 12:27:43,215] jepsen worker 8 - jepsen.util 8 :fail :cas [2 [18 36]] :KeyNotFoundException INFO [2019-12-17 12:27:43,215] jepsen worker 9 - jepsen.util 9 :ok :read [3 :nil] INFO [2019-12-17 12:27:43,215] jepsen worker 29 - jepsen.util 29 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,215] jepsen worker 8 - jepsen.util 8 :invoke :read [2 nil] INFO [2019-12-17 12:27:43,216] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [14 42]] INFO [2019-12-17 12:27:43,217] jepsen worker 29 - jepsen.util 29 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,231] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [14 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,231] jepsen worker 29 - jepsen.util 29 :ok :read [9 :nil] INFO [2019-12-17 12:27:43,231] jepsen worker 8 - jepsen.util 8 :ok :read [2 :nil] INFO [2019-12-17 12:27:43,231] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [6 48]] INFO [2019-12-17 12:27:43,232] jepsen worker 29 - jepsen.util 29 :invoke :write [9 6] INFO [2019-12-17 12:27:43,232] jepsen worker 8 - jepsen.util 8 :invoke :write [2 0] INFO [2019-12-17 12:27:43,233] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [6 48]] :KeyNotFoundException INFO [2019-12-17 12:27:43,233] jepsen worker 9 - jepsen.util 9 :invoke :cas [3 [3 18]] INFO [2019-12-17 12:27:43,234] jepsen worker 9 - jepsen.util 9 :fail :cas [3 [3 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,234] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,234] jepsen worker 9 - jepsen.util 9 :invoke :read [3 nil] INFO [2019-12-17 12:27:43,235] jepsen worker 9 - jepsen.util 9 :ok :read [3 :nil] INFO [2019-12-17 12:27:43,235] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,235] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,236] jepsen worker 9 - jepsen.util 9 :invoke :write [3 2] INFO [2019-12-17 12:27:43,236] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,236] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,237] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,237] jepsen worker 83 - jepsen.util 83 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,238] jepsen worker 83 - jepsen.util 83 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,238] jepsen worker 83 - jepsen.util 83 :invoke :write [27 21] INFO [2019-12-17 12:27:43,269] jepsen worker 16 - jepsen.util 16 :invoke :cas [5 [20 12]] INFO [2019-12-17 12:27:43,269] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [28 45]] INFO [2019-12-17 12:27:43,269] jepsen worker 28 - jepsen.util 28 :invoke :write [9 30] INFO [2019-12-17 12:27:43,269] jepsen worker 19 - jepsen.util 19 :invoke :cas [6 [20 25]] INFO [2019-12-17 12:27:43,269] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,269] jepsen worker 37 - jepsen.util 37 :invoke :write [12 23] INFO [2019-12-17 12:27:43,269] jepsen worker 18 - jepsen.util 18 :invoke :write [6 47] INFO [2019-12-17 12:27:43,269] jepsen worker 40 - jepsen.util 40 :invoke :read [13 nil] INFO [2019-12-17 12:27:43,269] jepsen worker 30 - jepsen.util 30 :invoke :write [10 23] INFO [2019-12-17 12:27:43,269] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [21 33]] INFO [2019-12-17 12:27:43,269] jepsen worker 42 - jepsen.util 42 :invoke :cas [14 [16 32]] INFO [2019-12-17 12:27:43,269] jepsen worker 22 - jepsen.util 22 :invoke :write [7 23] INFO [2019-12-17 12:27:43,269] jepsen worker 12 - jepsen.util 12 :invoke :cas [4 [32 1]] INFO [2019-12-17 12:27:43,269] jepsen worker 41 - jepsen.util 41 :invoke :write [13 9] INFO [2019-12-17 12:27:43,269] jepsen worker 17 - jepsen.util 17 :invoke :cas [5 [11 19]] INFO [2019-12-17 12:27:43,269] jepsen worker 32 - jepsen.util 32 :invoke :write [10 32] INFO [2019-12-17 12:27:43,269] jepsen worker 20 - jepsen.util 20 :invoke :cas [6 [24 0]] INFO [2019-12-17 12:27:43,269] jepsen worker 21 - jepsen.util 21 :invoke :write [7 20] INFO [2019-12-17 12:27:43,269] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [37 34]] INFO [2019-12-17 12:27:43,269] jepsen worker 15 - jepsen.util 15 :invoke :cas [5 [37 32]] INFO [2019-12-17 12:27:43,269] jepsen worker 0 - jepsen.util 0 :invoke :write [0 0] INFO [2019-12-17 12:27:43,269] jepsen worker 7 - jepsen.util 7 :invoke :write [2 33] INFO [2019-12-17 12:27:43,269] jepsen worker 43 - jepsen.util 43 :invoke :cas [14 [3 37]] INFO [2019-12-17 12:27:43,269] jepsen worker 31 - jepsen.util 31 :invoke :cas [10 [32 37]] INFO [2019-12-17 12:27:43,269] jepsen worker 33 - jepsen.util 33 :invoke :write [11 28] INFO [2019-12-17 12:27:43,269] jepsen worker 3 - jepsen.util 3 :invoke :write [1 29] INFO [2019-12-17 12:27:43,269] jepsen worker 13 - jepsen.util 13 :invoke :write [4 13] INFO [2019-12-17 12:27:43,269] jepsen worker 46 - jepsen.util 46 :invoke :read [15 nil] INFO [2019-12-17 12:27:43,269] jepsen worker 10 - jepsen.util 10 :invoke :write [3 42] INFO [2019-12-17 12:27:43,269] jepsen worker 45 - jepsen.util 45 :invoke :cas [15 [49 35]] INFO [2019-12-17 12:27:43,269] jepsen worker 48 - jepsen.util 48 :invoke :write [16 41] INFO [2019-12-17 12:27:43,270] jepsen worker 47 - jepsen.util 47 :invoke :read [15 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 51 - jepsen.util 51 :invoke :write [17 0] INFO [2019-12-17 12:27:43,270] jepsen worker 52 - jepsen.util 52 :invoke :cas [17 [36 33]] INFO [2019-12-17 12:27:43,270] jepsen worker 53 - jepsen.util 53 :invoke :write [17 31] INFO [2019-12-17 12:27:43,270] jepsen worker 49 - jepsen.util 49 :invoke :read [16 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 55 - jepsen.util 55 :invoke :write [18 34] INFO [2019-12-17 12:27:43,270] jepsen worker 57 - jepsen.util 57 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 59 - jepsen.util 59 :invoke :cas [19 [21 5]] INFO [2019-12-17 12:27:43,270] jepsen worker 60 - jepsen.util 60 :invoke :write [20 41] INFO [2019-12-17 12:27:43,270] jepsen worker 61 - jepsen.util 61 :invoke :write [20 7] INFO [2019-12-17 12:27:43,270] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [4 26]] INFO [2019-12-17 12:27:43,270] jepsen worker 38 - jepsen.util 38 :invoke :cas [12 [39 31]] INFO [2019-12-17 12:27:43,270] jepsen worker 65 - jepsen.util 65 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 67 - jepsen.util 67 :invoke :read [22 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 70 - jepsen.util 70 :invoke :cas [23 [2 45]] INFO [2019-12-17 12:27:43,270] jepsen worker 68 - jepsen.util 68 :invoke :write [22 41] INFO [2019-12-17 12:27:43,270] jepsen worker 71 - jepsen.util 71 :invoke :write [23 12] INFO [2019-12-17 12:27:43,270] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [16 18]] INFO [2019-12-17 12:27:43,271] jepsen worker 63 - jepsen.util 63 :invoke :cas [21 [10 46]] INFO [2019-12-17 12:27:43,271] jepsen worker 72 - jepsen.util 72 :invoke :cas [24 [38 21]] INFO [2019-12-17 12:27:43,271] jepsen worker 73 - jepsen.util 73 :invoke :cas [24 [39 37]] INFO [2019-12-17 12:27:43,271] jepsen worker 74 - jepsen.util 74 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,272] jepsen worker 19 - jepsen.util 19 :fail :cas [6 [20 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [21 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 16 - jepsen.util 16 :fail :cas [5 [20 12]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 75 - jepsen.util 75 :invoke :read [25 nil] INFO [2019-12-17 12:27:43,272] jepsen worker 40 - jepsen.util 40 :ok :read [13 :nil] INFO [2019-12-17 12:27:43,272] jepsen worker 42 - jepsen.util 42 :fail :cas [14 [16 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:43,272] jepsen worker 17 - jepsen.util 17 :fail :cas [5 [11 19]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 20 - jepsen.util 20 :fail :cas [6 [24 0]] :KeyNotFoundException INFO [2019-12-17 12:27:43,272] jepsen worker 77 - jepsen.util 77 :invoke :write [25 2] INFO [2019-12-17 12:27:43,273] jepsen worker 25 - jepsen.util 25 :invoke :write [8 8] INFO [2019-12-17 12:27:43,273] jepsen worker 78 - jepsen.util 78 :invoke :write [26 49] INFO [2019-12-17 12:27:43,273] jepsen worker 66 - jepsen.util 66 :invoke :write [22 40] INFO [2019-12-17 12:27:43,273] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [37 34]] :KeyNotFoundException INFO [2019-12-17 12:27:43,275] jepsen worker 79 - jepsen.util 79 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,276] jepsen worker 80 - jepsen.util 80 :invoke :cas [26 [20 39]] INFO [2019-12-17 12:27:43,280] jepsen worker 82 - jepsen.util 82 :invoke :cas [27 [33 26]] INFO [2019-12-17 12:27:43,280] jepsen worker 81 - jepsen.util 81 :invoke :cas [27 [46 6]] INFO [2019-12-17 12:27:43,282] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,282] jepsen worker 43 - jepsen.util 43 :fail :cas [14 [3 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,283] jepsen worker 85 - jepsen.util 85 :invoke :write [28 32] INFO [2019-12-17 12:27:43,284] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:43,284] jepsen worker 46 - jepsen.util 46 :ok :read [15 :nil] INFO [2019-12-17 12:27:43,287] jepsen worker 88 - jepsen.util 88 :invoke :write [29 40] INFO [2019-12-17 12:27:43,288] jepsen worker 45 - jepsen.util 45 :fail :cas [15 [49 35]] :KeyNotFoundException INFO [2019-12-17 12:27:43,290] jepsen worker 47 - jepsen.util 47 :ok :read [15 :nil] INFO [2019-12-17 12:27:43,293] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,295] jepsen worker 65 - jepsen.util 65 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,295] jepsen worker 67 - jepsen.util 67 :ok :read [22 :nil] INFO [2019-12-17 12:27:43,295] jepsen worker 70 - jepsen.util 70 :fail :cas [23 [2 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,295] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [16 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,312] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [22 45]] INFO [2019-12-17 12:27:43,312] jepsen worker 19 - jepsen.util 19 :invoke :write [6 45] INFO [2019-12-17 12:27:43,312] jepsen worker 16 - jepsen.util 16 :invoke :write [5 32] INFO [2019-12-17 12:27:43,312] jepsen worker 40 - jepsen.util 40 :invoke :cas [13 [30 18]] INFO [2019-12-17 12:27:43,312] jepsen worker 42 - jepsen.util 42 :invoke :write [14 29] INFO [2019-12-17 12:27:43,312] jepsen worker 17 - jepsen.util 17 :invoke :cas [5 [17 19]] INFO [2019-12-17 12:27:43,312] jepsen worker 20 - jepsen.util 20 :invoke :cas [6 [9 30]] INFO [2019-12-17 12:27:43,312] jepsen worker 38 - jepsen.util 38 :fail :cas [12 [39 31]] :KeyNotFoundException INFO [2019-12-17 12:27:43,312] jepsen worker 74 - jepsen.util 74 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,313] jepsen worker 72 - jepsen.util 72 :fail :cas [24 [38 21]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [4 26]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:43,313] jepsen worker 59 - jepsen.util 59 :fail :cas [19 [21 5]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 63 - jepsen.util 63 :fail :cas [21 [10 46]] :KeyNotFoundException INFO [2019-12-17 12:27:43,313] jepsen worker 75 - jepsen.util 75 :ok :read [25 :nil] INFO [2019-12-17 12:27:43,313] jepsen worker 76 - jepsen.util 76 :ok :read [25 :nil] INFO [2019-12-17 12:27:43,313] jepsen worker 43 - jepsen.util 43 :invoke :cas [14 [43 32]] INFO [2019-12-17 12:27:43,314] jepsen worker 46 - jepsen.util 46 :invoke :write [15 32] INFO [2019-12-17 12:27:43,314] jepsen worker 79 - jepsen.util 79 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,314] jepsen worker 80 - jepsen.util 80 :fail :cas [26 [20 39]] :KeyNotFoundException INFO [2019-12-17 12:27:43,314] jepsen worker 87 - jepsen.util 87 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,314] jepsen worker 45 - jepsen.util 45 :invoke :cas [15 [38 42]] INFO [2019-12-17 12:27:43,314] jepsen worker 47 - jepsen.util 47 :invoke :write [15 44] INFO [2019-12-17 12:27:43,314] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,314] jepsen worker 67 - jepsen.util 67 :invoke :read [22 nil] INFO [2019-12-17 12:27:43,314] jepsen worker 65 - jepsen.util 65 :invoke :cas [21 [10 43]] INFO [2019-12-17 12:27:43,314] jepsen worker 70 - jepsen.util 70 :invoke :write [23 29] INFO [2019-12-17 12:27:43,314] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 74 - jepsen.util 74 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 38 - jepsen.util 38 :invoke :read [12 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 72 - jepsen.util 72 :invoke :write [24 14] INFO [2019-12-17 12:27:43,315] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [43 38]] INFO [2019-12-17 12:27:43,315] jepsen worker 63 - jepsen.util 63 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,315] jepsen worker 59 - jepsen.util 59 :invoke :write [19 12] INFO [2019-12-17 12:27:43,315] jepsen worker 75 - jepsen.util 75 :invoke :cas [25 [30 38]] INFO [2019-12-17 12:27:43,315] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:43,316] jepsen worker 79 - jepsen.util 79 :invoke :cas [26 [35 34]] INFO [2019-12-17 12:27:43,316] jepsen worker 5 - jepsen.util 5 :ok :read [1 :nil] INFO [2019-12-17 12:27:43,316] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,316] jepsen worker 87 - jepsen.util 87 :invoke :cas [29 [6 8]] INFO [2019-12-17 12:27:43,316] jepsen worker 20 - jepsen.util 20 :fail :cas [6 [9 30]] :KeyNotFoundException INFO [2019-12-17 12:27:43,317] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [22 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,317] jepsen worker 45 - jepsen.util 45 :fail :cas [15 [38 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,320] jepsen worker 43 - jepsen.util 43 :fail :cas [14 [43 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,324] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,327] jepsen worker 38 - jepsen.util 38 :ok :read [12 :nil] INFO [2019-12-17 12:27:43,328] jepsen worker 5 - jepsen.util 5 :invoke :write [1 16] INFO [2019-12-17 12:27:43,328] jepsen worker 20 - jepsen.util 20 :invoke :read [6 nil] INFO [2019-12-17 12:27:43,328] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [16 42]] INFO [2019-12-17 12:27:43,329] jepsen worker 45 - jepsen.util 45 :invoke :write [15 35] INFO [2019-12-17 12:27:43,329] jepsen worker 79 - jepsen.util 79 :fail :cas [26 [35 34]] :KeyNotFoundException INFO [2019-12-17 12:27:43,329] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,329] jepsen worker 43 - jepsen.util 43 :invoke :write [14 43] INFO [2019-12-17 12:27:43,330] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,330] jepsen worker 87 - jepsen.util 87 :fail :cas [29 [6 8]] :KeyNotFoundException INFO [2019-12-17 12:27:43,330] jepsen worker 38 - jepsen.util 38 :invoke :cas [12 [29 37]] INFO [2019-12-17 12:27:43,330] jepsen worker 79 - jepsen.util 79 :invoke :cas [26 [47 4]] INFO [2019-12-17 12:27:43,331] jepsen worker 20 - jepsen.util 20 :ok :read [6 :nil] INFO [2019-12-17 12:27:43,331] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,331] jepsen worker 65 - jepsen.util 65 :fail :cas [21 [10 43]] :KeyNotFoundException INFO [2019-12-17 12:27:43,331] jepsen worker 74 - jepsen.util 74 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,332] jepsen worker 87 - jepsen.util 87 :invoke :cas [29 [30 7]] INFO [2019-12-17 12:27:43,333] jepsen worker 20 - jepsen.util 20 :invoke :write [6 3] INFO [2019-12-17 12:27:43,333] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,333] jepsen worker 65 - jepsen.util 65 :invoke :write [21 36] INFO [2019-12-17 12:27:43,334] jepsen worker 74 - jepsen.util 74 :invoke :write [24 35] INFO [2019-12-17 12:27:43,334] jepsen worker 79 - jepsen.util 79 :fail :cas [26 [47 4]] :KeyNotFoundException INFO [2019-12-17 12:27:43,334] jepsen worker 38 - jepsen.util 38 :fail :cas [12 [29 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,334] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,335] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,340] jepsen worker 87 - jepsen.util 87 :fail :cas [29 [30 7]] :KeyNotFoundException INFO [2019-12-17 12:27:43,340] jepsen worker 79 - jepsen.util 79 :invoke :write [26 47] INFO [2019-12-17 12:27:43,341] jepsen worker 38 - jepsen.util 38 :invoke :write [12 12] INFO [2019-12-17 12:27:43,341] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,341] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,344] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,344] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [20 47]] INFO [2019-12-17 12:27:43,345] jepsen worker 87 - jepsen.util 87 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,345] jepsen worker 87 - jepsen.util 87 :invoke :write [29 45] INFO [2019-12-17 12:27:43,346] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,346] jepsen worker 80 - jepsen.util 80 :invoke :read [26 nil] INFO [2019-12-17 12:27:43,347] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [20 47]] :KeyNotFoundException INFO [2019-12-17 12:27:43,347] jepsen worker 69 - jepsen.util 69 :invoke :read [23 nil] INFO [2019-12-17 12:27:43,348] jepsen worker 40 - jepsen.util 40 :fail :cas [13 [30 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,348] jepsen worker 82 - jepsen.util 82 :fail :cas [27 [33 26]] :KeyNotFoundException INFO [2019-12-17 12:27:43,348] jepsen worker 40 - jepsen.util 40 :invoke :cas [13 [15 44]] INFO [2019-12-17 12:27:43,348] jepsen worker 82 - jepsen.util 82 :invoke :cas [27 [36 39]] INFO [2019-12-17 12:27:43,348] jepsen worker 80 - jepsen.util 80 :ok :read [26 :nil] INFO [2019-12-17 12:27:43,348] jepsen worker 80 - jepsen.util 80 :invoke :write [26 46] INFO [2019-12-17 12:27:43,348] jepsen worker 84 - jepsen.util 84 :ok :read [28 :nil] INFO [2019-12-17 12:27:43,350] jepsen worker 67 - jepsen.util 67 :ok :read [22 :nil] INFO [2019-12-17 12:27:43,350] jepsen worker 84 - jepsen.util 84 :invoke :write [28 39] INFO [2019-12-17 12:27:43,350] jepsen worker 67 - jepsen.util 67 :invoke :cas [22 [18 11]] INFO [2019-12-17 12:27:43,350] jepsen worker 69 - jepsen.util 69 :ok :read [23 :nil] INFO [2019-12-17 12:27:43,352] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [18 25]] INFO [2019-12-17 12:27:43,353] jepsen worker 17 - jepsen.util 17 :fail :cas [5 [17 19]] :KeyNotFoundException INFO [2019-12-17 12:27:43,353] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,353] jepsen worker 17 - jepsen.util 17 :invoke :write [5 1] INFO [2019-12-17 12:27:43,353] jepsen worker 82 - jepsen.util 82 :fail :cas [27 [36 39]] :KeyNotFoundException INFO [2019-12-17 12:27:43,353] jepsen worker 40 - jepsen.util 40 :fail :cas [13 [15 44]] :KeyNotFoundException INFO [2019-12-17 12:27:43,353] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,353] jepsen worker 40 - jepsen.util 40 :invoke :write [13 26] INFO [2019-12-17 12:27:43,354] jepsen worker 82 - jepsen.util 82 :invoke :write [27 15] INFO [2019-12-17 12:27:43,354] jepsen worker 67 - jepsen.util 67 :fail :cas [22 [18 11]] :KeyNotFoundException INFO [2019-12-17 12:27:43,354] jepsen worker 67 - jepsen.util 67 :invoke :write [22 4] INFO [2019-12-17 12:27:43,356] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [18 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,357] jepsen worker 69 - jepsen.util 69 :invoke :write [23 5] INFO [2019-12-17 12:27:43,361] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,361] jepsen worker 63 - jepsen.util 63 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,361] jepsen worker 58 - jepsen.util 58 :invoke :cas [19 [9 27]] INFO [2019-12-17 12:27:43,361] jepsen worker 63 - jepsen.util 63 :invoke :cas [21 [15 28]] INFO [2019-12-17 12:27:43,362] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [16 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,362] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [41 40]] INFO [2019-12-17 12:27:43,363] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [43 38]] :KeyNotFoundException INFO [2019-12-17 12:27:43,363] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [48 18]] INFO [2019-12-17 12:27:43,363] jepsen worker 63 - jepsen.util 63 :fail :cas [21 [15 28]] :KeyNotFoundException INFO [2019-12-17 12:27:43,363] jepsen worker 63 - jepsen.util 63 :invoke :write [21 37] INFO [2019-12-17 12:27:43,363] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [41 40]] :KeyNotFoundException INFO [2019-12-17 12:27:43,364] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [48 18]] :KeyNotFoundException INFO [2019-12-17 12:27:43,364] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [6 28]] INFO [2019-12-17 12:27:43,364] jepsen worker 64 - jepsen.util 64 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,364] jepsen worker 75 - jepsen.util 75 :fail :cas [25 [30 38]] :KeyNotFoundException INFO [2019-12-17 12:27:43,364] jepsen worker 76 - jepsen.util 76 :ok :read [25 :nil] INFO [2019-12-17 12:27:43,364] jepsen worker 75 - jepsen.util 75 :invoke :cas [25 [1 27]] INFO [2019-12-17 12:27:43,364] jepsen worker 76 - jepsen.util 76 :invoke :write [25 48] INFO [2019-12-17 12:27:43,364] jepsen worker 58 - jepsen.util 58 :fail :cas [19 [9 27]] :KeyNotFoundException INFO [2019-12-17 12:27:43,365] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [6 28]] :KeyNotFoundException INFO [2019-12-17 12:27:43,365] jepsen worker 58 - jepsen.util 58 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,365] jepsen worker 64 - jepsen.util 64 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,365] jepsen worker 11 - jepsen.util 11 :invoke :cas [3 [8 1]] INFO [2019-12-17 12:27:43,365] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [1 25]] INFO [2019-12-17 12:27:43,365] jepsen worker 75 - jepsen.util 75 :fail :cas [25 [1 27]] :KeyNotFoundException INFO [2019-12-17 12:27:43,365] jepsen worker 58 - jepsen.util 58 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,366] jepsen worker 75 - jepsen.util 75 :invoke :cas [25 [20 39]] INFO [2019-12-17 12:27:43,366] jepsen worker 58 - jepsen.util 58 :invoke :write [19 37] INFO [2019-12-17 12:27:43,366] jepsen worker 11 - jepsen.util 11 :fail :cas [3 [8 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,366] jepsen worker 11 - jepsen.util 11 :invoke :write [3 3] INFO [2019-12-17 12:27:43,366] jepsen worker 75 - jepsen.util 75 :fail :cas [25 [20 39]] :KeyNotFoundException INFO [2019-12-17 12:27:43,367] jepsen worker 75 - jepsen.util 75 :invoke :write [25 49] INFO [2019-12-17 12:27:43,367] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [1 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,367] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [44 37]] INFO [2019-12-17 12:27:43,368] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [44 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,368] jepsen worker 64 - jepsen.util 64 :invoke :read [21 nil] INFO [2019-12-17 12:27:43,369] jepsen worker 64 - jepsen.util 64 :ok :read [21 :nil] INFO [2019-12-17 12:27:43,369] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [33 21]] INFO [2019-12-17 12:27:43,369] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [22 29]] INFO [2019-12-17 12:27:43,369] jepsen worker 49 - jepsen.util 49 :ok :read [16 :nil] INFO [2019-12-17 12:27:43,369] jepsen worker 49 - jepsen.util 49 :invoke :read [16 nil] INFO [2019-12-17 12:27:43,270] jepsen worker 50 - jepsen.util 50 :invoke :cas [16 [3 25]] INFO [2019-12-17 12:27:43,270] jepsen worker 62 - jepsen.util 62 :invoke :cas [20 [11 33]] INFO [2019-12-17 12:27:43,371] jepsen worker 12 - jepsen.util 12 :fail :cas [4 [32 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,371] jepsen worker 12 - jepsen.util 12 :invoke :write [4 4] INFO [2019-12-17 12:27:43,372] jepsen worker 56 - jepsen.util 56 :invoke :cas [18 [28 37]] INFO [2019-12-17 12:27:43,372] jepsen worker 52 - jepsen.util 52 :fail :cas [17 [36 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,373] jepsen worker 52 - jepsen.util 52 :invoke :cas [17 [36 27]] INFO [2019-12-17 12:27:43,373] jepsen worker 54 - jepsen.util 54 :invoke :cas [18 [5 15]] INFO [2019-12-17 12:27:43,375] jepsen worker 89 - jepsen.util 89 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,375] jepsen worker 89 - jepsen.util 89 :invoke :cas [29 [45 1]] INFO [2019-12-17 12:27:43,376] jepsen worker 56 - jepsen.util 56 :fail :cas [18 [28 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,376] jepsen worker 52 - jepsen.util 52 :fail :cas [17 [36 27]] :KeyNotFoundException INFO [2019-12-17 12:27:43,376] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:43,376] jepsen worker 52 - jepsen.util 52 :invoke :read [17 nil] INFO [2019-12-17 12:27:43,376] jepsen worker 54 - jepsen.util 54 :fail :cas [18 [5 15]] :KeyNotFoundException INFO [2019-12-17 12:27:43,377] jepsen worker 89 - jepsen.util 89 :fail :cas [29 [45 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,377] jepsen worker 54 - jepsen.util 54 :invoke :cas [18 [39 11]] INFO [2019-12-17 12:27:43,377] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,378] jepsen worker 52 - jepsen.util 52 :ok :read [17 :nil] INFO [2019-12-17 12:27:43,378] jepsen worker 56 - jepsen.util 56 :ok :read [18 :nil] INFO [2019-12-17 12:27:43,378] jepsen worker 52 - jepsen.util 52 :invoke :read [17 nil] INFO [2019-12-17 12:27:43,378] jepsen worker 54 - jepsen.util 54 :fail :cas [18 [39 11]] :KeyNotFoundException INFO [2019-12-17 12:27:43,378] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:43,379] jepsen worker 54 - jepsen.util 54 :invoke :cas [18 [5 46]] INFO [2019-12-17 12:27:43,379] jepsen worker 89 - jepsen.util 89 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,379] jepsen worker 52 - jepsen.util 52 :ok :read [17 :nil] INFO [2019-12-17 12:27:43,379] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:43,379] jepsen worker 52 - jepsen.util 52 :invoke :cas [17 [5 1]] INFO [2019-12-17 12:27:43,379] jepsen worker 31 - jepsen.util 31 :fail :cas [10 [32 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 56 - jepsen.util 56 :ok :read [18 :nil] INFO [2019-12-17 12:27:43,380] jepsen worker 64 - jepsen.util 64 :fail :cas [21 [33 21]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 31 - jepsen.util 31 :invoke :read [10 nil] INFO [2019-12-17 12:27:43,380] jepsen worker 56 - jepsen.util 56 :invoke :write [18 28] INFO [2019-12-17 12:27:43,380] jepsen worker 54 - jepsen.util 54 :fail :cas [18 [5 46]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 89 - jepsen.util 89 :ok :read [29 :nil] INFO [2019-12-17 12:27:43,380] jepsen worker 52 - jepsen.util 52 :fail :cas [17 [5 1]] :KeyNotFoundException INFO [2019-12-17 12:27:43,380] jepsen worker 64 - jepsen.util 64 :invoke :write [21 49] INFO [2019-12-17 12:27:43,380] jepsen worker 54 - jepsen.util 54 :invoke :read [18 nil] INFO [2019-12-17 12:27:43,380] jepsen worker 89 - jepsen.util 89 :invoke :write [29 5] INFO [2019-12-17 12:27:43,380] jepsen worker 52 - jepsen.util 52 :invoke :write [17 15] INFO [2019-12-17 12:27:43,381] jepsen worker 54 - jepsen.util 54 :ok :read [18 :nil] INFO [2019-12-17 12:27:43,381] jepsen worker 54 - jepsen.util 54 :invoke :write [18 47] INFO [2019-12-17 12:27:43,384] jepsen worker 73 - jepsen.util 73 :fail :cas [24 [39 37]] :KeyNotFoundException INFO [2019-12-17 12:27:43,384] jepsen worker 73 - jepsen.util 73 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,389] jepsen worker 62 - jepsen.util 62 :fail :cas [20 [11 33]] :KeyNotFoundException INFO [2019-12-17 12:27:43,390] jepsen worker 62 - jepsen.util 62 :invoke :cas [20 [14 24]] INFO [2019-12-17 12:27:43,390] jepsen worker 73 - jepsen.util 73 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,390] jepsen worker 31 - jepsen.util 31 :ok :read [10 :nil] INFO [2019-12-17 12:27:43,390] jepsen worker 73 - jepsen.util 73 :invoke :read [24 nil] INFO [2019-12-17 12:27:43,390] jepsen worker 31 - jepsen.util 31 :invoke :cas [10 [38 0]] INFO [2019-12-17 12:27:43,391] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [22 29]] :KeyNotFoundException INFO [2019-12-17 12:27:43,391] jepsen worker 49 - jepsen.util 49 :ok :read [16 :nil] INFO [2019-12-17 12:27:43,391] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [28 36]] INFO [2019-12-17 12:27:43,391] jepsen worker 49 - jepsen.util 49 :invoke :write [16 8] INFO [2019-12-17 12:27:43,391] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [28 45]] :KeyNotFoundException INFO [2019-12-17 12:27:43,391] jepsen worker 62 - jepsen.util 62 :fail :cas [20 [14 24]] :KeyNotFoundException INFO [2019-12-17 12:27:43,391] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [19 42]] INFO [2019-12-17 12:27:43,391] jepsen worker 62 - jepsen.util 62 :invoke :write [20 4] INFO [2019-12-17 12:27:43,392] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [28 36]] :KeyNotFoundException INFO [2019-12-17 12:27:43,392] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [13 4]] INFO [2019-12-17 12:27:43,392] jepsen worker 15 - jepsen.util 15 :fail :cas [5 [37 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,392] jepsen worker 15 - jepsen.util 15 :invoke :cas [5 [25 32]] INFO [2019-12-17 12:27:43,392] jepsen worker 81 - jepsen.util 81 :fail :cas [27 [46 6]] :KeyNotFoundException INFO [2019-12-17 12:27:43,393] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,393] jepsen worker 57 - jepsen.util 57 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,393] jepsen worker 57 - jepsen.util 57 :invoke :read [19 nil] INFO [2019-12-17 12:27:43,393] jepsen worker 50 - jepsen.util 50 :fail :cas [16 [3 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,394] jepsen worker 50 - jepsen.util 50 :invoke :write [16 10] INFO [2019-12-17 12:27:43,394] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [13 4]] :KeyNotFoundException INFO [2019-12-17 12:27:43,394] jepsen worker 86 - jepsen.util 86 :invoke :write [28 49] INFO [2019-12-17 12:27:43,395] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [19 42]] :KeyNotFoundException INFO [2019-12-17 12:27:43,396] jepsen worker 39 - jepsen.util 39 :invoke :read [13 nil] INFO [2019-12-17 12:27:43,396] jepsen worker 15 - jepsen.util 15 :fail :cas [5 [25 32]] :KeyNotFoundException INFO [2019-12-17 12:27:43,396] jepsen worker 15 - jepsen.util 15 :invoke :write [5 42] INFO [2019-12-17 12:27:43,396] jepsen worker 81 - jepsen.util 81 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,396] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,397] jepsen worker 57 - jepsen.util 57 :ok :read [19 :nil] INFO [2019-12-17 12:27:43,397] jepsen worker 57 - jepsen.util 57 :invoke :write [19 2] INFO [2019-12-17 12:27:43,400] jepsen worker 39 - jepsen.util 39 :ok :read [13 :nil] INFO [2019-12-17 12:27:43,400] jepsen worker 73 - jepsen.util 73 :ok :read [24 :nil] INFO [2019-12-17 12:27:43,400] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [31 25]] INFO [2019-12-17 12:27:43,400] jepsen worker 73 - jepsen.util 73 :invoke :write [24 29] INFO [2019-12-17 12:27:43,400] jepsen worker 81 - jepsen.util 81 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,400] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:43,401] jepsen worker 31 - jepsen.util 31 :fail :cas [10 [38 0]] :KeyNotFoundException INFO [2019-12-17 12:27:43,401] jepsen worker 31 - jepsen.util 31 :invoke :write [10 44] INFO [2019-12-17 12:27:43,406] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [31 25]] :KeyNotFoundException INFO [2019-12-17 12:27:43,406] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [30 16]] INFO [2019-12-17 12:27:43,407] jepsen worker 81 - jepsen.util 81 :ok :read [27 :nil] INFO [2019-12-17 12:27:43,407] jepsen worker 81 - jepsen.util 81 :invoke :cas [27 [26 14]] INFO [2019-12-17 12:27:43,408] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [30 16]] :KeyNotFoundException INFO [2019-12-17 12:27:43,408] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [37 19]] INFO [2019-12-17 12:27:43,408] jepsen worker 81 - jepsen.util 81 :fail :cas [27 [26 14]] :KeyNotFoundException INFO [2019-12-17 12:27:43,408] jepsen worker 81 - jepsen.util 81 :invoke :cas [27 [26 29]] INFO [2019-12-17 12:27:43,409] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [37 19]] :KeyNotFoundException INFO [2019-12-17 12:27:43,410] jepsen worker 39 - jepsen.util 39 :invoke :cas [13 [36 36]] INFO [2019-12-17 12:27:43,410] jepsen worker 81 - jepsen.util 81 :fail :cas [27 [26 29]] :KeyNotFoundException INFO [2019-12-17 12:27:43,411] jepsen worker 81 - jepsen.util 81 :invoke :write [27 18] INFO [2019-12-17 12:27:43,412] jepsen worker 39 - jepsen.util 39 :fail :cas [13 [36 36]] :KeyNotFoundException INFO [2019-12-17 12:27:43,412] jepsen worker 39 - jepsen.util 39 :invoke :write [13 22] INFO [2019-12-17 12:27:43,898] jepsen worker 27 - jepsen.util 27 :ok :write [9 7] INFO [2019-12-17 12:27:43,898] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:43,922] jepsen worker 86 - jepsen.util 86 :ok :write [28 49] INFO [2019-12-17 12:27:43,922] jepsen worker 86 - jepsen.util 86 :invoke :write [28 14] INFO [2019-12-17 12:27:44,015] jepsen worker 71 - jepsen.util 71 :ok :write [23 12] INFO [2019-12-17 12:27:44,016] jepsen worker 71 - jepsen.util 71 :invoke :cas [23 [17 9]] INFO [2019-12-17 12:27:44,024] jepsen worker 67 - jepsen.util 67 :ok :write [22 4] INFO [2019-12-17 12:27:44,024] jepsen worker 67 - jepsen.util 67 :invoke :cas [22 [10 44]] INFO [2019-12-17 12:27:44,051] jepsen worker 10 - jepsen.util 10 :ok :write [3 42] INFO [2019-12-17 12:27:44,052] jepsen worker 10 - jepsen.util 10 :invoke :write [3 14] INFO [2019-12-17 12:27:44,058] jepsen worker 13 - jepsen.util 13 :ok :write [4 13] INFO [2019-12-17 12:27:44,059] jepsen worker 13 - jepsen.util 13 :invoke :write [4 43] INFO [2019-12-17 12:27:44,065] jepsen worker 3 - jepsen.util 3 :ok :write [1 29] INFO [2019-12-17 12:27:44,066] jepsen worker 3 - jepsen.util 3 :invoke :read [1 nil] INFO [2019-12-17 12:27:44,068] jepsen worker 32 - jepsen.util 32 :ok :write [10 32] INFO [2019-12-17 12:27:44,068] jepsen worker 32 - jepsen.util 32 :invoke :cas [10 [44 28]] INFO [2019-12-17 12:27:44,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":451,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":454,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:44,074] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:44,124] jepsen worker 41 - jepsen.util 41 :ok :write [13 9] INFO [2019-12-17 12:27:44,124] jepsen worker 41 - jepsen.util 41 :invoke :read [13 nil] INFO [2019-12-17 12:27:44,146] jepsen worker 0 - jepsen.util 0 :ok :write [0 0] INFO [2019-12-17 12:27:44,146] jepsen worker 0 - jepsen.util 0 :invoke :cas [0 [0 19]] INFO [2019-12-17 12:27:44,151] jepsen worker 23 - jepsen.util 23 :ok :write [7 48] INFO [2019-12-17 12:27:44,151] jepsen worker 23 - jepsen.util 23 :invoke :read [7 nil] INFO [2019-12-17 12:27:44,160] jepsen worker 67 - jepsen.util 67 :fail :cas [22 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,161] jepsen worker 67 - jepsen.util 67 :invoke :read [22 nil] INFO [2019-12-17 12:27:44,168] jepsen worker 33 - jepsen.util 33 :ok :write [11 28] INFO [2019-12-17 12:27:44,168] jepsen worker 33 - jepsen.util 33 :invoke :cas [11 [14 0]] INFO [2019-12-17 12:27:44,184] jepsen worker 44 - jepsen.util 44 :ok :write [14 27] INFO [2019-12-17 12:27:44,184] jepsen worker 44 - jepsen.util 44 :invoke :cas [14 [15 40]] INFO [2019-12-17 12:27:44,250] jepsen worker 29 - jepsen.util 29 :ok :write [9 6] INFO [2019-12-17 12:27:44,251] jepsen worker 29 - jepsen.util 29 :invoke :read [9 nil] INFO [2019-12-17 12:27:44,261] jepsen worker 64 - jepsen.util 64 :ok :write [21 49] INFO [2019-12-17 12:27:44,261] jepsen worker 64 - jepsen.util 64 :invoke :read [21 nil] INFO [2019-12-17 12:27:44,277] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:44,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:44,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:44,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":476,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:44,354] jepsen worker 17 - jepsen.util 17 :ok :write [5 1] INFO [2019-12-17 12:27:44,354] jepsen worker 17 - jepsen.util 17 :invoke :write [5 6] INFO [2019-12-17 12:27:44,391] jepsen worker 56 - jepsen.util 56 :ok :write [18 28] INFO [2019-12-17 12:27:44,391] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:44,395] jepsen worker 83 - jepsen.util 83 :ok :write [27 21] INFO [2019-12-17 12:27:44,395] jepsen worker 83 - jepsen.util 83 :invoke :write [27 41] INFO [2019-12-17 12:27:44,395] jepsen worker 49 - jepsen.util 49 :ok :write [16 8] INFO [2019-12-17 12:27:44,395] jepsen worker 49 - jepsen.util 49 :invoke :write [16 35] INFO [2019-12-17 12:27:44,406] jepsen worker 19 - jepsen.util 19 :ok :write [6 45] INFO [2019-12-17 12:27:44,406] jepsen worker 72 - jepsen.util 72 :ok :write [24 14] INFO [2019-12-17 12:27:44,406] jepsen worker 19 - jepsen.util 19 :invoke :cas [6 [38 45]] INFO [2019-12-17 12:27:44,407] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:44,420] jepsen worker 52 - jepsen.util 52 :ok :write [17 15] INFO [2019-12-17 12:27:44,420] jepsen worker 52 - jepsen.util 52 :invoke :write [17 10] INFO [2019-12-17 12:27:44,443] jepsen worker 35 - jepsen.util 35 :ok :write [11 22] INFO [2019-12-17 12:27:44,443] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:44,451] jepsen worker 86 - jepsen.util 86 :ok :write [28 14] INFO [2019-12-17 12:27:44,451] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [43 33]] INFO [2019-12-17 12:27:44,475] jepsen worker 8 - jepsen.util 8 :ok :write [2 0] INFO [2019-12-17 12:27:44,475] jepsen worker 8 - jepsen.util 8 :invoke :cas [2 [18 11]] INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":486,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:44,476] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":489,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:44,506] jepsen worker 27 - jepsen.util 27 :ok :read [9 6] INFO [2019-12-17 12:27:44,506] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:44,514] jepsen worker 88 - jepsen.util 88 :ok :write [29 40] INFO [2019-12-17 12:27:44,514] jepsen worker 88 - jepsen.util 88 :invoke :write [29 15] INFO [2019-12-17 12:27:44,523] jepsen worker 65 - jepsen.util 65 :ok :write [21 36] INFO [2019-12-17 12:27:44,523] jepsen worker 65 - jepsen.util 65 :invoke :write [21 49] INFO [2019-12-17 12:27:44,546] jepsen worker 76 - jepsen.util 76 :ok :write [25 48] INFO [2019-12-17 12:27:44,546] jepsen worker 24 - jepsen.util 24 :ok :write [8 14] INFO [2019-12-17 12:27:44,546] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:44,546] jepsen worker 24 - jepsen.util 24 :invoke :cas [8 [11 7]] INFO [2019-12-17 12:27:44,561] jepsen worker 54 - jepsen.util 54 :ok :write [18 47] INFO [2019-12-17 12:27:44,562] jepsen worker 54 - jepsen.util 54 :invoke :write [18 0] INFO [2019-12-17 12:27:44,574] jepsen worker 69 - jepsen.util 69 :ok :write [23 5] INFO [2019-12-17 12:27:44,574] jepsen worker 69 - jepsen.util 69 :invoke :cas [23 [13 27]] INFO [2019-12-17 12:27:44,588] jepsen worker 67 - jepsen.util 67 :ok :read [22 4] INFO [2019-12-17 12:27:44,588] jepsen worker 67 - jepsen.util 67 :invoke :write [22 19] INFO [2019-12-17 12:27:44,631] jepsen worker 23 - jepsen.util 23 :ok :read [7 48] INFO [2019-12-17 12:27:44,631] jepsen worker 23 - jepsen.util 23 :invoke :read [7 nil] INFO [2019-12-17 12:27:44,647] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,648] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [4 24]] INFO [2019-12-17 12:27:44,694] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":504,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":507,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:44,695] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":508,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:44,715] jepsen worker 19 - jepsen.util 19 :fail :cas [6 [38 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,715] jepsen worker 19 - jepsen.util 19 :invoke :cas [6 [43 33]] INFO [2019-12-17 12:27:44,725] jepsen worker 1 - jepsen.util 1 :ok :write [0 14] INFO [2019-12-17 12:27:44,726] jepsen worker 1 - jepsen.util 1 :invoke :cas [0 [0 34]] INFO [2019-12-17 12:27:44,728] jepsen worker 35 - jepsen.util 35 :ok :read [11 22] INFO [2019-12-17 12:27:44,728] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [36 17]] INFO [2019-12-17 12:27:44,752] jepsen worker 59 - jepsen.util 59 :ok :write [19 12] INFO [2019-12-17 12:27:44,753] jepsen worker 59 - jepsen.util 59 :invoke :cas [19 [44 19]] INFO [2019-12-17 12:27:44,780] jepsen worker 3 - jepsen.util 3 :ok :read [1 29] INFO [2019-12-17 12:27:44,780] jepsen worker 3 - jepsen.util 3 :invoke :cas [1 [10 44]] INFO [2019-12-17 12:27:44,801] jepsen worker 66 - jepsen.util 66 :ok :write [22 40] INFO [2019-12-17 12:27:44,801] jepsen worker 66 - jepsen.util 66 :invoke :read [22 nil] INFO [2019-12-17 12:27:44,833] jepsen worker 37 - jepsen.util 37 :ok :write [12 23] INFO [2019-12-17 12:27:44,834] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [8 37]] INFO [2019-12-17 12:27:44,842] jepsen worker 19 - jepsen.util 19 :fail :cas [6 [43 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,842] jepsen worker 19 - jepsen.util 19 :invoke :write [6 6] INFO [2019-12-17 12:27:44,894] jepsen worker 41 - jepsen.util 41 :ok :read [13 9] INFO [2019-12-17 12:27:44,895] jepsen worker 41 - jepsen.util 41 :invoke :cas [13 [7 6]] INFO [2019-12-17 12:27:44,902] jepsen worker 0 - jepsen.util 0 :fail :cas [0 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,903] jepsen worker 0 - jepsen.util 0 :invoke :cas [0 [28 9]] INFO [2019-12-17 12:27:44,913] jepsen worker 21 - jepsen.util 21 :ok :write [7 20] INFO [2019-12-17 12:27:44,913] jepsen worker 21 - jepsen.util 21 :invoke :write [7 17] INFO [2019-12-17 12:27:44,946] jepsen worker 59 - jepsen.util 59 :fail :cas [19 [44 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,947] jepsen worker 59 - jepsen.util 59 :invoke :write [19 11] INFO [2019-12-17 12:27:44,965] jepsen worker 33 - jepsen.util 33 :fail :cas [11 [14 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,965] jepsen worker 33 - jepsen.util 33 :invoke :write [11 44] INFO [2019-12-17 12:27:44,979] jepsen worker 6 - jepsen.util 6 :ok :write [2 15] INFO [2019-12-17 12:27:44,979] jepsen worker 6 - jepsen.util 6 :invoke :cas [2 [12 20]] INFO [2019-12-17 12:27:44,985] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:44,986] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [41 17]] INFO [2019-12-17 12:27:44,998] jepsen worker 78 - jepsen.util 78 :ok :write [26 49] INFO [2019-12-17 12:27:44,998] jepsen worker 78 - jepsen.util 78 :invoke :read [26 nil] INFO [2019-12-17 12:27:45,045] jepsen worker 51 - jepsen.util 51 :ok :write [17 0] INFO [2019-12-17 12:27:45,046] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:45,054] jepsen worker 30 - jepsen.util 30 :ok :write [10 23] INFO [2019-12-17 12:27:45,055] jepsen worker 30 - jepsen.util 30 :invoke :read [10 nil] INFO [2019-12-17 12:27:45,055] jepsen worker 45 - jepsen.util 45 :ok :write [15 35] INFO [2019-12-17 12:27:45,056] jepsen worker 45 - jepsen.util 45 :invoke :write [15 13] INFO [2019-12-17 12:27:45,061] jepsen worker 5 - jepsen.util 5 :ok :write [1 16] INFO [2019-12-17 12:27:45,061] jepsen worker 87 - jepsen.util 87 :ok :write [29 45] INFO [2019-12-17 12:27:45,061] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:45,061] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:45,073] jepsen worker 70 - jepsen.util 70 :ok :write [23 29] INFO [2019-12-17 12:27:45,073] jepsen worker 70 - jepsen.util 70 :invoke :write [23 16] INFO [2019-12-17 12:27:45,073] jepsen worker 16 - jepsen.util 16 :ok :write [5 32] INFO [2019-12-17 12:27:45,074] jepsen worker 16 - jepsen.util 16 :invoke :read [5 nil] INFO [2019-12-17 12:27:45,077] jepsen worker 34 - jepsen.util 34 :ok :write [11 7] INFO [2019-12-17 12:27:45,077] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:45,132] jepsen worker 61 - jepsen.util 61 :ok :write [20 7] INFO [2019-12-17 12:27:45,132] jepsen worker 61 - jepsen.util 61 :invoke :write [20 29] INFO [2019-12-17 12:27:45,141] jepsen worker 84 - jepsen.util 84 :ok :write [28 39] INFO [2019-12-17 12:27:45,142] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:45,162] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,162] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [22 21]] INFO [2019-12-17 12:27:45,168] jepsen worker 69 - jepsen.util 69 :fail :cas [23 [13 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,168] jepsen worker 69 - jepsen.util 69 :invoke :write [23 40] INFO [2019-12-17 12:27:45,170] jepsen worker 28 - jepsen.util 28 :ok :write [9 30] INFO [2019-12-17 12:27:45,171] jepsen worker 28 - jepsen.util 28 :invoke :write [9 38] INFO [2019-12-17 12:27:45,191] jepsen worker 59 - jepsen.util 59 :ok :write [19 11] INFO [2019-12-17 12:27:45,192] jepsen worker 59 - jepsen.util 59 :invoke :write [19 37] INFO [2019-12-17 12:27:45,212] jepsen worker 76 - jepsen.util 76 :ok :read [25 48] INFO [2019-12-17 12:27:45,213] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [42 10]] INFO [2019-12-17 12:27:45,226] jepsen worker 0 - jepsen.util 0 :fail :cas [0 [28 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,227] jepsen worker 0 - jepsen.util 0 :invoke :cas [0 [11 40]] INFO [2019-12-17 12:27:45,228] jepsen worker 12 - jepsen.util 12 :ok :write [4 4] INFO [2019-12-17 12:27:45,229] jepsen worker 12 - jepsen.util 12 :invoke :cas [4 [34 28]] INFO [2019-12-17 12:27:45,244] jepsen worker 3 - jepsen.util 3 :fail :cas [1 [10 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,244] jepsen worker 3 - jepsen.util 3 :invoke :read [1 nil] INFO [2019-12-17 12:27:45,270] jepsen worker 41 - jepsen.util 41 :fail :cas [13 [7 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,270] jepsen worker 41 - jepsen.util 41 :invoke :write [13 18] INFO [2019-12-17 12:27:45,277] jepsen worker 66 - jepsen.util 66 :ok :read [22 40] INFO [2019-12-17 12:27:45,277] jepsen worker 66 - jepsen.util 66 :invoke :read [22 nil] INFO [2019-12-17 12:27:45,289] jepsen worker 34 - jepsen.util 34 :ok :read [11 7] INFO [2019-12-17 12:27:45,290] jepsen worker 34 - jepsen.util 34 :invoke :cas [11 [36 7]] INFO [2019-12-17 12:27:45,310] jepsen worker 84 - jepsen.util 84 :ok :read [28 39] INFO [2019-12-17 12:27:45,311] jepsen worker 84 - jepsen.util 84 :invoke :write [28 15] INFO [2019-12-17 12:27:45,312] jepsen worker 24 - jepsen.util 24 :fail :cas [8 [11 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,312] jepsen worker 24 - jepsen.util 24 :invoke :cas [8 [20 23]] INFO [2019-12-17 12:27:45,312] jepsen worker 44 - jepsen.util 44 :fail :cas [14 [15 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,312] jepsen worker 44 - jepsen.util 44 :invoke :read [14 nil] INFO [2019-12-17 12:27:45,315] jepsen worker 10 - jepsen.util 10 :ok :write [3 14] INFO [2019-12-17 12:27:45,315] jepsen worker 74 - jepsen.util 74 :ok :write [24 35] INFO [2019-12-17 12:27:45,315] jepsen worker 10 - jepsen.util 10 :invoke :write [3 32] INFO [2019-12-17 12:27:45,315] jepsen worker 74 - jepsen.util 74 :invoke :cas [24 [3 1]] INFO [2019-12-17 12:27:45,334] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [42 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,335] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [49 6]] INFO [2019-12-17 12:27:45,376] jepsen worker 66 - jepsen.util 66 :ok :read [22 40] INFO [2019-12-17 12:27:45,376] jepsen worker 66 - jepsen.util 66 :invoke :cas [22 [25 44]] INFO [2019-12-17 12:27:45,392] jepsen worker 29 - jepsen.util 29 :ok :read [9 30] INFO [2019-12-17 12:27:45,392] jepsen worker 29 - jepsen.util 29 :invoke :write [9 5] INFO [2019-12-17 12:27:45,402] jepsen worker 0 - jepsen.util 0 :fail :cas [0 [11 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,402] jepsen worker 0 - jepsen.util 0 :invoke :read [0 nil] INFO [2019-12-17 12:27:45,402] jepsen worker 64 - jepsen.util 64 :ok :read [21 36] INFO [2019-12-17 12:27:45,403] jepsen worker 64 - jepsen.util 64 :invoke :cas [21 [16 0]] INFO [2019-12-17 12:27:45,432] jepsen worker 32 - jepsen.util 32 :fail :cas [10 [44 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,432] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:45,447] jepsen worker 34 - jepsen.util 34 :fail :cas [11 [36 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,447] jepsen worker 34 - jepsen.util 34 :invoke :read [11 nil] INFO [2019-12-17 12:27:45,451] jepsen worker 6 - jepsen.util 6 :fail :cas [2 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,451] jepsen worker 6 - jepsen.util 6 :invoke :cas [2 [5 14]] INFO [2019-12-17 12:27:45,454] jepsen worker 81 - jepsen.util 81 :ok :write [27 18] INFO [2019-12-17 12:27:45,455] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:45,457] jepsen worker 87 - jepsen.util 87 :ok :read [29 45] INFO [2019-12-17 12:27:45,457] jepsen worker 87 - jepsen.util 87 :invoke :read [29 nil] INFO [2019-12-17 12:27:45,492] jepsen worker 74 - jepsen.util 74 :fail :cas [24 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,492] jepsen worker 74 - jepsen.util 74 :invoke :read [24 nil] INFO [2019-12-17 12:27:45,506] jepsen worker 22 - jepsen.util 22 :info :write [7 23] :RequestCanceledException INFO [2019-12-17 12:27:45,507] jepsen worker 4 - jepsen.util 4 :info :write [1 23] :RequestCanceledException INFO [2019-12-17 12:27:45,507] jepsen worker 40 - jepsen.util 40 :info :write [13 26] :RequestCanceledException INFO [2019-12-17 12:27:45,508] jepsen worker 53 - jepsen.util 53 :info :write [17 31] :RequestCanceledException INFO [2019-12-17 12:27:45,517] jepsen worker 4 - jepsen.util 94 :invoke :read [1 nil] INFO [2019-12-17 12:27:45,517] jepsen worker 53 - jepsen.util 143 :invoke :write [17 4] INFO [2019-12-17 12:27:45,517] jepsen worker 22 - jepsen.util 112 :invoke :cas [7 [24 47]] INFO [2019-12-17 12:27:45,517] jepsen worker 40 - jepsen.util 130 :invoke :cas [13 [30 3]] INFO [2019-12-17 12:27:45,519] jepsen worker 55 - jepsen.util 55 :info :write [18 34] :RequestCanceledException INFO [2019-12-17 12:27:45,520] jepsen worker 60 - jepsen.util 60 :info :write [20 41] :RequestCanceledException INFO [2019-12-17 12:27:45,522] jepsen worker 36 - jepsen.util 36 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:27:45,522] jepsen worker 9 - jepsen.util 9 :info :write [3 2] :RequestCanceledException INFO [2019-12-17 12:27:45,524] jepsen worker 39 - jepsen.util 39 :info :write [13 22] :RequestCanceledException INFO [2019-12-17 12:27:45,528] jepsen worker 36 - jepsen.util 126 :invoke :read [12 nil] INFO [2019-12-17 12:27:45,528] jepsen worker 60 - jepsen.util 150 :invoke :write [20 33] INFO [2019-12-17 12:27:45,528] jepsen worker 55 - jepsen.util 145 :invoke :read [18 nil] INFO [2019-12-17 12:27:45,528] jepsen worker 9 - jepsen.util 99 :invoke :read [3 nil] INFO [2019-12-17 12:27:45,529] jepsen worker 39 - jepsen.util 129 :invoke :write [13 37] INFO [2019-12-17 12:27:45,532] jepsen worker 14 - jepsen.util 14 :info :write [4 24] :RequestCanceledException INFO [2019-12-17 12:27:45,533] jepsen worker 26 - jepsen.util 26 :info :write [8 25] :RequestCanceledException INFO [2019-12-17 12:27:45,534] jepsen worker 2 - jepsen.util 2 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:27:45,537] jepsen worker 14 - jepsen.util 104 :invoke :write [4 32] INFO [2019-12-17 12:27:45,539] jepsen worker 2 - jepsen.util 92 :invoke :cas [0 [34 24]] INFO [2019-12-17 12:27:45,539] jepsen worker 26 - jepsen.util 116 :invoke :write [8 20] INFO [2019-12-17 12:27:45,540] jepsen worker 89 - jepsen.util 89 :ok :write [29 5] INFO [2019-12-17 12:27:45,540] jepsen worker 89 - jepsen.util 89 :invoke :cas [29 [3 1]] INFO [2019-12-17 12:27:45,546] jepsen worker 85 - jepsen.util 85 :info :write [28 32] :RequestCanceledException INFO [2019-12-17 12:27:45,546] jepsen worker 48 - jepsen.util 48 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:27:45,548] jepsen worker 25 - jepsen.util 25 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:27:45,549] jepsen worker 79 - jepsen.util 79 :info :write [26 47] :RequestCanceledException INFO [2019-12-17 12:27:45,552] jepsen worker 68 - jepsen.util 68 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:27:45,552] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:45,557] jepsen worker 73 - jepsen.util 73 :info :write [24 29] :RequestCanceledException INFO [2019-12-17 12:27:45,557] jepsen worker 31 - jepsen.util 31 :info :write [10 44] :RequestCanceledException INFO [2019-12-17 12:27:45,557] jepsen worker 7 - jepsen.util 7 :info :write [2 33] :RequestCanceledException INFO [2019-12-17 12:27:45,557] jepsen worker 50 - jepsen.util 50 :info :write [16 10] :RequestCanceledException INFO [2019-12-17 12:27:45,558] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:45,561] jepsen worker 85 - jepsen.util 175 :invoke :read [28 nil] INFO [2019-12-17 12:27:45,562] jepsen worker 25 - jepsen.util 115 :invoke :write [8 28] INFO [2019-12-17 12:27:45,562] jepsen worker 48 - jepsen.util 138 :invoke :cas [16 [4 37]] INFO [2019-12-17 12:27:45,562] jepsen worker 50 - jepsen.util 140 :invoke :cas [16 [15 13]] INFO [2019-12-17 12:27:45,562] jepsen worker 31 - jepsen.util 121 :invoke :write [10 29] INFO [2019-12-17 12:27:45,562] jepsen worker 7 - jepsen.util 97 :invoke :write [2 35] INFO [2019-12-17 12:27:45,562] jepsen worker 68 - jepsen.util 158 :invoke :write [22 13] INFO [2019-12-17 12:27:45,562] jepsen worker 79 - jepsen.util 169 :invoke :read [26 nil] INFO [2019-12-17 12:27:45,562] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:45,567] jepsen worker 3 - jepsen.util 3 :ok :read [1 16] INFO [2019-12-17 12:27:45,567] jepsen worker 3 - jepsen.util 3 :invoke :write [1 16] INFO [2019-12-17 12:27:45,586] jepsen worker 46 - jepsen.util 46 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:27:45,586] jepsen worker 11 - jepsen.util 11 :info :write [3 3] :RequestCanceledException INFO [2019-12-17 12:27:45,587] jepsen worker 63 - jepsen.util 63 :info :write [21 37] :RequestCanceledException INFO [2019-12-17 12:27:45,589] jepsen worker 77 - jepsen.util 77 :info :write [25 2] :RequestCanceledException INFO [2019-12-17 12:27:45,589] jepsen worker 82 - jepsen.util 82 :info :write [27 15] :RequestCanceledException INFO [2019-12-17 12:27:45,592] jepsen worker 11 - jepsen.util 101 :invoke :read [3 nil] INFO [2019-12-17 12:27:45,592] jepsen worker 77 - jepsen.util 167 :invoke :read [25 nil] INFO [2019-12-17 12:27:45,592] jepsen worker 46 - jepsen.util 136 :invoke :write [15 49] INFO [2019-12-17 12:27:45,592] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [18 24]] INFO [2019-12-17 12:27:45,593] jepsen worker 82 - jepsen.util 172 :invoke :cas [27 [2 3]] INFO [2019-12-17 12:27:45,604] jepsen worker 47 - jepsen.util 47 :info :write [15 44] :RequestCanceledException INFO [2019-12-17 12:27:45,604] jepsen worker 58 - jepsen.util 58 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:27:45,607] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:45,607] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:45,621] jepsen worker 18 - jepsen.util 18 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:27:45,622] jepsen worker 15 - jepsen.util 15 :info :write [5 42] :RequestCanceledException INFO [2019-12-17 12:27:45,622] jepsen worker 42 - jepsen.util 42 :info :write [14 29] :RequestCanceledException INFO [2019-12-17 12:27:45,622] jepsen worker 57 - jepsen.util 57 :info :write [19 2] :RequestCanceledException INFO [2019-12-17 12:27:45,623] jepsen worker 75 - jepsen.util 75 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:27:45,626] jepsen worker 75 - jepsen.util 165 :invoke :read [25 nil] INFO [2019-12-17 12:27:45,628] jepsen worker 15 - jepsen.util 105 :invoke :write [5 28] INFO [2019-12-17 12:27:45,628] jepsen worker 18 - jepsen.util 108 :invoke :cas [6 [3 34]] INFO [2019-12-17 12:27:45,628] jepsen worker 42 - jepsen.util 132 :invoke :write [14 28] INFO [2019-12-17 12:27:45,629] jepsen worker 43 - jepsen.util 43 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:27:45,629] jepsen worker 80 - jepsen.util 80 :info :write [26 46] :RequestCanceledException INFO [2019-12-17 12:27:45,629] jepsen worker 57 - jepsen.util 147 :invoke :read [19 nil] INFO [2019-12-17 12:27:45,631] jepsen worker 20 - jepsen.util 20 :info :write [6 3] :RequestCanceledException INFO [2019-12-17 12:27:45,631] jepsen worker 38 - jepsen.util 38 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:27:45,631] jepsen worker 62 - jepsen.util 62 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:27:45,635] jepsen worker 38 - jepsen.util 128 :invoke :read [12 nil] INFO [2019-12-17 12:27:45,635] jepsen worker 62 - jepsen.util 152 :invoke :read [20 nil] INFO [2019-12-17 12:27:45,635] jepsen worker 80 - jepsen.util 170 :invoke :write [26 19] INFO [2019-12-17 12:27:45,635] jepsen worker 43 - jepsen.util 133 :invoke :write [14 27] INFO [2019-12-17 12:27:45,635] jepsen worker 20 - jepsen.util 110 :invoke :write [6 11] INFO [2019-12-17 12:27:45,636] jepsen worker 78 - jepsen.util 78 :ok :read [26 49] INFO [2019-12-17 12:27:45,636] jepsen worker 78 - jepsen.util 78 :invoke :write [26 26] INFO [2019-12-17 12:27:45,642] jepsen worker 74 - jepsen.util 74 :ok :read [24 35] INFO [2019-12-17 12:27:45,642] jepsen worker 74 - jepsen.util 74 :invoke :cas [24 [5 31]] INFO [2019-12-17 12:27:45,772] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [49 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,772] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [27 48]] INFO [2019-12-17 12:27:45,995] jepsen worker 24 - jepsen.util 24 :fail :cas [8 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:45,995] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:46,071] jepsen worker 87 - jepsen.util 87 :ok :read [29 5] INFO [2019-12-17 12:27:46,071] jepsen worker 87 - jepsen.util 87 :invoke :cas [29 [40 3]] INFO [2019-12-17 12:27:46,080] jepsen worker 44 - jepsen.util 44 :ok :read [14 27] INFO [2019-12-17 12:27:46,081] jepsen worker 44 - jepsen.util 44 :invoke :cas [14 [25 12]] INFO [2019-12-17 12:27:46,084] jepsen worker 23 - jepsen.util 23 :ok :read [7 20] INFO [2019-12-17 12:27:46,084] jepsen worker 23 - jepsen.util 23 :invoke :cas [7 [19 30]] INFO [2019-12-17 12:27:46,088] jepsen worker 71 - jepsen.util 71 :info :cas [23 [17 9]] :RequestCanceledException INFO [2019-12-17 12:27:46,091] jepsen worker 71 - jepsen.util 161 :invoke :cas [23 [49 3]] INFO [2019-12-17 12:27:46,134] jepsen worker 65 - jepsen.util 65 :ok :write [21 49] INFO [2019-12-17 12:27:46,134] jepsen worker 65 - jepsen.util 65 :invoke :write [21 38] INFO [2019-12-17 12:27:46,139] jepsen worker 6 - jepsen.util 6 :fail :cas [2 [5 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,139] jepsen worker 6 - jepsen.util 6 :invoke :read [2 nil] INFO [2019-12-17 12:27:46,152] jepsen worker 13 - jepsen.util 13 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:27:46,155] jepsen worker 13 - jepsen.util 103 :invoke :cas [4 [17 16]] INFO [2019-12-17 12:27:46,194] jepsen worker 34 - jepsen.util 34 :ok :read [11 7] INFO [2019-12-17 12:27:46,194] jepsen worker 34 - jepsen.util 34 :invoke :write [11 46] INFO [2019-12-17 12:27:46,199] jepsen worker 12 - jepsen.util 12 :fail :cas [4 [34 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,199] jepsen worker 12 - jepsen.util 12 :invoke :write [4 44] INFO [2019-12-17 12:27:46,216] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:46,216] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":645,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:46,291] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:46,292] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:46,309] jepsen worker 84 - jepsen.util 84 :ok :write [28 15] INFO [2019-12-17 12:27:46,310] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:46,369] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:46,370] jepsen worker 72 - jepsen.util 72 :invoke :cas [24 [10 10]] INFO [2019-12-17 12:27:46,371] jepsen worker 0 - jepsen.util 0 :ok :read [0 14] INFO [2019-12-17 12:27:46,371] jepsen worker 0 - jepsen.util 0 :invoke :write [0 24] INFO [2019-12-17 12:27:46,386] jepsen worker 17 - jepsen.util 17 :info :write [5 6] :RequestCanceledException INFO [2019-12-17 12:27:46,388] jepsen worker 17 - jepsen.util 107 :invoke :write [5 20] INFO [2019-12-17 12:27:46,420] jepsen worker 27 - jepsen.util 27 :ok :read [9 30] INFO [2019-12-17 12:27:46,420] jepsen worker 49 - jepsen.util 49 :info :write [16 35] :RequestCanceledException INFO [2019-12-17 12:27:46,421] jepsen worker 27 - jepsen.util 27 :invoke :write [9 46] INFO [2019-12-17 12:27:46,423] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:46,434] jepsen worker 56 - jepsen.util 56 :fail :read [18 nil] :RequestTimeoutException INFO [2019-12-17 12:27:46,434] jepsen worker 56 - jepsen.util 56 :invoke :read [18 nil] INFO [2019-12-17 12:27:46,453] jepsen worker 81 - jepsen.util 81 :ok :read [27 18] INFO [2019-12-17 12:27:46,466] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [18 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,472] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:46,476] jepsen worker 63 - jepsen.util 153 :invoke :write [21 19] INFO [2019-12-17 12:27:46,503] jepsen worker 18 - jepsen.util 108 :fail :cas [6 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,504] jepsen worker 18 - jepsen.util 108 :invoke :cas [6 [16 20]] INFO [2019-12-17 12:27:46,505] jepsen worker 83 - jepsen.util 83 :info :write [27 41] :RequestCanceledException INFO [2019-12-17 12:27:46,508] jepsen worker 83 - jepsen.util 173 :invoke :write [27 37] INFO [2019-12-17 12:27:46,512] jepsen worker 52 - jepsen.util 52 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:27:46,515] jepsen worker 52 - jepsen.util 142 :invoke :read [17 nil] INFO [2019-12-17 12:27:46,526] jepsen worker 8 - jepsen.util 8 :info :cas [2 [18 11]] :RequestCanceledException INFO [2019-12-17 12:27:46,527] jepsen worker 8 - jepsen.util 98 :invoke :read [2 nil] INFO [2019-12-17 12:27:46,539] jepsen worker 79 - jepsen.util 169 :ok :read [26 47] INFO [2019-12-17 12:27:46,540] jepsen worker 79 - jepsen.util 169 :invoke :write [26 29] INFO [2019-12-17 12:27:46,579] jepsen worker 54 - jepsen.util 54 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:27:46,582] jepsen worker 54 - jepsen.util 144 :invoke :write [18 21] INFO [2019-12-17 12:27:46,593] jepsen worker 66 - jepsen.util 66 :fail :cas [22 [25 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,593] jepsen worker 66 - jepsen.util 66 :invoke :cas [22 [21 24]] INFO [2019-12-17 12:27:46,595] jepsen worker 72 - jepsen.util 72 :fail :cas [24 [10 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,595] jepsen worker 72 - jepsen.util 72 :invoke :cas [24 [14 33]] INFO [2019-12-17 12:27:46,604] jepsen worker 88 - jepsen.util 88 :info :write [29 15] :RequestCanceledException INFO [2019-12-17 12:27:46,606] jepsen worker 88 - jepsen.util 178 :invoke :cas [29 [48 14]] INFO [2019-12-17 12:27:46,634] jepsen worker 67 - jepsen.util 67 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:27:46,636] jepsen worker 67 - jepsen.util 157 :invoke :write [22 5] INFO [2019-12-17 12:27:46,653] jepsen worker 85 - jepsen.util 175 :ok :read [28 15] INFO [2019-12-17 12:27:46,654] jepsen worker 85 - jepsen.util 175 :invoke :read [28 nil] INFO [2019-12-17 12:27:46,669] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:46,670] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [35 35]] INFO [2019-12-17 12:27:46,683] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:46,683] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:46,696] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [36 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,696] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:46,710] jepsen worker 40 - jepsen.util 130 :fail :cas [13 [30 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,710] jepsen worker 40 - jepsen.util 130 :invoke :write [13 29] INFO [2019-12-17 12:27:46,713] jepsen worker 4 - jepsen.util 94 :ok :read [1 23] INFO [2019-12-17 12:27:46,713] jepsen worker 4 - jepsen.util 94 :invoke :write [1 1] INFO [2019-12-17 12:27:46,739] jepsen worker 6 - jepsen.util 6 :ok :read [2 15] INFO [2019-12-17 12:27:46,740] jepsen worker 6 - jepsen.util 6 :invoke :write [2 43] INFO [2019-12-17 12:27:46,746] jepsen worker 1 - jepsen.util 1 :info :cas [0 [0 34]] :RequestCanceledException INFO [2019-12-17 12:27:46,747] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,747] jepsen worker 86 - jepsen.util 86 :invoke :write [28 34] INFO [2019-12-17 12:27:46,748] jepsen worker 1 - jepsen.util 91 :invoke :cas [0 [29 2]] INFO [2019-12-17 12:27:46,758] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [22 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,758] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [6 12]] INFO [2019-12-17 12:27:46,773] jepsen worker 22 - jepsen.util 112 :fail :cas [7 [24 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,773] jepsen worker 22 - jepsen.util 112 :invoke :cas [7 [23 7]] INFO [2019-12-17 12:27:46,776] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [27 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,776] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:46,820] jepsen worker 55 - jepsen.util 145 :ok :read [18 47] INFO [2019-12-17 12:27:46,820] jepsen worker 55 - jepsen.util 145 :invoke :read [18 nil] INFO [2019-12-17 12:27:46,821] jepsen worker 2 - jepsen.util 92 :fail :cas [0 [34 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,821] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:46,828] jepsen worker 50 - jepsen.util 140 :fail :cas [16 [15 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,829] jepsen worker 50 - jepsen.util 140 :invoke :write [16 0] INFO [2019-12-17 12:27:46,832] jepsen worker 89 - jepsen.util 89 :fail :cas [29 [3 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,833] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:46,872] jepsen worker 27 - jepsen.util 27 :ok :write [9 46] INFO [2019-12-17 12:27:46,873] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:46,875] jepsen worker 81 - jepsen.util 81 :ok :read [27 18] INFO [2019-12-17 12:27:46,875] jepsen worker 81 - jepsen.util 81 :invoke :read [27 nil] INFO [2019-12-17 12:27:46,890] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [35 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,890] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [30 22]] INFO [2019-12-17 12:27:46,895] jepsen worker 68 - jepsen.util 158 :ok :write [22 13] INFO [2019-12-17 12:27:46,908] jepsen worker 8 - jepsen.util 98 :ok :read [2 15] INFO [2019-12-17 12:27:46,923] jepsen worker 84 - jepsen.util 84 :ok :read [28 15] INFO [2019-12-17 12:27:46,948] jepsen worker 19 - jepsen.util 19 :info :write [6 6] :RequestCanceledException INFO [2019-12-17 12:27:46,952] jepsen worker 9 - jepsen.util 99 :ok :read [3 14] INFO [2019-12-17 12:27:46,956] jepsen worker 52 - jepsen.util 142 :ok :read [17 0] INFO [2019-12-17 12:27:46,962] jepsen worker 71 - jepsen.util 161 :fail :cas [23 [49 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,980] jepsen worker 30 - jepsen.util 30 :ok :read [10 23] INFO [2019-12-17 12:27:46,988] jepsen worker 23 - jepsen.util 23 :fail :cas [7 [19 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:46,999] jepsen worker 8 - jepsen.util 98 :invoke :cas [2 [0 45]] INFO [2019-12-17 12:27:46,999] jepsen worker 68 - jepsen.util 158 :invoke :write [22 36] INFO [2019-12-17 12:27:46,999] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:47,000] jepsen worker 9 - jepsen.util 99 :invoke :cas [3 [8 32]] INFO [2019-12-17 12:27:47,000] jepsen worker 52 - jepsen.util 142 :invoke :read [17 nil] INFO [2019-12-17 12:27:47,000] jepsen worker 71 - jepsen.util 161 :invoke :write [23 2] INFO [2019-12-17 12:27:47,000] jepsen worker 30 - jepsen.util 30 :invoke :read [10 nil] INFO [2019-12-17 12:27:47,000] jepsen worker 23 - jepsen.util 23 :invoke :write [7 26] INFO [2019-12-17 12:27:47,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:47,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":703,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:47,001] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:47,002] jepsen worker 19 - jepsen.util 109 :invoke :cas [6 [21 49]] INFO [2019-12-17 12:27:47,002] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":706,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:47,017] jepsen worker 21 - jepsen.util 21 :info :write [7 17] :RequestCanceledException INFO [2019-12-17 12:27:47,017] jepsen worker 33 - jepsen.util 33 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:27:47,019] jepsen worker 33 - jepsen.util 123 :invoke :write [11 13] INFO [2019-12-17 12:27:47,019] jepsen worker 21 - jepsen.util 111 :invoke :cas [7 [46 20]] INFO [2019-12-17 12:27:47,064] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:47,064] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:47,077] jepsen worker 89 - jepsen.util 89 :ok :read [29 5] INFO [2019-12-17 12:27:47,077] jepsen worker 89 - jepsen.util 89 :invoke :read [29 nil] INFO [2019-12-17 12:27:47,088] jepsen worker 48 - jepsen.util 138 :fail :cas [16 [4 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,088] jepsen worker 48 - jepsen.util 138 :invoke :write [16 16] INFO [2019-12-17 12:27:47,101] jepsen worker 16 - jepsen.util 16 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,102] jepsen worker 70 - jepsen.util 70 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:27:47,102] jepsen worker 16 - jepsen.util 16 :invoke :read [5 nil] INFO [2019-12-17 12:27:47,103] jepsen worker 70 - jepsen.util 160 :invoke :cas [23 [40 49]] INFO [2019-12-17 12:27:47,112] jepsen worker 45 - jepsen.util 45 :info :write [15 13] :RequestCanceledException INFO [2019-12-17 12:27:47,114] jepsen worker 45 - jepsen.util 135 :invoke :cas [15 [42 13]] INFO [2019-12-17 12:27:47,127] jepsen worker 51 - jepsen.util 51 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,127] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:47,138] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [6 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,139] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [9 11]] INFO [2019-12-17 12:27:47,140] jepsen worker 76 - jepsen.util 76 :ok :read [25 2] INFO [2019-12-17 12:27:47,140] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [31 9]] INFO [2019-12-17 12:27:47,143] jepsen worker 63 - jepsen.util 153 :ok :write [21 19] INFO [2019-12-17 12:27:47,144] jepsen worker 63 - jepsen.util 153 :invoke :write [21 26] INFO [2019-12-17 12:27:47,145] jepsen worker 58 - jepsen.util 148 :ok :read [19 2] INFO [2019-12-17 12:27:47,145] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [32 23]] INFO [2019-12-17 12:27:47,151] jepsen worker 61 - jepsen.util 61 :info :write [20 29] :RequestCanceledException INFO [2019-12-17 12:27:47,154] jepsen worker 61 - jepsen.util 151 :invoke :write [20 17] INFO [2019-12-17 12:27:47,263] jepsen worker 28 - jepsen.util 28 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:27:47,263] jepsen worker 69 - jepsen.util 69 :info :write [23 40] :RequestCanceledException INFO [2019-12-17 12:27:47,269] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:47,270] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,271] jepsen worker 38 - jepsen.util 128 :ok :read [12 12] INFO [2019-12-17 12:27:47,272] jepsen worker 38 - jepsen.util 128 :invoke :cas [12 [1 40]] INFO [2019-12-17 12:27:47,272] jepsen worker 28 - jepsen.util 118 :invoke :write [9 15] INFO [2019-12-17 12:27:47,272] jepsen worker 69 - jepsen.util 159 :invoke :cas [23 [44 4]] INFO [2019-12-17 12:27:47,277] jepsen worker 18 - jepsen.util 108 :fail :cas [6 [16 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,277] jepsen worker 22 - jepsen.util 112 :fail :cas [7 [23 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,278] jepsen worker 18 - jepsen.util 108 :invoke :write [6 37] INFO [2019-12-17 12:27:47,278] jepsen worker 22 - jepsen.util 112 :invoke :write [7 24] INFO [2019-12-17 12:27:47,279] jepsen worker 55 - jepsen.util 145 :ok :read [18 47] INFO [2019-12-17 12:27:47,279] jepsen worker 55 - jepsen.util 145 :invoke :cas [18 [35 46]] INFO [2019-12-17 12:27:47,280] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [30 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,280] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:47,284] jepsen worker 59 - jepsen.util 59 :info :write [19 37] :RequestCanceledException INFO [2019-12-17 12:27:47,284] jepsen worker 41 - jepsen.util 41 :info :write [13 18] :RequestCanceledException INFO [2019-12-17 12:27:47,285] jepsen worker 57 - jepsen.util 147 :ok :read [19 2] INFO [2019-12-17 12:27:47,285] jepsen worker 57 - jepsen.util 147 :invoke :read [19 nil] INFO [2019-12-17 12:27:47,286] jepsen worker 41 - jepsen.util 131 :invoke :read [13 nil] INFO [2019-12-17 12:27:47,286] jepsen worker 59 - jepsen.util 149 :invoke :cas [19 [47 37]] INFO [2019-12-17 12:27:47,303] jepsen worker 72 - jepsen.util 72 :fail :cas [24 [14 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,304] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,326] jepsen worker 81 - jepsen.util 81 :ok :read [27 18] INFO [2019-12-17 12:27:47,326] jepsen worker 81 - jepsen.util 81 :invoke :write [27 21] INFO [2019-12-17 12:27:47,327] jepsen worker 21 - jepsen.util 111 :fail :cas [7 [46 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,327] jepsen worker 21 - jepsen.util 111 :invoke :read [7 nil] INFO [2019-12-17 12:27:47,333] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [31 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,333] jepsen worker 76 - jepsen.util 76 :invoke :read [25 nil] INFO [2019-12-17 12:27:47,336] jepsen worker 52 - jepsen.util 142 :ok :read [17 0] INFO [2019-12-17 12:27:47,337] jepsen worker 52 - jepsen.util 142 :invoke :cas [17 [0 21]] INFO [2019-12-17 12:27:47,345] jepsen worker 13 - jepsen.util 103 :fail :cas [4 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,346] jepsen worker 13 - jepsen.util 103 :invoke :read [4 nil] INFO [2019-12-17 12:27:47,346] jepsen worker 82 - jepsen.util 172 :fail :cas [27 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,347] jepsen worker 82 - jepsen.util 172 :invoke :write [27 30] INFO [2019-12-17 12:27:47,350] jepsen worker 27 - jepsen.util 27 :ok :read [9 46] INFO [2019-12-17 12:27:47,351] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [2 24]] INFO [2019-12-17 12:27:47,355] jepsen worker 45 - jepsen.util 135 :fail :cas [15 [42 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,356] jepsen worker 45 - jepsen.util 135 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,356] jepsen worker 8 - jepsen.util 98 :fail :cas [2 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,357] jepsen worker 8 - jepsen.util 98 :invoke :write [2 5] INFO [2019-12-17 12:27:47,377] jepsen worker 45 - jepsen.util 135 :ok :read [15 35] INFO [2019-12-17 12:27:47,377] jepsen worker 45 - jepsen.util 135 :invoke :write [15 18] INFO [2019-12-17 12:27:47,390] jepsen worker 89 - jepsen.util 89 :ok :read [29 5] INFO [2019-12-17 12:27:47,390] jepsen worker 89 - jepsen.util 89 :invoke :write [29 10] INFO [2019-12-17 12:27:47,402] jepsen worker 21 - jepsen.util 111 :ok :read [7 20] INFO [2019-12-17 12:27:47,402] jepsen worker 21 - jepsen.util 111 :invoke :cas [7 [3 46]] INFO [2019-12-17 12:27:47,405] jepsen worker 10 - jepsen.util 10 :info :write [3 32] :RequestCanceledException INFO [2019-12-17 12:27:47,407] jepsen worker 10 - jepsen.util 100 :invoke :cas [3 [17 16]] INFO [2019-12-17 12:27:47,420] jepsen worker 64 - jepsen.util 64 :info :cas [21 [16 0]] :RequestCanceledException INFO [2019-12-17 12:27:47,422] jepsen worker 64 - jepsen.util 154 :invoke :cas [21 [10 29]] INFO [2019-12-17 12:27:47,428] jepsen worker 36 - jepsen.util 126 :ok :read [12 12] INFO [2019-12-17 12:27:47,428] jepsen worker 36 - jepsen.util 126 :invoke :write [12 20] INFO [2019-12-17 12:27:47,438] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [32 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,438] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:47,461] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [9 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,461] jepsen worker 37 - jepsen.util 37 :invoke :cas [12 [23 9]] INFO [2019-12-17 12:27:47,469] jepsen worker 21 - jepsen.util 111 :fail :cas [7 [3 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,469] jepsen worker 21 - jepsen.util 111 :invoke :write [7 40] INFO [2019-12-17 12:27:47,478] jepsen worker 56 - jepsen.util 56 :ok :read [18 47] INFO [2019-12-17 12:27:47,478] jepsen worker 56 - jepsen.util 56 :invoke :write [18 39] INFO [2019-12-17 12:27:47,481] jepsen worker 69 - jepsen.util 159 :fail :cas [23 [44 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,482] jepsen worker 69 - jepsen.util 159 :invoke :write [23 8] INFO [2019-12-17 12:27:47,502] jepsen worker 86 - jepsen.util 86 :ok :write [28 34] INFO [2019-12-17 12:27:47,502] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:47,504] jepsen worker 29 - jepsen.util 29 :info :write [9 5] :RequestCanceledException INFO [2019-12-17 12:27:47,506] jepsen worker 29 - jepsen.util 119 :invoke :write [9 12] INFO [2019-12-17 12:27:47,510] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:47,511] jepsen worker 24 - jepsen.util 24 :invoke :cas [8 [16 30]] INFO [2019-12-17 12:27:47,512] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:47,512] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [29 33]] INFO [2019-12-17 12:27:47,527] jepsen worker 16 - jepsen.util 16 :ok :read [5 32] INFO [2019-12-17 12:27:47,528] jepsen worker 16 - jepsen.util 16 :invoke :cas [5 [6 44]] INFO [2019-12-17 12:27:47,552] jepsen worker 60 - jepsen.util 150 :info :write [20 33] :RequestCanceledException INFO [2019-12-17 12:27:47,554] jepsen worker 60 - jepsen.util 240 :invoke :read [20 nil] INFO [2019-12-17 12:27:47,561] jepsen worker 88 - jepsen.util 178 :fail :cas [29 [48 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,561] jepsen worker 88 - jepsen.util 178 :invoke :write [29 29] INFO [2019-12-17 12:27:47,572] jepsen worker 24 - jepsen.util 24 :fail :cas [8 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,572] jepsen worker 11 - jepsen.util 101 :ok :read [3 14] INFO [2019-12-17 12:27:47,572] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,572] jepsen worker 11 - jepsen.util 101 :invoke :write [3 28] INFO [2019-12-17 12:27:47,573] jepsen worker 57 - jepsen.util 147 :ok :read [19 2] INFO [2019-12-17 12:27:47,573] jepsen worker 57 - jepsen.util 147 :invoke :write [19 41] INFO [2019-12-17 12:27:47,575] jepsen worker 63 - jepsen.util 153 :ok :write [21 26] INFO [2019-12-17 12:27:47,575] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,586] jepsen worker 25 - jepsen.util 115 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:27:47,588] jepsen worker 25 - jepsen.util 205 :invoke :cas [8 [26 18]] INFO [2019-12-17 12:27:47,601] jepsen worker 39 - jepsen.util 129 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:27:47,601] jepsen worker 73 - jepsen.util 163 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,601] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,603] jepsen worker 39 - jepsen.util 219 :invoke :cas [13 [23 1]] INFO [2019-12-17 12:27:47,606] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,606] jepsen worker 47 - jepsen.util 137 :invoke :cas [15 [46 48]] INFO [2019-12-17 12:27:47,607] jepsen worker 70 - jepsen.util 160 :fail :cas [23 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,607] jepsen worker 70 - jepsen.util 160 :invoke :write [23 18] INFO [2019-12-17 12:27:47,610] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:47,610] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:47,620] jepsen worker 53 - jepsen.util 143 :info :write [17 4] :RequestCanceledException INFO [2019-12-17 12:27:47,620] jepsen worker 14 - jepsen.util 104 :info :write [4 32] :RequestCanceledException INFO [2019-12-17 12:27:47,620] jepsen worker 3 - jepsen.util 3 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:27:47,622] jepsen worker 53 - jepsen.util 233 :invoke :cas [17 [48 25]] INFO [2019-12-17 12:27:47,622] jepsen worker 3 - jepsen.util 93 :invoke :cas [1 [48 20]] INFO [2019-12-17 12:27:47,622] jepsen worker 14 - jepsen.util 194 :invoke :cas [4 [14 49]] INFO [2019-12-17 12:27:47,635] jepsen worker 77 - jepsen.util 167 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,635] jepsen worker 31 - jepsen.util 121 :info :write [10 29] :RequestCanceledException INFO [2019-12-17 12:27:47,635] jepsen worker 77 - jepsen.util 167 :invoke :cas [25 [39 43]] INFO [2019-12-17 12:27:47,636] jepsen worker 31 - jepsen.util 211 :invoke :write [10 7] INFO [2019-12-17 12:27:47,643] jepsen worker 51 - jepsen.util 51 :ok :read [17 0] INFO [2019-12-17 12:27:47,644] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:47,652] jepsen worker 7 - jepsen.util 97 :info :write [2 35] :RequestCanceledException INFO [2019-12-17 12:27:47,652] jepsen worker 26 - jepsen.util 116 :info :write [8 20] :RequestCanceledException INFO [2019-12-17 12:27:47,652] jepsen worker 46 - jepsen.util 136 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:27:47,652] jepsen worker 15 - jepsen.util 105 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:27:47,654] jepsen worker 26 - jepsen.util 206 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,654] jepsen worker 15 - jepsen.util 195 :invoke :write [5 16] INFO [2019-12-17 12:27:47,654] jepsen worker 46 - jepsen.util 226 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,654] jepsen worker 7 - jepsen.util 187 :invoke :write [2 47] INFO [2019-12-17 12:27:47,662] jepsen worker 47 - jepsen.util 137 :fail :cas [15 [46 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,663] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,664] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:47,664] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,671] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [2 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,671] jepsen worker 27 - jepsen.util 27 :invoke :write [9 23] INFO [2019-12-17 12:27:47,678] jepsen worker 25 - jepsen.util 205 :fail :cas [8 [26 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,678] jepsen worker 25 - jepsen.util 205 :invoke :write [8 15] INFO [2019-12-17 12:27:47,678] jepsen worker 42 - jepsen.util 132 :info :write [14 28] :RequestCanceledException INFO [2019-12-17 12:27:47,678] jepsen worker 20 - jepsen.util 110 :info :write [6 11] :RequestCanceledException INFO [2019-12-17 12:27:47,680] jepsen worker 20 - jepsen.util 200 :invoke :cas [6 [7 12]] INFO [2019-12-17 12:27:47,680] jepsen worker 42 - jepsen.util 222 :invoke :write [14 21] INFO [2019-12-17 12:27:47,694] jepsen worker 64 - jepsen.util 154 :fail :cas [21 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,694] jepsen worker 64 - jepsen.util 154 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,696] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:47,696] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:47,702] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,702] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,715] jepsen worker 76 - jepsen.util 76 :ok :read [25 2] INFO [2019-12-17 12:27:47,715] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [18 22]] INFO [2019-12-17 12:27:47,720] jepsen worker 43 - jepsen.util 133 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:27:47,720] jepsen worker 78 - jepsen.util 78 :info :write [26 26] :RequestCanceledException INFO [2019-12-17 12:27:47,722] jepsen worker 78 - jepsen.util 168 :invoke :read [26 nil] INFO [2019-12-17 12:27:47,722] jepsen worker 43 - jepsen.util 223 :invoke :write [14 3] INFO [2019-12-17 12:27:47,725] jepsen worker 26 - jepsen.util 206 :ok :read [8 14] INFO [2019-12-17 12:27:47,725] jepsen worker 63 - jepsen.util 153 :ok :read [21 26] INFO [2019-12-17 12:27:47,725] jepsen worker 26 - jepsen.util 206 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,725] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,735] jepsen worker 75 - jepsen.util 165 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,735] jepsen worker 62 - jepsen.util 152 :fail :read [20 nil] :RequestTimeoutException INFO [2019-12-17 12:27:47,735] jepsen worker 74 - jepsen.util 74 :info :cas [24 [5 31]] :RequestCanceledException INFO [2019-12-17 12:27:47,735] jepsen worker 75 - jepsen.util 165 :invoke :write [25 46] INFO [2019-12-17 12:27:47,735] jepsen worker 62 - jepsen.util 152 :invoke :cas [20 [31 14]] INFO [2019-12-17 12:27:47,736] jepsen worker 74 - jepsen.util 164 :invoke :cas [24 [25 41]] INFO [2019-12-17 12:27:47,746] jepsen worker 80 - jepsen.util 170 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:27:47,748] jepsen worker 80 - jepsen.util 260 :invoke :write [26 22] INFO [2019-12-17 12:27:47,755] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:47,755] jepsen worker 72 - jepsen.util 72 :invoke :read [24 nil] INFO [2019-12-17 12:27:47,762] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,762] jepsen worker 47 - jepsen.util 137 :invoke :read [15 nil] INFO [2019-12-17 12:27:47,791] jepsen worker 47 - jepsen.util 137 :ok :read [15 35] INFO [2019-12-17 12:27:47,791] jepsen worker 47 - jepsen.util 137 :invoke :cas [15 [17 31]] INFO [2019-12-17 12:27:47,792] jepsen worker 14 - jepsen.util 194 :fail :cas [4 [14 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,792] jepsen worker 14 - jepsen.util 194 :invoke :read [4 nil] INFO [2019-12-17 12:27:47,796] jepsen worker 53 - jepsen.util 233 :fail :cas [17 [48 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,796] jepsen worker 53 - jepsen.util 233 :invoke :write [17 9] INFO [2019-12-17 12:27:47,803] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:47,803] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:47,809] jepsen worker 47 - jepsen.util 137 :fail :cas [15 [17 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,809] jepsen worker 47 - jepsen.util 137 :invoke :write [15 10] INFO [2019-12-17 12:27:47,812] jepsen worker 27 - jepsen.util 27 :ok :write [9 23] INFO [2019-12-17 12:27:47,812] jepsen worker 27 - jepsen.util 27 :invoke :write [9 7] INFO [2019-12-17 12:27:47,823] jepsen worker 37 - jepsen.util 37 :fail :cas [12 [23 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,823] jepsen worker 37 - jepsen.util 37 :invoke :read [12 nil] INFO [2019-12-17 12:27:47,837] jepsen worker 72 - jepsen.util 72 :ok :read [24 35] INFO [2019-12-17 12:27:47,837] jepsen worker 72 - jepsen.util 72 :invoke :write [24 12] INFO [2019-12-17 12:27:47,849] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:47,849] jepsen worker 24 - jepsen.util 24 :invoke :read [8 nil] INFO [2019-12-17 12:27:47,859] jepsen worker 3 - jepsen.util 93 :fail :cas [1 [48 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,859] jepsen worker 3 - jepsen.util 93 :invoke :read [1 nil] INFO [2019-12-17 12:27:47,878] jepsen worker 26 - jepsen.util 206 :ok :read [8 14] INFO [2019-12-17 12:27:47,879] jepsen worker 55 - jepsen.util 145 :fail :cas [18 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:47,879] jepsen worker 26 - jepsen.util 206 :invoke :write [8 10] INFO [2019-12-17 12:27:47,879] jepsen worker 55 - jepsen.util 145 :invoke :write [18 36] INFO [2019-12-17 12:27:47,899] jepsen worker 67 - jepsen.util 157 :ok :write [22 5] INFO [2019-12-17 12:27:47,899] jepsen worker 67 - jepsen.util 157 :invoke :write [22 5] INFO [2019-12-17 12:27:47,902] jepsen worker 37 - jepsen.util 37 :ok :read [12 12] INFO [2019-12-17 12:27:47,902] jepsen worker 37 - jepsen.util 37 :invoke :read [12 nil] INFO [2019-12-17 12:27:47,924] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:47,924] jepsen worker 49 - jepsen.util 139 :invoke :cas [16 [32 4]] INFO [2019-12-17 12:27:47,928] jepsen worker 3 - jepsen.util 93 :ok :read [1 23] INFO [2019-12-17 12:27:47,928] jepsen worker 37 - jepsen.util 37 :ok :read [12 12] INFO [2019-12-17 12:27:47,928] jepsen worker 3 - jepsen.util 93 :invoke :read [1 nil] INFO [2019-12-17 12:27:47,928] jepsen worker 37 - jepsen.util 37 :invoke :write [12 25] INFO [2019-12-17 12:27:47,945] jepsen worker 27 - jepsen.util 27 :ok :write [9 7] INFO [2019-12-17 12:27:47,946] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [3 30]] INFO [2019-12-17 12:27:47,986] jepsen worker 64 - jepsen.util 154 :ok :read [21 26] INFO [2019-12-17 12:27:47,986] jepsen worker 64 - jepsen.util 154 :invoke :read [21 nil] INFO [2019-12-17 12:27:47,996] jepsen worker 60 - jepsen.util 240 :ok :read [20 7] INFO [2019-12-17 12:27:47,996] jepsen worker 60 - jepsen.util 240 :invoke :write [20 48] INFO [2019-12-17 12:27:48,005] jepsen worker 63 - jepsen.util 153 :ok :read [21 26] INFO [2019-12-17 12:27:48,005] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [27 36]] INFO [2019-12-17 12:27:48,018] jepsen worker 64 - jepsen.util 154 :ok :read [21 26] INFO [2019-12-17 12:27:48,019] jepsen worker 64 - jepsen.util 154 :invoke :write [21 25] INFO [2019-12-17 12:27:48,074] jepsen worker 51 - jepsen.util 51 :ok :read [17 0] INFO [2019-12-17 12:27:48,074] jepsen worker 51 - jepsen.util 51 :invoke :read [17 nil] INFO [2019-12-17 12:27:48,076] jepsen worker 38 - jepsen.util 128 :fail :cas [12 [1 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,076] jepsen worker 38 - jepsen.util 128 :invoke :cas [12 [38 17]] INFO [2019-12-17 12:27:48,089] jepsen nemesis - jepsen.util :nemesis :info :failover nil INFO [2019-12-17 12:27:48,090] jepsen nemesis - couchbase.nemesis Failing over node 172.28.128.125 INFO [2019-12-17 12:27:48,098] jepsen worker 13 - jepsen.util 103 :ok :read [4 4] INFO [2019-12-17 12:27:48,099] jepsen worker 13 - jepsen.util 103 :invoke :write [4 39] INFO [2019-12-17 12:27:48,106] jepsen worker 67 - jepsen.util 157 :ok :write [22 5] INFO [2019-12-17 12:27:48,107] jepsen worker 67 - jepsen.util 157 :invoke :read [22 nil] INFO [2019-12-17 12:27:48,121] jepsen worker 87 - jepsen.util 87 :info :cas [29 [40 3]] :RequestCanceledException INFO [2019-12-17 12:27:48,123] jepsen worker 87 - jepsen.util 177 :invoke :read [29 nil] INFO [2019-12-17 12:27:48,131] jepsen worker 49 - jepsen.util 139 :fail :cas [16 [32 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,132] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:48,146] jepsen worker 67 - jepsen.util 157 :ok :read [22 5] INFO [2019-12-17 12:27:48,229] jepsen worker 12 - jepsen.util 12 :info :write [4 44] :RequestCanceledException INFO [2019-12-17 12:27:48,230] jepsen worker 12 - jepsen.util 102 :invoke :read [4 nil] INFO [2019-12-17 12:27:48,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:48,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":863,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:48,235] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:48,236] jepsen worker 67 - jepsen.util 157 :invoke :write [22 1] INFO [2019-12-17 12:27:48,236] jepsen worker 65 - jepsen.util 65 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:27:48,238] jepsen worker 65 - jepsen.util 155 :invoke :cas [21 [6 4]] INFO [2019-12-17 12:27:48,241] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [18 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,242] jepsen worker 76 - jepsen.util 76 :invoke :cas [25 [26 37]] INFO [2019-12-17 12:27:48,245] jepsen worker 44 - jepsen.util 44 :fail :cas [14 [25 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,246] jepsen worker 44 - jepsen.util 44 :invoke :write [14 32] INFO [2019-12-17 12:27:48,246] jepsen worker 24 - jepsen.util 24 :ok :read [8 14] INFO [2019-12-17 12:27:48,246] jepsen worker 24 - jepsen.util 24 :invoke :write [8 32] INFO [2019-12-17 12:27:48,246] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:48,247] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [18 47]] INFO [2019-12-17 12:27:48,247] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:48,247] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:48,247] jepsen worker 78 - jepsen.util 168 :ok :read [26 47] INFO [2019-12-17 12:27:48,247] jepsen worker 32 - jepsen.util 32 :invoke :read [10 nil] INFO [2019-12-17 12:27:48,247] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:48,247] jepsen worker 78 - jepsen.util 168 :invoke :cas [26 [5 15]] INFO [2019-12-17 12:27:48,263] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [27 36]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,263] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:48,282] jepsen worker 32 - jepsen.util 32 :ok :read [10 23] INFO [2019-12-17 12:27:48,282] jepsen worker 32 - jepsen.util 32 :invoke :write [10 15] INFO [2019-12-17 12:27:48,290] jepsen nemesis - couchbase.util Rebalance status: {} INFO [2019-12-17 12:27:48,290] jepsen nemesis - couchbase.util Rebalance check count: 0 INFO [2019-12-17 12:27:48,291] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:48,306] jepsen worker 34 - jepsen.util 34 :info :write [11 46] :RequestCanceledException INFO [2019-12-17 12:27:48,307] jepsen worker 34 - jepsen.util 124 :invoke :cas [11 [21 19]] INFO [2019-12-17 12:27:48,345] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:48,345] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:48,345] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:48,369] jepsen worker 38 - jepsen.util 128 :fail :cas [12 [38 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,369] jepsen worker 38 - jepsen.util 128 :invoke :read [12 nil] INFO [2019-12-17 12:27:48,370] jepsen worker 67 - jepsen.util 157 :ok :write [22 1] INFO [2019-12-17 12:27:48,370] jepsen worker 67 - jepsen.util 157 :invoke :cas [22 [31 24]] INFO [2019-12-17 12:27:48,387] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:48,388] jepsen worker 49 - jepsen.util 139 :invoke :cas [16 [25 34]] INFO [2019-12-17 12:27:48,395] jepsen worker 78 - jepsen.util 168 :fail :cas [26 [5 15]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,395] jepsen worker 78 - jepsen.util 168 :invoke :read [26 nil] INFO [2019-12-17 12:27:48,400] jepsen worker 67 - jepsen.util 157 :fail :cas [22 [31 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,400] jepsen worker 67 - jepsen.util 157 :invoke :read [22 nil] INFO [2019-12-17 12:27:48,410] jepsen worker 49 - jepsen.util 139 :fail :cas [16 [25 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,410] jepsen worker 49 - jepsen.util 139 :invoke :read [16 nil] INFO [2019-12-17 12:27:48,421] jepsen worker 0 - jepsen.util 0 :info :write [0 24] :RequestCanceledException INFO [2019-12-17 12:27:48,423] jepsen worker 0 - jepsen.util 90 :invoke :read [0 nil] INFO [2019-12-17 12:27:48,423] jepsen worker 38 - jepsen.util 128 :ok :read [12 12] INFO [2019-12-17 12:27:48,423] jepsen worker 38 - jepsen.util 128 :invoke :write [12 41] INFO [2019-12-17 12:27:48,428] jepsen worker 17 - jepsen.util 107 :info :write [5 20] :RequestCanceledException INFO [2019-12-17 12:27:48,429] jepsen worker 17 - jepsen.util 197 :invoke :read [5 nil] INFO [2019-12-17 12:27:48,436] jepsen worker 67 - jepsen.util 157 :ok :read [22 1] INFO [2019-12-17 12:27:48,436] jepsen worker 67 - jepsen.util 157 :invoke :cas [22 [18 44]] INFO [2019-12-17 12:27:48,459] jepsen worker 49 - jepsen.util 139 :ok :read [16 8] INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:48,459] jepsen worker 49 - jepsen.util 139 :invoke :write [16 41] INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:48,459] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":895,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:48,475] jepsen worker 0 - jepsen.util 90 :ok :read [0 14] INFO [2019-12-17 12:27:48,475] jepsen worker 0 - jepsen.util 90 :invoke :write [0 32] INFO [2019-12-17 12:27:48,478] jepsen worker 28 - jepsen.util 118 :ok :write [9 15] INFO [2019-12-17 12:27:48,478] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [3 4]] INFO [2019-12-17 12:27:48,487] jepsen worker 67 - jepsen.util 157 :fail :cas [22 [18 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,487] jepsen worker 67 - jepsen.util 157 :invoke :write [22 48] INFO [2019-12-17 12:27:48,497] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:48,498] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [14 34]] INFO [2019-12-17 12:27:48,501] jepsen worker 51 - jepsen.util 51 :ok :read [17 0] INFO [2019-12-17 12:27:48,501] jepsen worker 51 - jepsen.util 51 :invoke :write [17 30] INFO [2019-12-17 12:27:48,511] jepsen worker 3 - jepsen.util 93 :ok :read [1 23] INFO [2019-12-17 12:27:48,512] jepsen worker 3 - jepsen.util 93 :invoke :cas [1 [30 0]] INFO [2019-12-17 12:27:48,522] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,522] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [16 1]] INFO [2019-12-17 12:27:48,530] jepsen worker 78 - jepsen.util 168 :ok :read [26 47] INFO [2019-12-17 12:27:48,530] jepsen worker 78 - jepsen.util 168 :invoke :cas [26 [32 20]] INFO [2019-12-17 12:27:48,538] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [14 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,539] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [44 41]] INFO [2019-12-17 12:27:48,563] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [16 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,564] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [25 16]] INFO [2019-12-17 12:27:48,583] jepsen worker 79 - jepsen.util 169 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:27:48,583] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [44 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,584] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [28 16]] INFO [2019-12-17 12:27:48,584] jepsen worker 79 - jepsen.util 259 :invoke :cas [26 [3 34]] INFO [2019-12-17 12:27:48,597] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [29 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,598] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:48,600] jepsen worker 3 - jepsen.util 93 :fail :cas [1 [30 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,600] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,600] jepsen worker 3 - jepsen.util 93 :invoke :read [1 nil] INFO [2019-12-17 12:27:48,600] jepsen worker 28 - jepsen.util 118 :invoke :write [9 0] INFO [2019-12-17 12:27:48,613] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [28 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,613] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [9 39]] INFO [2019-12-17 12:27:48,620] jepsen worker 83 - jepsen.util 173 :info :write [27 37] :RequestCanceledException INFO [2019-12-17 12:27:48,620] jepsen worker 66 - jepsen.util 66 :info :cas [22 [21 24]] :RequestCanceledException INFO [2019-12-17 12:27:48,621] jepsen worker 54 - jepsen.util 144 :info :write [18 21] :RequestCanceledException INFO [2019-12-17 12:27:48,622] jepsen worker 83 - jepsen.util 263 :invoke :read [27 nil] INFO [2019-12-17 12:27:48,622] jepsen worker 66 - jepsen.util 156 :invoke :write [22 12] INFO [2019-12-17 12:27:48,623] jepsen worker 54 - jepsen.util 234 :invoke :write [18 3] INFO [2019-12-17 12:27:48,668] jepsen worker 3 - jepsen.util 93 :ok :read [1 23] INFO [2019-12-17 12:27:48,668] jepsen worker 3 - jepsen.util 93 :invoke :cas [1 [7 19]] INFO [2019-12-17 12:27:48,678] jepsen worker 77 - jepsen.util 167 :fail :cas [25 [39 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,678] jepsen worker 77 - jepsen.util 167 :invoke :write [25 41] INFO [2019-12-17 12:27:48,699] jepsen worker 28 - jepsen.util 118 :ok :write [9 0] INFO [2019-12-17 12:27:48,699] jepsen worker 28 - jepsen.util 118 :invoke :write [9 11] INFO [2019-12-17 12:27:48,709] jepsen worker 67 - jepsen.util 157 :ok :write [22 48] INFO [2019-12-17 12:27:48,710] jepsen worker 67 - jepsen.util 157 :invoke :write [22 41] INFO [2019-12-17 12:27:48,714] jepsen worker 3 - jepsen.util 93 :fail :cas [1 [7 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,715] jepsen worker 3 - jepsen.util 93 :invoke :write [1 8] INFO [2019-12-17 12:27:48,721] jepsen worker 85 - jepsen.util 175 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:48,721] jepsen worker 85 - jepsen.util 175 :invoke :cas [28 [44 5]] INFO [2019-12-17 12:27:48,727] jepsen worker 62 - jepsen.util 152 :fail :cas [20 [31 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,727] jepsen worker 62 - jepsen.util 152 :invoke :cas [20 [17 27]] INFO [2019-12-17 12:27:48,788] jepsen worker 84 - jepsen.util 84 :ok :read [28 34] INFO [2019-12-17 12:27:48,788] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [39 14]] INFO [2019-12-17 12:27:48,795] jepsen worker 28 - jepsen.util 118 :ok :write [9 11] INFO [2019-12-17 12:27:48,795] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [5 13]] INFO [2019-12-17 12:27:48,805] jepsen worker 4 - jepsen.util 94 :info :write [1 1] :RequestCanceledException INFO [2019-12-17 12:27:48,806] jepsen worker 4 - jepsen.util 184 :invoke :cas [1 [41 46]] INFO [2019-12-17 12:27:48,810] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [5 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,810] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:48,820] jepsen worker 40 - jepsen.util 130 :info :write [13 29] :RequestCanceledException INFO [2019-12-17 12:27:48,820] jepsen worker 6 - jepsen.util 6 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:27:48,820] jepsen worker 1 - jepsen.util 91 :info :cas [0 [29 2]] :RequestCanceledException INFO [2019-12-17 12:27:48,822] jepsen worker 40 - jepsen.util 220 :invoke :cas [13 [26 24]] INFO [2019-12-17 12:27:48,822] jepsen worker 6 - jepsen.util 96 :invoke :write [2 43] INFO [2019-12-17 12:27:48,822] jepsen worker 1 - jepsen.util 181 :invoke :write [0 28] INFO [2019-12-17 12:27:48,828] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [9 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,828] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:48,868] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:48,868] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [36 20]] INFO [2019-12-17 12:27:48,878] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,878] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:48,892] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,892] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:48,910] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:48,910] jepsen worker 73 - jepsen.util 163 :invoke :cas [24 [39 47]] INFO [2019-12-17 12:27:48,917] jepsen worker 76 - jepsen.util 76 :fail :cas [25 [26 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,917] jepsen worker 76 - jepsen.util 76 :invoke :write [25 19] INFO [2019-12-17 12:27:48,921] jepsen worker 50 - jepsen.util 140 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:27:48,922] jepsen worker 50 - jepsen.util 230 :invoke :write [16 33] INFO [2019-12-17 12:27:48,936] jepsen worker 68 - jepsen.util 158 :ok :write [22 36] INFO [2019-12-17 12:27:48,936] jepsen worker 68 - jepsen.util 158 :invoke :cas [22 [29 29]] INFO [2019-12-17 12:27:48,939] jepsen worker 28 - jepsen.util 118 :ok :read [9 11] INFO [2019-12-17 12:27:48,940] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:48,952] jepsen worker 17 - jepsen.util 197 :ok :read [5 32] INFO [2019-12-17 12:27:48,953] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [0 45]] INFO [2019-12-17 12:27:48,969] jepsen worker 9 - jepsen.util 99 :fail :cas [3 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,969] jepsen worker 9 - jepsen.util 99 :invoke :read [3 nil] INFO [2019-12-17 12:27:48,982] jepsen worker 30 - jepsen.util 30 :ok :read [10 23] INFO [2019-12-17 12:27:48,982] jepsen worker 30 - jepsen.util 30 :invoke :cas [10 [11 24]] INFO [2019-12-17 12:27:48,993] jepsen worker 85 - jepsen.util 175 :fail :cas [28 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:48,993] jepsen worker 85 - jepsen.util 175 :invoke :cas [28 [18 47]] INFO [2019-12-17 12:27:49,009] jepsen worker 19 - jepsen.util 109 :fail :cas [6 [21 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,009] jepsen worker 19 - jepsen.util 109 :invoke :write [6 16] INFO [2019-12-17 12:27:49,012] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,013] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,023] jepsen worker 62 - jepsen.util 152 :fail :cas [20 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,024] jepsen worker 62 - jepsen.util 152 :invoke :write [20 27] INFO [2019-12-17 12:27:49,028] jepsen worker 40 - jepsen.util 220 :fail :cas [13 [26 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,028] jepsen worker 40 - jepsen.util 220 :invoke :cas [13 [34 37]] INFO [2019-12-17 12:27:49,041] jepsen worker 68 - jepsen.util 158 :fail :cas [22 [29 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,041] jepsen worker 68 - jepsen.util 158 :invoke :read [22 nil] INFO [2019-12-17 12:27:49,054] jepsen worker 87 - jepsen.util 177 :ok :read [29 5] INFO [2019-12-17 12:27:49,055] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [26 14]] INFO [2019-12-17 12:27:49,069] jepsen worker 17 - jepsen.util 197 :fail :cas [5 [0 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,069] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [41 14]] INFO [2019-12-17 12:27:49,074] jepsen worker 28 - jepsen.util 118 :ok :read [9 11] INFO [2019-12-17 12:27:49,074] jepsen worker 28 - jepsen.util 118 :invoke :write [9 0] INFO [2019-12-17 12:27:49,087] jepsen worker 23 - jepsen.util 23 :info :write [7 26] :RequestCanceledException INFO [2019-12-17 12:27:49,089] jepsen worker 23 - jepsen.util 113 :invoke :read [7 nil] INFO [2019-12-17 12:27:49,089] jepsen worker 34 - jepsen.util 124 :fail :cas [11 [21 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,090] jepsen worker 34 - jepsen.util 124 :invoke :write [11 42] INFO [2019-12-17 12:27:49,092] jepsen worker 85 - jepsen.util 175 :fail :cas [28 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,093] jepsen worker 85 - jepsen.util 175 :invoke :cas [28 [48 16]] INFO [2019-12-17 12:27:49,096] jepsen worker 79 - jepsen.util 259 :fail :cas [26 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,096] jepsen worker 79 - jepsen.util 259 :invoke :read [26 nil] INFO [2019-12-17 12:27:49,102] jepsen worker 33 - jepsen.util 123 :info :write [11 13] :RequestCanceledException INFO [2019-12-17 12:27:49,102] jepsen worker 71 - jepsen.util 161 :info :write [23 2] :RequestCanceledException INFO [2019-12-17 12:27:49,104] jepsen worker 71 - jepsen.util 251 :invoke :read [23 nil] INFO [2019-12-17 12:27:49,104] jepsen worker 33 - jepsen.util 213 :invoke :cas [11 [42 45]] INFO [2019-12-17 12:27:49,112] jepsen worker 84 - jepsen.util 84 :ok :read [28 34] INFO [2019-12-17 12:27:49,112] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [33 10]] INFO [2019-12-17 12:27:49,115] jepsen worker 73 - jepsen.util 163 :fail :cas [24 [39 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,115] jepsen worker 73 - jepsen.util 163 :invoke :read [24 nil] INFO [2019-12-17 12:27:49,134] jepsen worker 40 - jepsen.util 220 :fail :cas [13 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,135] jepsen worker 40 - jepsen.util 220 :invoke :read [13 nil] INFO [2019-12-17 12:27:49,149] jepsen worker 29 - jepsen.util 119 :ok :write [9 12] INFO [2019-12-17 12:27:49,149] jepsen worker 29 - jepsen.util 119 :invoke :read [9 nil] INFO [2019-12-17 12:27:49,168] jepsen worker 68 - jepsen.util 158 :ok :read [22 36] INFO [2019-12-17 12:27:49,168] jepsen worker 68 - jepsen.util 158 :invoke :cas [22 [15 12]] INFO [2019-12-17 12:27:49,175] jepsen worker 12 - jepsen.util 102 :ok :read [4 4] INFO [2019-12-17 12:27:49,175] jepsen worker 12 - jepsen.util 102 :invoke :read [4 nil] INFO [2019-12-17 12:27:49,188] jepsen worker 17 - jepsen.util 197 :fail :cas [5 [41 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,188] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [19 20]] INFO [2019-12-17 12:27:49,197] jepsen worker 85 - jepsen.util 175 :fail :cas [28 [48 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,197] jepsen worker 85 - jepsen.util 175 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,203] jepsen worker 48 - jepsen.util 138 :info :write [16 16] :RequestCanceledException INFO [2019-12-17 12:27:49,205] jepsen worker 48 - jepsen.util 228 :invoke :cas [16 [33 38]] INFO [2019-12-17 12:27:49,223] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [33 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,224] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [28 42]] INFO [2019-12-17 12:27:49,236] jepsen worker 40 - jepsen.util 220 :ok :read [13 9] INFO [2019-12-17 12:27:49,236] jepsen worker 40 - jepsen.util 220 :invoke :write [13 3] INFO [2019-12-17 12:27:49,245] jepsen worker 45 - jepsen.util 135 :info :write [15 18] :SyncWriteAmbiguous INFO [2019-12-17 12:27:49,246] jepsen worker 45 - jepsen.util 225 :invoke :cas [15 [37 48]] INFO [2019-12-17 12:27:49,251] jepsen worker 4 - jepsen.util 184 :fail :cas [1 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,252] jepsen worker 4 - jepsen.util 184 :invoke :read [1 nil] INFO [2019-12-17 12:27:49,254] jepsen worker 68 - jepsen.util 158 :fail :cas [22 [15 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,254] jepsen worker 68 - jepsen.util 158 :invoke :cas [22 [19 15]] INFO [2019-12-17 12:27:49,256] jepsen worker 78 - jepsen.util 168 :fail :cas [26 [32 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,257] jepsen worker 78 - jepsen.util 168 :invoke :write [26 48] INFO [2019-12-17 12:27:49,269] jepsen worker 17 - jepsen.util 197 :fail :cas [5 [19 20]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,270] jepsen worker 17 - jepsen.util 197 :invoke :cas [5 [21 37]] INFO [2019-12-17 12:27:49,288] jepsen worker 85 - jepsen.util 175 :ok :read [28 34] INFO [2019-12-17 12:27:49,289] jepsen worker 85 - jepsen.util 175 :invoke :write [28 1] INFO [2019-12-17 12:27:49,299] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:49,299] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:49,302] jepsen worker 10 - jepsen.util 100 :fail :cas [3 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,302] jepsen worker 10 - jepsen.util 100 :invoke :write [3 10] INFO [2019-12-17 12:27:49,311] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,311] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,317] jepsen worker 73 - jepsen.util 163 :ok :read [24 35] INFO [2019-12-17 12:27:49,317] jepsen worker 73 - jepsen.util 163 :invoke :write [24 36] INFO [2019-12-17 12:27:49,320] jepsen worker 61 - jepsen.util 151 :info :write [20 17] :RequestCanceledException INFO [2019-12-17 12:27:49,320] jepsen worker 18 - jepsen.util 108 :info :write [6 37] :RequestCanceledException INFO [2019-12-17 12:27:49,321] jepsen worker 41 - jepsen.util 131 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:49,321] jepsen worker 61 - jepsen.util 241 :invoke :cas [20 [16 35]] INFO [2019-12-17 12:27:49,321] jepsen worker 18 - jepsen.util 198 :invoke :cas [6 [24 39]] INFO [2019-12-17 12:27:49,322] jepsen worker 41 - jepsen.util 131 :invoke :read [13 nil] INFO [2019-12-17 12:27:49,330] jepsen worker 12 - jepsen.util 102 :ok :read [4 4] INFO [2019-12-17 12:27:49,330] jepsen worker 12 - jepsen.util 102 :invoke :cas [4 [0 19]] INFO [2019-12-17 12:27:49,334] jepsen worker 59 - jepsen.util 149 :info :cas [19 [47 37]] :RequestCanceledException INFO [2019-12-17 12:27:49,335] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [15 22]] INFO [2019-12-17 12:27:49,349] jepsen worker 66 - jepsen.util 156 :ok :write [22 12] INFO [2019-12-17 12:27:49,349] jepsen worker 66 - jepsen.util 156 :invoke :cas [22 [46 19]] INFO [2019-12-17 12:27:49,378] jepsen worker 12 - jepsen.util 102 :fail :cas [4 [0 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,378] jepsen worker 12 - jepsen.util 102 :invoke :write [4 28] INFO [2019-12-17 12:27:49,383] jepsen worker 82 - jepsen.util 172 :info :write [27 30] :RequestCanceledException INFO [2019-12-17 12:27:49,383] jepsen worker 22 - jepsen.util 112 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:27:49,385] jepsen worker 22 - jepsen.util 202 :invoke :write [7 16] INFO [2019-12-17 12:27:49,385] jepsen worker 82 - jepsen.util 262 :invoke :write [27 27] INFO [2019-12-17 12:27:49,388] jepsen worker 71 - jepsen.util 251 :ok :read [23 29] INFO [2019-12-17 12:27:49,389] jepsen worker 71 - jepsen.util 251 :invoke :write [23 30] INFO [2019-12-17 12:27:49,393] jepsen worker 28 - jepsen.util 118 :ok :write [9 0] INFO [2019-12-17 12:27:49,394] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [11 39]] INFO [2019-12-17 12:27:49,401] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:49,402] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [44 25]] INFO [2019-12-17 12:27:49,405] jepsen worker 8 - jepsen.util 98 :info :write [2 5] :RequestCanceledException INFO [2019-12-17 12:27:49,407] jepsen worker 8 - jepsen.util 188 :invoke :read [2 nil] INFO [2019-12-17 12:27:49,407] jepsen worker 33 - jepsen.util 213 :fail :cas [11 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,407] jepsen worker 33 - jepsen.util 213 :invoke :cas [11 [31 17]] INFO [2019-12-17 12:27:49,413] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0}, :ns_1@172.28.128.126 {:progress 0}, :ns_1@172.28.128.127 {:progress 0}, :ns_1@172.28.128.128 {:progress 0}} INFO [2019-12-17 12:27:49,414] jepsen nemesis - couchbase.util Rebalance check count: 1 INFO [2019-12-17 12:27:49,414] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:49,421] jepsen worker 81 - jepsen.util 81 :info :write [27 21] :RequestCanceledException INFO [2019-12-17 12:27:49,423] jepsen worker 81 - jepsen.util 171 :invoke :write [27 46] INFO [2019-12-17 12:27:49,435] jepsen worker 36 - jepsen.util 126 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:27:49,436] jepsen worker 36 - jepsen.util 216 :invoke :cas [12 [13 0]] INFO [2019-12-17 12:27:49,441] jepsen worker 58 - jepsen.util 148 :ok :read [19 2] INFO [2019-12-17 12:27:49,441] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [15 9]] INFO [2019-12-17 12:27:49,487] jepsen worker 89 - jepsen.util 89 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:27:49,489] jepsen worker 89 - jepsen.util 179 :invoke :write [29 23] INFO [2019-12-17 12:27:49,500] jepsen worker 16 - jepsen.util 16 :fail :cas [5 [6 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,501] jepsen worker 16 - jepsen.util 16 :invoke :write [5 32] INFO [2019-12-17 12:27:49,502] jepsen worker 41 - jepsen.util 131 :ok :read [13 9] INFO [2019-12-17 12:27:49,502] jepsen worker 41 - jepsen.util 131 :invoke :write [13 37] INFO [2019-12-17 12:27:49,511] jepsen worker 66 - jepsen.util 156 :fail :cas [22 [46 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,511] jepsen worker 66 - jepsen.util 156 :invoke :write [22 15] INFO [2019-12-17 12:27:49,521] jepsen worker 21 - jepsen.util 111 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:27:49,523] jepsen worker 21 - jepsen.util 201 :invoke :cas [7 [34 40]] INFO [2019-12-17 12:27:49,533] jepsen worker 56 - jepsen.util 56 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:27:49,534] jepsen worker 56 - jepsen.util 146 :invoke :cas [18 [40 32]] INFO [2019-12-17 12:27:49,539] jepsen worker 33 - jepsen.util 213 :fail :cas [11 [31 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,540] jepsen worker 86 - jepsen.util 86 :ok :read [28 34] INFO [2019-12-17 12:27:49,540] jepsen worker 33 - jepsen.util 213 :invoke :cas [11 [44 32]] INFO [2019-12-17 12:27:49,540] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:49,543] jepsen worker 45 - jepsen.util 225 :fail :cas [15 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,543] jepsen worker 45 - jepsen.util 225 :invoke :cas [15 [40 31]] INFO [2019-12-17 12:27:49,555] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [44 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,555] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:49,556] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:49,557] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [41 33]] INFO [2019-12-17 12:27:49,566] jepsen worker 4 - jepsen.util 184 :ok :read [1 23] INFO [2019-12-17 12:27:49,567] jepsen worker 4 - jepsen.util 184 :invoke :cas [1 [36 19]] INFO [2019-12-17 12:27:49,569] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [11 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,570] jepsen worker 28 - jepsen.util 118 :invoke :cas [9 [37 34]] INFO [2019-12-17 12:27:49,573] jepsen worker 39 - jepsen.util 219 :fail :cas [13 [23 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,574] jepsen worker 39 - jepsen.util 219 :invoke :cas [13 [11 48]] INFO [2019-12-17 12:27:49,587] jepsen worker 69 - jepsen.util 159 :info :write [23 8] :RequestCanceledException INFO [2019-12-17 12:27:49,587] jepsen worker 61 - jepsen.util 241 :fail :cas [20 [16 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,587] jepsen worker 61 - jepsen.util 241 :invoke :read [20 nil] INFO [2019-12-17 12:27:49,588] jepsen worker 69 - jepsen.util 249 :invoke :cas [23 [5 10]] INFO [2019-12-17 12:27:49,598] jepsen worker 28 - jepsen.util 118 :fail :cas [9 [37 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,599] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:49,603] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [26 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,603] jepsen worker 87 - jepsen.util 177 :invoke :read [29 nil] INFO [2019-12-17 12:27:49,604] jepsen worker 57 - jepsen.util 147 :info :write [19 41] :RequestCanceledException INFO [2019-12-17 12:27:49,606] jepsen worker 57 - jepsen.util 237 :invoke :cas [19 [5 41]] INFO [2019-12-17 12:27:49,610] jepsen worker 48 - jepsen.util 228 :fail :cas [16 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,610] jepsen worker 48 - jepsen.util 228 :invoke :write [16 17] INFO [2019-12-17 12:27:49,612] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:49,612] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:49,621] jepsen worker 11 - jepsen.util 101 :info :write [3 28] :RequestCanceledException INFO [2019-12-17 12:27:49,623] jepsen worker 11 - jepsen.util 191 :invoke :cas [3 [35 48]] INFO [2019-12-17 12:27:49,633] jepsen worker 28 - jepsen.util 118 :ok :read [9 0] INFO [2019-12-17 12:27:49,634] jepsen worker 28 - jepsen.util 118 :invoke :write [9 45] INFO [2019-12-17 12:27:49,634] jepsen worker 70 - jepsen.util 160 :info :write [23 18] :RequestCanceledException INFO [2019-12-17 12:27:49,634] jepsen worker 88 - jepsen.util 178 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:27:49,636] jepsen worker 88 - jepsen.util 268 :invoke :cas [29 [18 26]] INFO [2019-12-17 12:27:49,636] jepsen worker 70 - jepsen.util 250 :invoke :cas [23 [8 26]] INFO [2019-12-17 12:27:49,669] jepsen worker 83 - jepsen.util 263 :ok :read [27 18] INFO [2019-12-17 12:27:49,669] jepsen worker 83 - jepsen.util 263 :invoke :write [27 48] INFO [2019-12-17 12:27:49,674] jepsen worker 79 - jepsen.util 259 :ok :read [26 47] INFO [2019-12-17 12:27:49,675] jepsen worker 79 - jepsen.util 259 :invoke :cas [26 [31 23]] INFO [2019-12-17 12:27:49,687] jepsen worker 52 - jepsen.util 142 :info :cas [17 [0 21]] :RequestCanceledException INFO [2019-12-17 12:27:49,688] jepsen worker 52 - jepsen.util 232 :invoke :cas [17 [25 30]] INFO [2019-12-17 12:27:49,690] jepsen worker 45 - jepsen.util 225 :fail :cas [15 [40 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,690] jepsen worker 45 - jepsen.util 225 :invoke :write [15 42] INFO [2019-12-17 12:27:49,697] jepsen worker 70 - jepsen.util 250 :fail :cas [23 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,697] jepsen worker 70 - jepsen.util 250 :invoke :write [23 41] INFO [2019-12-17 12:27:49,705] jepsen worker 31 - jepsen.util 211 :info :write [10 7] :RequestCanceledException INFO [2019-12-17 12:27:49,705] jepsen worker 42 - jepsen.util 222 :info :write [14 21] :RequestCanceledException INFO [2019-12-17 12:27:49,707] jepsen worker 31 - jepsen.util 301 :invoke :read [10 nil] INFO [2019-12-17 12:27:49,707] jepsen worker 42 - jepsen.util 312 :invoke :read [14 nil] INFO [2019-12-17 12:27:49,720] jepsen worker 7 - jepsen.util 187 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:27:49,721] jepsen worker 25 - jepsen.util 205 :info :write [8 15] :RequestCanceledException INFO [2019-12-17 12:27:49,722] jepsen worker 7 - jepsen.util 277 :invoke :cas [2 [44 26]] INFO [2019-12-17 12:27:49,723] jepsen worker 25 - jepsen.util 295 :invoke :read [8 nil] INFO [2019-12-17 12:27:49,726] jepsen worker 61 - jepsen.util 241 :ok :read [20 7] INFO [2019-12-17 12:27:49,727] jepsen worker 61 - jepsen.util 241 :invoke :read [20 nil] INFO [2019-12-17 12:27:49,734] jepsen worker 15 - jepsen.util 195 :info :write [5 16] :RequestCanceledException INFO [2019-12-17 12:27:49,736] jepsen worker 15 - jepsen.util 285 :invoke :read [5 nil] INFO [2019-12-17 12:27:49,750] jepsen worker 74 - jepsen.util 164 :info :cas [24 [25 41]] :RequestCanceledException INFO [2019-12-17 12:27:49,750] jepsen worker 46 - jepsen.util 226 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:27:49,750] jepsen worker 46 - jepsen.util 226 :invoke :write [15 2] INFO [2019-12-17 12:27:49,751] jepsen worker 74 - jepsen.util 254 :invoke :read [24 nil] INFO [2019-12-17 12:27:49,756] jepsen worker 31 - jepsen.util 301 :ok :read [10 23] INFO [2019-12-17 12:27:49,756] jepsen worker 31 - jepsen.util 301 :invoke :read [10 nil] INFO [2019-12-17 12:27:49,759] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [15 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,760] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:49,772] jepsen worker 31 - jepsen.util 301 :ok :read [10 23] INFO [2019-12-17 12:27:49,772] jepsen worker 31 - jepsen.util 301 :invoke :write [10 43] INFO [2019-12-17 12:27:49,785] jepsen worker 20 - jepsen.util 200 :info :cas [6 [7 12]] :RequestCanceledException INFO [2019-12-17 12:27:49,786] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:49,800] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,800] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [27 46]] INFO [2019-12-17 12:27:49,802] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:49,802] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:49,818] jepsen worker 80 - jepsen.util 260 :info :write [26 22] :RequestCanceledException INFO [2019-12-17 12:27:49,820] jepsen worker 80 - jepsen.util 350 :invoke :write [26 43] INFO [2019-12-17 12:27:49,834] jepsen worker 75 - jepsen.util 165 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:27:49,834] jepsen worker 43 - jepsen.util 223 :info :write [14 3] :RequestCanceledException INFO [2019-12-17 12:27:49,835] jepsen worker 75 - jepsen.util 255 :invoke :read [25 nil] INFO [2019-12-17 12:27:49,835] jepsen worker 43 - jepsen.util 313 :invoke :cas [14 [34 25]] INFO [2019-12-17 12:27:49,844] jepsen worker 69 - jepsen.util 249 :fail :cas [23 [5 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,845] jepsen worker 69 - jepsen.util 249 :invoke :cas [23 [47 42]] INFO [2019-12-17 12:27:49,846] jepsen worker 36 - jepsen.util 216 :fail :cas [12 [13 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,846] jepsen worker 36 - jepsen.util 216 :invoke :cas [12 [12 25]] INFO [2019-12-17 12:27:49,849] jepsen worker 53 - jepsen.util 233 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:27:49,849] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:49,850] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:49,851] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:49,888] jepsen worker 14 - jepsen.util 194 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:27:49,888] jepsen worker 14 - jepsen.util 194 :invoke :write [4 3] INFO [2019-12-17 12:27:49,913] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:49,913] jepsen worker 72 - jepsen.util 72 :info :write [24 12] :SyncWriteAmbiguous INFO [2019-12-17 12:27:49,913] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:49,915] jepsen worker 72 - jepsen.util 162 :invoke :read [24 nil] INFO [2019-12-17 12:27:49,920] jepsen worker 47 - jepsen.util 137 :info :write [15 10] :RequestCanceledException INFO [2019-12-17 12:27:49,920] jepsen worker 55 - jepsen.util 145 :info :write [18 36] :RequestCanceledException INFO [2019-12-17 12:27:49,920] jepsen worker 26 - jepsen.util 206 :info :write [8 10] :RequestCanceledException INFO [2019-12-17 12:27:49,921] jepsen worker 55 - jepsen.util 235 :invoke :write [18 15] INFO [2019-12-17 12:27:49,921] jepsen worker 47 - jepsen.util 227 :invoke :write [15 24] INFO [2019-12-17 12:27:49,921] jepsen worker 26 - jepsen.util 296 :invoke :cas [8 [47 9]] INFO [2019-12-17 12:27:49,944] jepsen worker 21 - jepsen.util 201 :fail :cas [7 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,945] jepsen worker 21 - jepsen.util 201 :invoke :write [7 34] INFO [2019-12-17 12:27:49,950] jepsen worker 37 - jepsen.util 37 :info :write [12 25] :RequestCanceledException INFO [2019-12-17 12:27:49,951] jepsen worker 37 - jepsen.util 127 :invoke :cas [12 [41 23]] INFO [2019-12-17 12:27:49,967] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [3 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,967] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [2 1]] INFO [2019-12-17 12:27:49,980] jepsen worker 28 - jepsen.util 118 :ok :write [9 45] INFO [2019-12-17 12:27:49,981] jepsen worker 28 - jepsen.util 118 :invoke :read [9 nil] INFO [2019-12-17 12:27:49,995] jepsen worker 8 - jepsen.util 188 :ok :read [2 15] INFO [2019-12-17 12:27:49,995] jepsen worker 8 - jepsen.util 188 :invoke :write [2 18] INFO [2019-12-17 12:27:49,996] jepsen worker 33 - jepsen.util 213 :fail :cas [11 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:49,996] jepsen worker 33 - jepsen.util 213 :invoke :write [11 42] INFO [2019-12-17 12:27:50,003] jepsen worker 60 - jepsen.util 240 :info :write [20 48] :RequestCanceledException INFO [2019-12-17 12:27:50,005] jepsen worker 60 - jepsen.util 330 :invoke :write [20 3] INFO [2019-12-17 12:27:50,022] jepsen worker 63 - jepsen.util 153 :ok :read [21 38] INFO [2019-12-17 12:27:50,023] jepsen worker 63 - jepsen.util 153 :invoke :write [21 49] INFO [2019-12-17 12:27:50,046] jepsen worker 28 - jepsen.util 118 :ok :read [9 45] INFO [2019-12-17 12:27:50,046] jepsen worker 28 - jepsen.util 118 :invoke :write [9 36] INFO [2019-12-17 12:27:50,081] jepsen worker 42 - jepsen.util 312 :ok :read [14 27] INFO [2019-12-17 12:27:50,081] jepsen worker 42 - jepsen.util 312 :invoke :read [14 nil] INFO [2019-12-17 12:27:50,084] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:50,084] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1095,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:50,098] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:50,098] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:50,099] jepsen worker 56 - jepsen.util 146 :fail :cas [18 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,099] jepsen worker 56 - jepsen.util 146 :invoke :cas [18 [28 11]] INFO [2019-12-17 12:27:50,113] jepsen worker 61 - jepsen.util 241 :ok :read [20 7] INFO [2019-12-17 12:27:50,113] jepsen worker 61 - jepsen.util 241 :invoke :write [20 49] INFO [2019-12-17 12:27:50,119] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [2 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,119] jepsen worker 27 - jepsen.util 27 :invoke :write [9 41] INFO [2019-12-17 12:27:50,121] jepsen worker 13 - jepsen.util 103 :info :write [4 39] :RequestCanceledException INFO [2019-12-17 12:27:50,121] jepsen worker 64 - jepsen.util 154 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:27:50,123] jepsen worker 13 - jepsen.util 193 :invoke :read [4 nil] INFO [2019-12-17 12:27:50,123] jepsen worker 64 - jepsen.util 244 :invoke :write [21 42] INFO [2019-12-17 12:27:50,123] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:50,123] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [31 25]] INFO [2019-12-17 12:27:50,125] jepsen worker 32 - jepsen.util 32 :info :write [10 15] :SyncWriteAmbiguous INFO [2019-12-17 12:27:50,127] jepsen worker 32 - jepsen.util 122 :invoke :write [10 16] INFO [2019-12-17 12:27:50,149] jepsen worker 69 - jepsen.util 249 :fail :cas [23 [47 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,150] jepsen worker 69 - jepsen.util 249 :invoke :cas [23 [29 44]] INFO [2019-12-17 12:27:50,157] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [31 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,157] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:50,168] jepsen worker 18 - jepsen.util 198 :fail :cas [6 [24 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,169] jepsen worker 18 - jepsen.util 198 :invoke :cas [6 [21 31]] INFO [2019-12-17 12:27:50,179] jepsen worker 58 - jepsen.util 148 :ok :read [19 41] INFO [2019-12-17 12:27:50,179] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [15 38]] INFO [2019-12-17 12:27:50,184] jepsen worker 25 - jepsen.util 295 :ok :read [8 14] INFO [2019-12-17 12:27:50,184] jepsen worker 25 - jepsen.util 295 :invoke :write [8 7] INFO [2019-12-17 12:27:50,195] jepsen worker 42 - jepsen.util 312 :ok :read [14 27] INFO [2019-12-17 12:27:50,196] jepsen worker 42 - jepsen.util 312 :invoke :write [14 13] INFO [2019-12-17 12:27:50,202] jepsen worker 11 - jepsen.util 191 :fail :cas [3 [35 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,202] jepsen worker 11 - jepsen.util 191 :invoke :read [3 nil] INFO [2019-12-17 12:27:50,204] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [15 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,204] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:50,204] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:50,204] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [30 29]] INFO [2019-12-17 12:27:50,223] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:50,223] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [10 40]] INFO [2019-12-17 12:27:50,236] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [30 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,236] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [41 20]] INFO [2019-12-17 12:27:50,241] jepsen worker 18 - jepsen.util 198 :fail :cas [6 [21 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,241] jepsen worker 18 - jepsen.util 198 :invoke :write [6 27] INFO [2019-12-17 12:27:50,243] jepsen worker 87 - jepsen.util 177 :ok :read [29 5] INFO [2019-12-17 12:27:50,243] jepsen worker 87 - jepsen.util 177 :invoke :read [29 nil] INFO [2019-12-17 12:27:50,270] jepsen worker 63 - jepsen.util 153 :ok :write [21 49] INFO [2019-12-17 12:27:50,270] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:50,272] jepsen worker 11 - jepsen.util 191 :ok :read [3 14] INFO [2019-12-17 12:27:50,272] jepsen worker 11 - jepsen.util 191 :invoke :read [3 nil] INFO [2019-12-17 12:27:50,272] jepsen worker 79 - jepsen.util 259 :fail :cas [26 [31 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,272] jepsen worker 79 - jepsen.util 259 :invoke :cas [26 [12 40]] INFO [2019-12-17 12:27:50,291] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:50,291] jepsen worker 53 - jepsen.util 323 :invoke :cas [17 [20 23]] INFO [2019-12-17 12:27:50,301] jepsen worker 52 - jepsen.util 232 :fail :cas [17 [25 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,301] jepsen worker 52 - jepsen.util 232 :invoke :cas [17 [40 45]] INFO [2019-12-17 12:27:50,321] jepsen worker 65 - jepsen.util 155 :info :cas [21 [6 4]] :RequestCanceledException INFO [2019-12-17 12:27:50,321] jepsen worker 24 - jepsen.util 24 :info :write [8 32] :RequestCanceledException INFO [2019-12-17 12:27:50,322] jepsen worker 65 - jepsen.util 245 :invoke :cas [21 [19 12]] INFO [2019-12-17 12:27:50,322] jepsen worker 24 - jepsen.util 114 :invoke :read [8 nil] INFO [2019-12-17 12:27:50,332] jepsen worker 43 - jepsen.util 313 :fail :cas [14 [34 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,332] jepsen worker 43 - jepsen.util 313 :invoke :read [14 nil] INFO [2019-12-17 12:27:50,352] jepsen worker 44 - jepsen.util 44 :info :write [14 32] :RequestCanceledException INFO [2019-12-17 12:27:50,353] jepsen worker 44 - jepsen.util 134 :invoke :cas [14 [19 7]] INFO [2019-12-17 12:27:50,357] jepsen worker 11 - jepsen.util 191 :ok :read [3 14] INFO [2019-12-17 12:27:50,357] jepsen worker 11 - jepsen.util 191 :invoke :cas [3 [23 29]] INFO [2019-12-17 12:27:50,358] jepsen worker 53 - jepsen.util 323 :fail :cas [17 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,358] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,360] jepsen worker 27 - jepsen.util 27 :ok :write [9 41] INFO [2019-12-17 12:27:50,361] jepsen worker 27 - jepsen.util 27 :invoke :write [9 34] INFO [2019-12-17 12:27:50,418] jepsen worker 11 - jepsen.util 191 :fail :cas [3 [23 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,419] jepsen worker 11 - jepsen.util 191 :invoke :cas [3 [2 46]] INFO [2019-12-17 12:27:50,419] jepsen worker 23 - jepsen.util 113 :ok :read [7 20] INFO [2019-12-17 12:27:50,419] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [46 39]] INFO [2019-12-17 12:27:50,423] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:50,423] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1147,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1150,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:50,424] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1151,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:50,435] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0}, :ns_1@172.28.128.126 {:progress 0}, :ns_1@172.28.128.127 {:progress 0}, :ns_1@172.28.128.128 {:progress 0.007547169811320753}} INFO [2019-12-17 12:27:50,436] jepsen nemesis - couchbase.util Rebalance check count: 2 INFO [2019-12-17 12:27:50,436] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:50,444] jepsen worker 56 - jepsen.util 146 :fail :cas [18 [28 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,444] jepsen worker 56 - jepsen.util 146 :invoke :cas [18 [30 14]] INFO [2019-12-17 12:27:50,467] jepsen worker 11 - jepsen.util 191 :fail :cas [3 [2 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,467] jepsen worker 11 - jepsen.util 191 :invoke :read [3 nil] INFO [2019-12-17 12:27:50,478] jepsen worker 13 - jepsen.util 193 :ok :read [4 4] INFO [2019-12-17 12:27:50,478] jepsen worker 13 - jepsen.util 193 :invoke :read [4 nil] INFO [2019-12-17 12:27:50,498] jepsen worker 11 - jepsen.util 191 :ok :read [3 14] INFO [2019-12-17 12:27:50,498] jepsen worker 11 - jepsen.util 191 :invoke :write [3 21] INFO [2019-12-17 12:27:50,501] jepsen worker 27 - jepsen.util 27 :ok :write [9 34] INFO [2019-12-17 12:27:50,501] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [22 24]] INFO [2019-12-17 12:27:50,521] jepsen worker 38 - jepsen.util 128 :info :write [12 41] :RequestCanceledException INFO [2019-12-17 12:27:50,523] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:50,527] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,528] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [19 26]] INFO [2019-12-17 12:27:50,534] jepsen worker 51 - jepsen.util 51 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:27:50,536] jepsen worker 51 - jepsen.util 141 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,540] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [10 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,540] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [22 2]] INFO [2019-12-17 12:27:50,551] jepsen worker 0 - jepsen.util 90 :info :write [0 32] :RequestCanceledException INFO [2019-12-17 12:27:50,551] jepsen worker 49 - jepsen.util 139 :info :write [16 41] :RequestCanceledException INFO [2019-12-17 12:27:50,553] jepsen worker 0 - jepsen.util 180 :invoke :cas [0 [47 40]] INFO [2019-12-17 12:27:50,553] jepsen worker 49 - jepsen.util 229 :invoke :cas [16 [1 19]] INFO [2019-12-17 12:27:50,565] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [19 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,565] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [47 49]] INFO [2019-12-17 12:27:50,570] jepsen worker 58 - jepsen.util 148 :ok :cas [19 [41 20]] INFO [2019-12-17 12:27:50,570] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [37 10]] INFO [2019-12-17 12:27:50,575] jepsen worker 87 - jepsen.util 177 :ok :read [29 5] INFO [2019-12-17 12:27:50,576] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [17 26]] INFO [2019-12-17 12:27:50,585] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [47 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,585] jepsen worker 27 - jepsen.util 27 :invoke :write [9 28] INFO [2019-12-17 12:27:50,590] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [37 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,591] jepsen worker 58 - jepsen.util 148 :invoke :write [19 17] INFO [2019-12-17 12:27:50,597] jepsen worker 43 - jepsen.util 313 :ok :read [14 27] INFO [2019-12-17 12:27:50,597] jepsen worker 43 - jepsen.util 313 :invoke :cas [14 [42 46]] INFO [2019-12-17 12:27:50,608] jepsen worker 4 - jepsen.util 184 :fail :cas [1 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,609] jepsen worker 4 - jepsen.util 184 :invoke :cas [1 [5 23]] INFO [2019-12-17 12:27:50,612] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [27 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,613] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [12 40]] INFO [2019-12-17 12:27:50,629] jepsen worker 79 - jepsen.util 259 :fail :cas [26 [12 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,629] jepsen worker 79 - jepsen.util 259 :invoke :read [26 nil] INFO [2019-12-17 12:27:50,635] jepsen worker 54 - jepsen.util 234 :info :write [18 3] :RequestCanceledException INFO [2019-12-17 12:27:50,636] jepsen worker 54 - jepsen.util 324 :invoke :read [18 nil] INFO [2019-12-17 12:27:50,653] jepsen worker 88 - jepsen.util 268 :fail :cas [29 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,653] jepsen worker 88 - jepsen.util 268 :invoke :cas [29 [28 29]] INFO [2019-12-17 12:27:50,692] jepsen worker 27 - jepsen.util 27 :ok :write [9 28] INFO [2019-12-17 12:27:50,693] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:50,694] jepsen worker 52 - jepsen.util 232 :fail :cas [17 [40 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,694] jepsen worker 52 - jepsen.util 232 :invoke :cas [17 [15 37]] INFO [2019-12-17 12:27:50,711] jepsen worker 58 - jepsen.util 148 :ok :write [19 17] INFO [2019-12-17 12:27:50,712] jepsen worker 58 - jepsen.util 148 :invoke :cas [19 [31 16]] INFO [2019-12-17 12:27:50,715] jepsen worker 27 - jepsen.util 27 :ok :read [9 28] INFO [2019-12-17 12:27:50,716] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:50,734] jepsen worker 77 - jepsen.util 167 :info :write [25 41] :RequestCanceledException INFO [2019-12-17 12:27:50,736] jepsen worker 77 - jepsen.util 257 :invoke :cas [25 [45 27]] INFO [2019-12-17 12:27:50,738] jepsen worker 27 - jepsen.util 27 :ok :read [9 28] INFO [2019-12-17 12:27:50,738] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [18 4]] INFO [2019-12-17 12:27:50,746] jepsen worker 37 - jepsen.util 127 :fail :cas [12 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,746] jepsen worker 37 - jepsen.util 127 :invoke :cas [12 [30 33]] INFO [2019-12-17 12:27:50,751] jepsen worker 67 - jepsen.util 157 :info :write [22 41] :RequestCanceledException INFO [2019-12-17 12:27:50,753] jepsen worker 67 - jepsen.util 247 :invoke :cas [22 [33 35]] INFO [2019-12-17 12:27:50,759] jepsen worker 43 - jepsen.util 313 :fail :cas [14 [42 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,759] jepsen worker 43 - jepsen.util 313 :invoke :write [14 36] INFO [2019-12-17 12:27:50,759] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [18 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,759] jepsen worker 27 - jepsen.util 27 :invoke :write [9 42] INFO [2019-12-17 12:27:50,821] jepsen worker 3 - jepsen.util 93 :info :write [1 8] :RequestCanceledException INFO [2019-12-17 12:27:50,822] jepsen worker 3 - jepsen.util 183 :invoke :cas [1 [35 12]] INFO [2019-12-17 12:27:50,837] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:50,837] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1194,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:50,855] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1196,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:50,868] jepsen worker 37 - jepsen.util 127 :fail :cas [12 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,868] jepsen worker 37 - jepsen.util 127 :invoke :read [12 nil] INFO [2019-12-17 12:27:50,878] jepsen worker 27 - jepsen.util 27 :ok :write [9 42] INFO [2019-12-17 12:27:50,878] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:50,888] jepsen worker 6 - jepsen.util 96 :info :write [2 43] :RequestCanceledException INFO [2019-12-17 12:27:50,889] jepsen worker 6 - jepsen.util 186 :invoke :cas [2 [28 13]] INFO [2019-12-17 12:27:50,903] jepsen worker 1 - jepsen.util 181 :info :write [0 28] :RequestCanceledException INFO [2019-12-17 12:27:50,905] jepsen worker 1 - jepsen.util 271 :invoke :cas [0 [2 25]] INFO [2019-12-17 12:27:50,907] jepsen worker 27 - jepsen.util 27 :ok :read [9 42] INFO [2019-12-17 12:27:50,907] jepsen worker 27 - jepsen.util 27 :invoke :write [9 17] INFO [2019-12-17 12:27:50,908] jepsen worker 58 - jepsen.util 148 :fail :cas [19 [31 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,909] jepsen worker 58 - jepsen.util 148 :invoke :write [19 11] INFO [2019-12-17 12:27:50,975] jepsen worker 51 - jepsen.util 141 :ok :read [17 0] INFO [2019-12-17 12:27:50,975] jepsen worker 51 - jepsen.util 141 :invoke :read [17 nil] INFO [2019-12-17 12:27:50,988] jepsen worker 9 - jepsen.util 99 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:27:50,988] jepsen worker 9 - jepsen.util 99 :invoke :cas [3 [5 12]] INFO [2019-12-17 12:27:50,988] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,988] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [40 1]] INFO [2019-12-17 12:27:50,992] jepsen worker 19 - jepsen.util 109 :info :write [6 16] :SyncWriteAmbiguous INFO [2019-12-17 12:27:50,993] jepsen worker 19 - jepsen.util 199 :invoke :read [6 nil] INFO [2019-12-17 12:27:50,999] jepsen worker 6 - jepsen.util 186 :fail :cas [2 [28 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:50,999] jepsen worker 6 - jepsen.util 186 :invoke :write [2 36] INFO [2019-12-17 12:27:51,002] jepsen worker 44 - jepsen.util 134 :fail :cas [14 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,002] jepsen worker 27 - jepsen.util 27 :ok :write [9 17] INFO [2019-12-17 12:27:51,002] jepsen worker 44 - jepsen.util 134 :invoke :write [14 48] INFO [2019-12-17 12:27:51,002] jepsen worker 27 - jepsen.util 27 :invoke :write [9 46] INFO [2019-12-17 12:27:51,004] jepsen worker 76 - jepsen.util 76 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:27:51,005] jepsen worker 76 - jepsen.util 166 :invoke :cas [25 [43 8]] INFO [2019-12-17 12:27:51,020] jepsen worker 50 - jepsen.util 230 :info :write [16 33] :RequestCanceledException INFO [2019-12-17 12:27:51,022] jepsen worker 50 - jepsen.util 320 :invoke :read [16 nil] INFO [2019-12-17 12:27:51,031] jepsen worker 30 - jepsen.util 30 :fail :cas [10 [11 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,031] jepsen worker 30 - jepsen.util 30 :invoke :write [10 11] INFO [2019-12-17 12:27:51,040] jepsen worker 29 - jepsen.util 119 :ok :read [9 17] INFO [2019-12-17 12:27:51,041] jepsen worker 29 - jepsen.util 119 :invoke :write [9 36] INFO [2019-12-17 12:27:51,074] jepsen worker 13 - jepsen.util 193 :ok :read [4 4] INFO [2019-12-17 12:27:51,074] jepsen worker 13 - jepsen.util 193 :invoke :cas [4 [49 45]] INFO [2019-12-17 12:27:51,091] jepsen worker 74 - jepsen.util 254 :ok :read [24 35] INFO [2019-12-17 12:27:51,091] jepsen worker 74 - jepsen.util 254 :invoke :cas [24 [2 8]] INFO [2019-12-17 12:27:51,117] jepsen worker 72 - jepsen.util 162 :ok :read [24 35] INFO [2019-12-17 12:27:51,118] jepsen worker 72 - jepsen.util 162 :invoke :read [24 nil] INFO [2019-12-17 12:27:51,130] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [12 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,130] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,135] jepsen worker 62 - jepsen.util 152 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:27:51,135] jepsen worker 34 - jepsen.util 124 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:27:51,137] jepsen worker 62 - jepsen.util 242 :invoke :cas [20 [30 8]] INFO [2019-12-17 12:27:51,137] jepsen worker 34 - jepsen.util 214 :invoke :write [11 33] INFO [2019-12-17 12:27:51,140] jepsen worker 40 - jepsen.util 220 :info :write [13 3] :SyncWriteAmbiguous INFO [2019-12-17 12:27:51,141] jepsen worker 40 - jepsen.util 310 :invoke :cas [13 [20 43]] INFO [2019-12-17 12:27:51,148] jepsen worker 9 - jepsen.util 99 :fail :cas [3 [5 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,148] jepsen worker 9 - jepsen.util 99 :invoke :write [3 42] INFO [2019-12-17 12:27:51,161] jepsen worker 58 - jepsen.util 148 :ok :write [19 11] INFO [2019-12-17 12:27:51,161] jepsen worker 58 - jepsen.util 148 :invoke :write [19 14] INFO [2019-12-17 12:27:51,172] jepsen worker 4 - jepsen.util 184 :fail :cas [1 [5 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,173] jepsen worker 4 - jepsen.util 184 :invoke :write [1 42] INFO [2019-12-17 12:27:51,190] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:51,190] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1230,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:51,206] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [15 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,206] jepsen worker 59 - jepsen.util 239 :invoke :write [19 21] INFO [2019-12-17 12:27:51,207] jepsen worker 56 - jepsen.util 146 :fail :cas [18 [30 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,208] jepsen worker 56 - jepsen.util 146 :invoke :write [18 32] INFO [2019-12-17 12:27:51,217] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,217] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [42 26]] INFO [2019-12-17 12:27:51,227] jepsen worker 76 - jepsen.util 166 :fail :cas [25 [43 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,227] jepsen worker 76 - jepsen.util 166 :invoke :write [25 38] INFO [2019-12-17 12:27:51,250] jepsen worker 54 - jepsen.util 324 :ok :read [18 47] INFO [2019-12-17 12:27:51,250] jepsen worker 54 - jepsen.util 324 :invoke :write [18 2] INFO [2019-12-17 12:27:51,267] jepsen worker 27 - jepsen.util 27 :ok :write [9 46] INFO [2019-12-17 12:27:51,267] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [6 45]] INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:51,289] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:51,291] jepsen worker 38 - jepsen.util 218 :ok :read [12 12] INFO [2019-12-17 12:27:51,291] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:51,296] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [6 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,296] jepsen worker 27 - jepsen.util 27 :invoke :write [9 18] INFO [2019-12-17 12:27:51,319] jepsen worker 24 - jepsen.util 114 :ok :read [8 14] INFO [2019-12-17 12:27:51,319] jepsen worker 24 - jepsen.util 114 :invoke :write [8 28] INFO [2019-12-17 12:27:51,320] jepsen worker 78 - jepsen.util 168 :info :write [26 48] :RequestCanceledException INFO [2019-12-17 12:27:51,321] jepsen worker 68 - jepsen.util 158 :info :cas [22 [19 15]] :RequestCanceledException INFO [2019-12-17 12:27:51,321] jepsen worker 17 - jepsen.util 197 :info :cas [5 [21 37]] :RequestCanceledException INFO [2019-12-17 12:27:51,321] jepsen worker 85 - jepsen.util 175 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:27:51,322] jepsen worker 78 - jepsen.util 258 :invoke :write [26 27] INFO [2019-12-17 12:27:51,322] jepsen worker 68 - jepsen.util 248 :invoke :cas [22 [28 6]] INFO [2019-12-17 12:27:51,322] jepsen worker 17 - jepsen.util 287 :invoke :cas [5 [7 1]] INFO [2019-12-17 12:27:51,322] jepsen worker 85 - jepsen.util 265 :invoke :write [28 44] INFO [2019-12-17 12:27:51,329] jepsen worker 62 - jepsen.util 242 :fail :cas [20 [30 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,329] jepsen worker 62 - jepsen.util 242 :invoke :write [20 42] INFO [2019-12-17 12:27:51,350] jepsen worker 10 - jepsen.util 100 :info :write [3 10] :RequestCanceledException INFO [2019-12-17 12:27:51,351] jepsen worker 10 - jepsen.util 190 :invoke :cas [3 [24 19]] INFO [2019-12-17 12:27:51,352] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [22 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,352] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [26 7]] INFO [2019-12-17 12:27:51,361] jepsen worker 88 - jepsen.util 268 :fail :cas [29 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,361] jepsen worker 88 - jepsen.util 268 :invoke :read [29 nil] INFO [2019-12-17 12:27:51,387] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [42 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,388] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [23 34]] INFO [2019-12-17 12:27:51,405] jepsen worker 73 - jepsen.util 163 :info :write [24 36] :RequestCanceledException INFO [2019-12-17 12:27:51,405] jepsen worker 12 - jepsen.util 102 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:27:51,407] jepsen worker 73 - jepsen.util 253 :invoke :read [24 nil] INFO [2019-12-17 12:27:51,407] jepsen worker 12 - jepsen.util 192 :invoke :cas [4 [38 18]] INFO [2019-12-17 12:27:51,411] jepsen worker 3 - jepsen.util 183 :fail :cas [1 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,411] jepsen worker 3 - jepsen.util 183 :invoke :cas [1 [41 17]] INFO [2019-12-17 12:27:51,418] jepsen worker 22 - jepsen.util 202 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:27:51,420] jepsen worker 22 - jepsen.util 292 :invoke :read [7 nil] INFO [2019-12-17 12:27:51,422] jepsen worker 84 - jepsen.util 84 :ok :read [28 1] INFO [2019-12-17 12:27:51,422] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:51,426] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [23 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,426] jepsen worker 35 - jepsen.util 35 :invoke :cas [11 [35 10]] INFO [2019-12-17 12:27:51,434] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:51,434] jepsen worker 53 - jepsen.util 323 :invoke :read [17 nil] INFO [2019-12-17 12:27:51,436] jepsen worker 79 - jepsen.util 259 :ok :read [26 47] INFO [2019-12-17 12:27:51,436] jepsen worker 79 - jepsen.util 259 :invoke :read [26 nil] INFO [2019-12-17 12:27:51,445] jepsen worker 27 - jepsen.util 27 :ok :write [9 18] INFO [2019-12-17 12:27:51,445] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [30 10]] INFO [2019-12-17 12:27:51,452] jepsen worker 71 - jepsen.util 251 :info :write [23 30] :RequestCanceledException INFO [2019-12-17 12:27:51,453] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,453] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [9 24]] INFO [2019-12-17 12:27:51,454] jepsen worker 71 - jepsen.util 341 :invoke :read [23 nil] INFO [2019-12-17 12:27:51,471] jepsen worker 37 - jepsen.util 127 :ok :read [12 12] INFO [2019-12-17 12:27:51,472] jepsen worker 37 - jepsen.util 127 :invoke :read [12 nil] INFO [2019-12-17 12:27:51,473] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.00390625}, :ns_1@172.28.128.126 {:progress 0.003597122302158251}, :ns_1@172.28.128.127 {:progress 0.007782101167315147}, :ns_1@172.28.128.128 {:progress 0.03018867924528301}} INFO [2019-12-17 12:27:51,473] jepsen nemesis - couchbase.util Rebalance check count: 3 INFO [2019-12-17 12:27:51,473] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:51,484] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,484] jepsen worker 27 - jepsen.util 27 :invoke :write [9 7] INFO [2019-12-17 12:27:51,486] jepsen worker 82 - jepsen.util 262 :info :write [27 27] :RequestCanceledException INFO [2019-12-17 12:27:51,488] jepsen worker 82 - jepsen.util 352 :invoke :write [27 14] INFO [2019-12-17 12:27:51,488] jepsen worker 86 - jepsen.util 86 :ok :read [28 1] INFO [2019-12-17 12:27:51,488] jepsen worker 86 - jepsen.util 86 :invoke :cas [28 [41 27]] INFO [2019-12-17 12:27:51,504] jepsen worker 15 - jepsen.util 285 :ok :read [5 32] INFO [2019-12-17 12:27:51,504] jepsen worker 15 - jepsen.util 285 :invoke :write [5 25] INFO [2019-12-17 12:27:51,505] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:51,505] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1267,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:51,512] jepsen worker 39 - jepsen.util 219 :fail :cas [13 [11 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,513] jepsen worker 39 - jepsen.util 219 :invoke :read [13 nil] INFO [2019-12-17 12:27:51,515] jepsen worker 52 - jepsen.util 232 :fail :cas [17 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,515] jepsen worker 52 - jepsen.util 232 :invoke :read [17 nil] INFO [2019-12-17 12:27:51,519] jepsen worker 41 - jepsen.util 131 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:27:51,519] jepsen worker 81 - jepsen.util 171 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:27:51,521] jepsen worker 41 - jepsen.util 221 :invoke :write [13 43] INFO [2019-12-17 12:27:51,521] jepsen worker 81 - jepsen.util 261 :invoke :read [27 nil] INFO [2019-12-17 12:27:51,530] jepsen worker 68 - jepsen.util 248 :fail :cas [22 [28 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,530] jepsen worker 68 - jepsen.util 248 :invoke :read [22 nil] INFO [2019-12-17 12:27:51,552] jepsen worker 89 - jepsen.util 179 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:27:51,553] jepsen worker 89 - jepsen.util 269 :invoke :cas [29 [32 1]] INFO [2019-12-17 12:27:51,556] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:51,556] jepsen worker 2 - jepsen.util 92 :invoke :read [0 nil] INFO [2019-12-17 12:27:51,568] jepsen worker 53 - jepsen.util 323 :ok :read [17 0] INFO [2019-12-17 12:27:51,568] jepsen worker 53 - jepsen.util 323 :invoke :write [17 22] INFO [2019-12-17 12:27:51,583] jepsen worker 35 - jepsen.util 35 :fail :cas [11 [35 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,583] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,601] jepsen worker 16 - jepsen.util 16 :info :write [5 32] :RequestCanceledException INFO [2019-12-17 12:27:51,602] jepsen worker 16 - jepsen.util 106 :invoke :write [5 23] INFO [2019-12-17 12:27:51,606] jepsen worker 84 - jepsen.util 84 :ok :read [28 1] INFO [2019-12-17 12:27:51,606] jepsen worker 84 - jepsen.util 84 :invoke :cas [28 [23 12]] INFO [2019-12-17 12:27:51,620] jepsen worker 66 - jepsen.util 156 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:27:51,621] jepsen worker 57 - jepsen.util 237 :info :cas [19 [5 41]] :RequestCanceledException INFO [2019-12-17 12:27:51,622] jepsen worker 66 - jepsen.util 246 :invoke :read [22 nil] INFO [2019-12-17 12:27:51,622] jepsen worker 57 - jepsen.util 327 :invoke :read [19 nil] INFO [2019-12-17 12:27:51,637] jepsen worker 58 - jepsen.util 148 :ok :write [19 14] INFO [2019-12-17 12:27:51,637] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:51,647] jepsen worker 49 - jepsen.util 229 :fail :cas [16 [1 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,647] jepsen worker 49 - jepsen.util 229 :invoke :read [16 nil] INFO [2019-12-17 12:27:51,653] jepsen worker 77 - jepsen.util 257 :fail :cas [25 [45 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,653] jepsen worker 77 - jepsen.util 257 :invoke :cas [25 [19 41]] INFO [2019-12-17 12:27:51,670] jepsen worker 27 - jepsen.util 27 :ok :write [9 7] INFO [2019-12-17 12:27:51,671] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:51,687] jepsen worker 48 - jepsen.util 228 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:27:51,688] jepsen worker 48 - jepsen.util 318 :invoke :cas [16 [42 7]] INFO [2019-12-17 12:27:51,690] jepsen worker 27 - jepsen.util 27 :ok :read [9 7] INFO [2019-12-17 12:27:51,690] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [19 43]] INFO [2019-12-17 12:27:51,690] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [40 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,690] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [39 35]] INFO [2019-12-17 12:27:51,698] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,698] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,705] jepsen worker 83 - jepsen.util 263 :info :write [27 48] :RequestCanceledException INFO [2019-12-17 12:27:51,707] jepsen worker 83 - jepsen.util 353 :invoke :read [27 nil] INFO [2019-12-17 12:27:51,709] jepsen worker 1 - jepsen.util 271 :fail :cas [0 [2 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,709] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [19 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,710] jepsen worker 1 - jepsen.util 271 :invoke :read [0 nil] INFO [2019-12-17 12:27:51,710] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [26 34]] INFO [2019-12-17 12:27:51,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:51,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1300,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:51,717] jepsen worker 70 - jepsen.util 250 :info :write [23 41] :RequestCanceledException INFO [2019-12-17 12:27:51,717] jepsen worker 51 - jepsen.util 141 :ok :read [17 0] INFO [2019-12-17 12:27:51,718] jepsen worker 51 - jepsen.util 141 :invoke :read [17 nil] INFO [2019-12-17 12:27:51,718] jepsen worker 70 - jepsen.util 340 :invoke :write [23 19] INFO [2019-12-17 12:27:51,726] jepsen worker 7 - jepsen.util 277 :info :cas [2 [44 26]] :RequestCanceledException INFO [2019-12-17 12:27:51,727] jepsen worker 7 - jepsen.util 367 :invoke :cas [2 [26 34]] INFO [2019-12-17 12:27:51,735] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,735] jepsen worker 5 - jepsen.util 5 :invoke :read [1 nil] INFO [2019-12-17 12:27:51,741] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [26 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,741] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [36 16]] INFO [2019-12-17 12:27:51,752] jepsen worker 40 - jepsen.util 310 :fail :cas [13 [20 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,752] jepsen worker 40 - jepsen.util 310 :invoke :write [13 17] INFO [2019-12-17 12:27:51,768] jepsen worker 86 - jepsen.util 86 :fail :cas [28 [41 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,768] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:51,773] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [36 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,773] jepsen worker 27 - jepsen.util 27 :invoke :write [9 14] INFO [2019-12-17 12:27:51,787] jepsen worker 77 - jepsen.util 257 :fail :cas [25 [19 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,787] jepsen worker 77 - jepsen.util 257 :invoke :read [25 nil] INFO [2019-12-17 12:27:51,802] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,802] jepsen worker 35 - jepsen.util 35 :invoke :read [11 nil] INFO [2019-12-17 12:27:51,804] jepsen worker 13 - jepsen.util 193 :fail :cas [4 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,804] jepsen worker 13 - jepsen.util 193 :invoke :write [4 36] INFO [2019-12-17 12:27:51,805] jepsen worker 45 - jepsen.util 225 :info :write [15 42] :RequestCanceledException INFO [2019-12-17 12:27:51,805] jepsen worker 31 - jepsen.util 301 :info :write [10 43] :RequestCanceledException INFO [2019-12-17 12:27:51,806] jepsen worker 45 - jepsen.util 315 :invoke :cas [15 [27 28]] INFO [2019-12-17 12:27:51,806] jepsen worker 31 - jepsen.util 391 :invoke :write [10 9] INFO [2019-12-17 12:27:51,810] jepsen worker 39 - jepsen.util 219 :ok :read [13 9] INFO [2019-12-17 12:27:51,810] jepsen worker 39 - jepsen.util 219 :invoke :read [13 nil] INFO [2019-12-17 12:27:51,827] jepsen worker 45 - jepsen.util 315 :fail :cas [15 [27 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:51,827] jepsen worker 45 - jepsen.util 315 :invoke :write [15 6] INFO [2019-12-17 12:27:51,833] jepsen worker 81 - jepsen.util 261 :ok :read [27 18] INFO [2019-12-17 12:27:51,833] jepsen worker 81 - jepsen.util 261 :invoke :write [27 7] INFO [2019-12-17 12:27:51,851] jepsen worker 46 - jepsen.util 226 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:27:51,853] jepsen worker 46 - jepsen.util 316 :invoke :cas [15 [16 30]] INFO [2019-12-17 12:27:51,873] jepsen worker 35 - jepsen.util 35 :ok :read [11 7] INFO [2019-12-17 12:27:51,873] jepsen worker 35 - jepsen.util 35 :invoke :write [11 3] INFO [2019-12-17 12:27:51,903] jepsen worker 2 - jepsen.util 92 :ok :read [0 14] INFO [2019-12-17 12:27:51,904] jepsen worker 2 - jepsen.util 92 :invoke :write [0 5] INFO [2019-12-17 12:27:51,921] jepsen worker 80 - jepsen.util 350 :info :write [26 43] :RequestCanceledException INFO [2019-12-17 12:27:51,923] jepsen worker 80 - jepsen.util 440 :invoke :read [26 nil] INFO [2019-12-17 12:27:51,934] jepsen worker 75 - jepsen.util 255 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:51,934] jepsen worker 26 - jepsen.util 296 :info :cas [8 [47 9]] :RequestCanceledException INFO [2019-12-17 12:27:51,934] jepsen worker 75 - jepsen.util 255 :invoke :cas [25 [43 45]] INFO [2019-12-17 12:27:51,935] jepsen worker 26 - jepsen.util 386 :invoke :write [8 33] INFO [2019-12-17 12:27:51,937] jepsen worker 49 - jepsen.util 229 :ok :read [16 8] INFO [2019-12-17 12:27:51,937] jepsen worker 49 - jepsen.util 229 :invoke :write [16 5] INFO [2019-12-17 12:27:51,946] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:51,947] jepsen worker 20 - jepsen.util 290 :invoke :read [6 nil] INFO [2019-12-17 12:27:51,956] jepsen worker 88 - jepsen.util 268 :ok :read [29 5] INFO [2019-12-17 12:27:51,956] jepsen worker 88 - jepsen.util 268 :invoke :cas [29 [23 9]] INFO [2019-12-17 12:27:51,956] jepsen worker 83 - jepsen.util 353 :ok :read [27 18] INFO [2019-12-17 12:27:51,957] jepsen worker 83 - jepsen.util 353 :invoke :write [27 33] INFO [2019-12-17 12:27:51,972] jepsen worker 1 - jepsen.util 271 :ok :read [0 14] INFO [2019-12-17 12:27:51,972] jepsen worker 1 - jepsen.util 271 :invoke :cas [0 [46 12]] INFO [2019-12-17 12:27:51,984] jepsen worker 59 - jepsen.util 239 :ok :write [19 21] INFO [2019-12-17 12:27:51,984] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:51,987] jepsen worker 14 - jepsen.util 194 :info :write [4 3] :RequestCanceledException INFO [2019-12-17 12:27:51,988] jepsen worker 14 - jepsen.util 284 :invoke :read [4 nil] INFO [2019-12-17 12:27:52,004] jepsen worker 55 - jepsen.util 235 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:27:52,005] jepsen worker 12 - jepsen.util 192 :fail :cas [4 [38 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,005] jepsen worker 12 - jepsen.util 192 :invoke :read [4 nil] INFO [2019-12-17 12:27:52,005] jepsen worker 55 - jepsen.util 325 :invoke :write [18 12] INFO [2019-12-17 12:27:52,018] jepsen worker 20 - jepsen.util 290 :ok :read [6 45] INFO [2019-12-17 12:27:52,019] jepsen worker 59 - jepsen.util 239 :ok :read [19 21] INFO [2019-12-17 12:27:52,019] jepsen worker 20 - jepsen.util 290 :invoke :cas [6 [1 14]] INFO [2019-12-17 12:27:52,019] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,020] jepsen worker 47 - jepsen.util 227 :info :write [15 24] :RequestCanceledException INFO [2019-12-17 12:27:52,020] jepsen worker 21 - jepsen.util 201 :info :write [7 34] :RequestCanceledException INFO [2019-12-17 12:27:52,020] jepsen worker 33 - jepsen.util 213 :info :write [11 42] :RequestCanceledException INFO [2019-12-17 12:27:52,021] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:52,021] jepsen worker 47 - jepsen.util 317 :invoke :read [15 nil] INFO [2019-12-17 12:27:52,021] jepsen worker 33 - jepsen.util 303 :invoke :write [11 22] INFO [2019-12-17 12:27:52,068] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:52,068] jepsen worker 37 - jepsen.util 127 :ok :read [12 12] INFO [2019-12-17 12:27:52,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1346,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:52,069] jepsen worker 37 - jepsen.util 127 :invoke :cas [12 [32 15]] INFO [2019-12-17 12:27:52,069] jepsen worker 85 - jepsen.util 265 :ok :write [28 44] INFO [2019-12-17 12:27:52,069] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:52,069] jepsen worker 85 - jepsen.util 265 :invoke :read [28 nil] INFO [2019-12-17 12:27:52,069] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1350,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:52,076] jepsen worker 20 - jepsen.util 290 :fail :cas [6 [1 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,076] jepsen worker 20 - jepsen.util 290 :invoke :write [6 38] INFO [2019-12-17 12:27:52,085] jepsen worker 86 - jepsen.util 86 :ok :read [28 44] INFO [2019-12-17 12:27:52,085] jepsen worker 86 - jepsen.util 86 :invoke :read [28 nil] INFO [2019-12-17 12:27:52,106] jepsen worker 28 - jepsen.util 118 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:27:52,106] jepsen worker 8 - jepsen.util 188 :info :write [2 18] :RequestCanceledException INFO [2019-12-17 12:27:52,107] jepsen worker 59 - jepsen.util 239 :ok :read [19 21] INFO [2019-12-17 12:27:52,107] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,107] jepsen worker 28 - jepsen.util 208 :invoke :write [9 28] INFO [2019-12-17 12:27:52,107] jepsen worker 8 - jepsen.util 278 :invoke :cas [2 [6 45]] INFO [2019-12-17 12:27:52,118] jepsen worker 32 - jepsen.util 122 :info :write [10 16] :SyncWriteAmbiguous INFO [2019-12-17 12:27:52,118] jepsen worker 60 - jepsen.util 330 :info :write [20 3] :RequestCanceledException INFO [2019-12-17 12:27:52,119] jepsen worker 32 - jepsen.util 212 :invoke :cas [10 [39 17]] INFO [2019-12-17 12:27:52,119] jepsen worker 60 - jepsen.util 420 :invoke :read [20 nil] INFO [2019-12-17 12:27:52,128] jepsen worker 3 - jepsen.util 183 :fail :cas [1 [41 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,128] jepsen worker 3 - jepsen.util 183 :invoke :write [1 42] INFO [2019-12-17 12:27:52,132] jepsen worker 36 - jepsen.util 216 :info :cas [12 [12 25]] :RequestCanceledException INFO [2019-12-17 12:27:52,134] jepsen worker 36 - jepsen.util 306 :invoke :cas [12 [22 38]] INFO [2019-12-17 12:27:52,136] jepsen worker 39 - jepsen.util 219 :ok :read [13 9] INFO [2019-12-17 12:27:52,136] jepsen worker 39 - jepsen.util 219 :invoke :write [13 37] INFO [2019-12-17 12:27:52,141] jepsen worker 59 - jepsen.util 239 :ok :read [19 21] INFO [2019-12-17 12:27:52,141] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [36 19]] INFO [2019-12-17 12:27:52,161] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [36 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,161] jepsen worker 59 - jepsen.util 239 :invoke :write [19 35] INFO [2019-12-17 12:27:52,164] jepsen worker 58 - jepsen.util 148 :ok :read [19 21] INFO [2019-12-17 12:27:52,164] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,169] jepsen worker 32 - jepsen.util 212 :fail :cas [10 [39 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,169] jepsen worker 32 - jepsen.util 212 :invoke :read [10 nil] INFO [2019-12-17 12:27:52,176] jepsen worker 10 - jepsen.util 190 :fail :cas [3 [24 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,176] jepsen worker 10 - jepsen.util 190 :invoke :cas [3 [44 14]] INFO [2019-12-17 12:27:52,181] jepsen worker 27 - jepsen.util 27 :ok :write [9 14] INFO [2019-12-17 12:27:52,181] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [41 40]] INFO [2019-12-17 12:27:52,192] jepsen worker 46 - jepsen.util 316 :fail :cas [15 [16 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,192] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:52,206] jepsen worker 64 - jepsen.util 244 :info :write [21 42] :RequestCanceledException INFO [2019-12-17 12:27:52,207] jepsen worker 64 - jepsen.util 334 :invoke :write [21 47] INFO [2019-12-17 12:27:52,220] jepsen worker 61 - jepsen.util 241 :info :write [20 49] :RequestCanceledException INFO [2019-12-17 12:27:52,220] jepsen worker 42 - jepsen.util 312 :info :write [14 13] :RequestCanceledException INFO [2019-12-17 12:27:52,222] jepsen worker 61 - jepsen.util 331 :invoke :cas [20 [7 13]] INFO [2019-12-17 12:27:52,222] jepsen worker 42 - jepsen.util 402 :invoke :write [14 40] INFO [2019-12-17 12:27:52,227] jepsen worker 51 - jepsen.util 141 :ok :read [17 0] INFO [2019-12-17 12:27:52,227] jepsen worker 51 - jepsen.util 141 :invoke :cas [17 [42 32]] INFO [2019-12-17 12:27:52,248] jepsen worker 25 - jepsen.util 295 :info :write [8 7] :RequestCanceledException INFO [2019-12-17 12:27:52,249] jepsen worker 25 - jepsen.util 385 :invoke :cas [8 [15 27]] INFO [2019-12-17 12:27:52,251] jepsen worker 57 - jepsen.util 327 :ok :read [19 21] INFO [2019-12-17 12:27:52,251] jepsen worker 57 - jepsen.util 327 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,253] jepsen worker 27 - jepsen.util 27 :fail :cas [9 [41 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,253] jepsen worker 27 - jepsen.util 27 :invoke :read [9 nil] INFO [2019-12-17 12:27:52,254] jepsen worker 47 - jepsen.util 317 :ok :read [15 35] INFO [2019-12-17 12:27:52,254] jepsen worker 47 - jepsen.util 317 :invoke :write [15 32] INFO [2019-12-17 12:27:52,259] jepsen worker 89 - jepsen.util 269 :fail :cas [29 [32 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,260] jepsen worker 89 - jepsen.util 269 :invoke :read [29 nil] INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:52,288] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:52,291] jepsen worker 32 - jepsen.util 212 :ok :read [10 23] INFO [2019-12-17 12:27:52,291] jepsen worker 32 - jepsen.util 212 :invoke :write [10 47] INFO [2019-12-17 12:27:52,300] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [9 24]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,300] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [19 11]] INFO [2019-12-17 12:27:52,316] jepsen worker 10 - jepsen.util 190 :fail :cas [3 [44 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,317] jepsen worker 10 - jepsen.util 190 :invoke :write [3 12] INFO [2019-12-17 12:27:52,320] jepsen worker 63 - jepsen.util 153 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:27:52,320] jepsen worker 63 - jepsen.util 153 :invoke :write [21 42] INFO [2019-12-17 12:27:52,335] jepsen worker 18 - jepsen.util 198 :info :write [6 27] :RequestCanceledException INFO [2019-12-17 12:27:52,336] jepsen worker 18 - jepsen.util 288 :invoke :write [6 12] INFO [2019-12-17 12:27:52,348] jepsen worker 65 - jepsen.util 245 :info :cas [21 [19 12]] :RequestCanceledException INFO [2019-12-17 12:27:52,349] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,354] jepsen worker 25 - jepsen.util 385 :fail :cas [8 [15 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,355] jepsen worker 25 - jepsen.util 385 :invoke :cas [8 [21 45]] INFO [2019-12-17 12:27:52,374] jepsen worker 11 - jepsen.util 191 :info :write [3 21] :SyncWriteAmbiguous INFO [2019-12-17 12:27:52,376] jepsen worker 11 - jepsen.util 281 :invoke :read [3 nil] INFO [2019-12-17 12:27:52,386] jepsen worker 27 - jepsen.util 27 :ok :read [9 14] INFO [2019-12-17 12:27:52,387] jepsen worker 27 - jepsen.util 27 :invoke :cas [9 [25 37]] INFO [2019-12-17 12:27:52,394] jepsen worker 36 - jepsen.util 306 :fail :cas [12 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,395] jepsen worker 36 - jepsen.util 306 :invoke :cas [12 [10 22]] INFO [2019-12-17 12:27:52,405] jepsen worker 88 - jepsen.util 268 :fail :cas [29 [23 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,405] jepsen worker 88 - jepsen.util 268 :invoke :write [29 40] INFO [2019-12-17 12:27:52,422] jepsen worker 46 - jepsen.util 316 :ok :read [15 35] INFO [2019-12-17 12:27:52,422] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:52,468] jepsen worker 73 - jepsen.util 253 :ok :read [24 35] INFO [2019-12-17 12:27:52,468] jepsen worker 73 - jepsen.util 253 :invoke :write [24 39] INFO [2019-12-17 12:27:52,497] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.01171875}, :ns_1@172.28.128.126 {:progress 0.01079136690647486}, :ns_1@172.28.128.127 {:progress 0.0311284046692607}, :ns_1@172.28.128.128 {:progress 0.03773584905660377}} INFO [2019-12-17 12:27:52,497] jepsen nemesis - couchbase.util Rebalance check count: 4 INFO [2019-12-17 12:27:52,497] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:52,500] jepsen worker 14 - jepsen.util 284 :ok :read [4 4] INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1412,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:52,500] jepsen worker 14 - jepsen.util 284 :invoke :read [4 nil] INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:52,500] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1415,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:52,516] jepsen worker 64 - jepsen.util 334 :ok :write [21 47] INFO [2019-12-17 12:27:52,516] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,519] jepsen worker 59 - jepsen.util 239 :ok :write [19 35] INFO [2019-12-17 12:27:52,520] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,532] jepsen worker 36 - jepsen.util 306 :fail :cas [12 [10 22]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,532] jepsen worker 36 - jepsen.util 306 :invoke :write [12 5] INFO [2019-12-17 12:27:52,545] jepsen worker 59 - jepsen.util 239 :ok :read [19 35] INFO [2019-12-17 12:27:52,545] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [35 8]] INFO [2019-12-17 12:27:52,549] jepsen worker 38 - jepsen.util 218 :ok :read [12 12] INFO [2019-12-17 12:27:52,549] jepsen worker 75 - jepsen.util 255 :fail :cas [25 [43 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,549] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:52,549] jepsen worker 75 - jepsen.util 255 :invoke :write [25 13] INFO [2019-12-17 12:27:52,555] jepsen worker 12 - jepsen.util 192 :ok :read [4 4] INFO [2019-12-17 12:27:52,556] jepsen worker 12 - jepsen.util 192 :invoke :write [4 18] INFO [2019-12-17 12:27:52,576] jepsen worker 37 - jepsen.util 127 :fail :cas [12 [32 15]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,577] jepsen worker 37 - jepsen.util 127 :invoke :write [12 36] INFO [2019-12-17 12:27:52,608] jepsen worker 80 - jepsen.util 440 :ok :read [26 47] INFO [2019-12-17 12:27:52,609] jepsen worker 80 - jepsen.util 440 :invoke :write [26 13] INFO [2019-12-17 12:27:52,621] jepsen worker 0 - jepsen.util 180 :info :cas [0 [47 40]] :RequestCanceledException INFO [2019-12-17 12:27:52,622] jepsen worker 0 - jepsen.util 270 :invoke :cas [0 [40 4]] INFO [2019-12-17 12:27:52,655] jepsen worker 51 - jepsen.util 141 :fail :cas [17 [42 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,656] jepsen worker 51 - jepsen.util 141 :invoke :cas [17 [19 1]] INFO [2019-12-17 12:27:52,661] jepsen worker 59 - jepsen.util 239 :ok :cas [19 [35 8]] INFO [2019-12-17 12:27:52,662] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [7 4]] INFO [2019-12-17 12:27:52,678] jepsen worker 0 - jepsen.util 270 :fail :cas [0 [40 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,678] jepsen worker 0 - jepsen.util 270 :invoke :read [0 nil] INFO [2019-12-17 12:27:52,690] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [7 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,691] jepsen worker 59 - jepsen.util 239 :invoke :cas [19 [6 5]] INFO [2019-12-17 12:27:52,695] jepsen worker 52 - jepsen.util 232 :ok :read [17 0] INFO [2019-12-17 12:27:52,695] jepsen worker 52 - jepsen.util 232 :invoke :write [17 23] INFO [2019-12-17 12:27:52,716] jepsen worker 0 - jepsen.util 270 :ok :read [0 14] INFO [2019-12-17 12:27:52,716] jepsen worker 0 - jepsen.util 270 :invoke :write [0 9] INFO [2019-12-17 12:27:52,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1433,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1436,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:52,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:52,731] jepsen worker 59 - jepsen.util 239 :fail :cas [19 [6 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,731] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:52,778] jepsen worker 67 - jepsen.util 247 :info :cas [22 [33 35]] :RequestCanceledException INFO [2019-12-17 12:27:52,780] jepsen worker 67 - jepsen.util 337 :invoke :write [22 34] INFO [2019-12-17 12:27:52,803] jepsen worker 69 - jepsen.util 249 :info :cas [23 [29 44]] :RequestCanceledException INFO [2019-12-17 12:27:52,805] jepsen worker 69 - jepsen.util 339 :invoke :cas [23 [12 12]] INFO [2019-12-17 12:27:52,831] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:52,831] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:52,848] jepsen worker 43 - jepsen.util 313 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:27:52,849] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [19 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,849] jepsen worker 23 - jepsen.util 113 :invoke :read [7 nil] INFO [2019-12-17 12:27:52,849] jepsen worker 43 - jepsen.util 403 :invoke :write [14 1] INFO [2019-12-17 12:27:52,851] jepsen worker 63 - jepsen.util 153 :ok :write [21 42] INFO [2019-12-17 12:27:52,852] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,897] jepsen worker 63 - jepsen.util 153 :ok :read [21 42] INFO [2019-12-17 12:27:52,897] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [24 45]] INFO [2019-12-17 12:27:52,906] jepsen worker 60 - jepsen.util 420 :ok :read [20 7] INFO [2019-12-17 12:27:52,907] jepsen worker 60 - jepsen.util 420 :invoke :write [20 22] INFO [2019-12-17 12:27:52,924] jepsen worker 51 - jepsen.util 141 :fail :cas [17 [19 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,924] jepsen worker 51 - jepsen.util 141 :invoke :cas [17 [20 46]] INFO [2019-12-17 12:27:52,929] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,929] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [36 40]] INFO [2019-12-17 12:27:52,932] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:52,933] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1449,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:52,949] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [36 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,949] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [0 13]] INFO [2019-12-17 12:27:52,966] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [0 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:52,967] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:52,986] jepsen worker 63 - jepsen.util 153 :ok :read [21 42] INFO [2019-12-17 12:27:52,986] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [16 19]] INFO [2019-12-17 12:27:53,008] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [16 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,008] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [48 6]] INFO [2019-12-17 12:27:53,021] jepsen worker 44 - jepsen.util 134 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:27:53,021] jepsen worker 6 - jepsen.util 186 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:27:53,023] jepsen worker 6 - jepsen.util 276 :invoke :cas [2 [3 34]] INFO [2019-12-17 12:27:53,023] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [21 11]] INFO [2019-12-17 12:27:53,039] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [48 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,039] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [4 25]] INFO [2019-12-17 12:27:53,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:53,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:53,051] jepsen worker 19 - jepsen.util 199 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,051] jepsen worker 19 - jepsen.util 199 :invoke :write [6 14] INFO [2019-12-17 12:27:53,066] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,066] jepsen worker 63 - jepsen.util 153 :invoke :write [21 22] INFO [2019-12-17 12:27:53,086] jepsen worker 50 - jepsen.util 320 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,086] jepsen worker 30 - jepsen.util 30 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:27:53,086] jepsen worker 29 - jepsen.util 119 :info :write [9 36] :RequestCanceledException INFO [2019-12-17 12:27:53,086] jepsen worker 50 - jepsen.util 320 :invoke :write [16 0] INFO [2019-12-17 12:27:53,087] jepsen worker 30 - jepsen.util 120 :invoke :cas [10 [28 29]] INFO [2019-12-17 12:27:53,087] jepsen worker 29 - jepsen.util 209 :invoke :write [9 39] INFO [2019-12-17 12:27:53,125] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:53,125] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:53,154] jepsen worker 22 - jepsen.util 292 :ok :read [7 20] INFO [2019-12-17 12:27:53,154] jepsen worker 22 - jepsen.util 292 :invoke :read [7 nil] INFO [2019-12-17 12:27:53,177] jepsen worker 63 - jepsen.util 153 :ok :write [21 22] INFO [2019-12-17 12:27:53,178] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,181] jepsen worker 14 - jepsen.util 284 :ok :read [4 4] INFO [2019-12-17 12:27:53,181] jepsen worker 14 - jepsen.util 284 :invoke :cas [4 [3 26]] INFO [2019-12-17 12:27:53,186] jepsen worker 72 - jepsen.util 162 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,186] jepsen worker 74 - jepsen.util 254 :info :cas [24 [2 8]] :RequestCanceledException INFO [2019-12-17 12:27:53,186] jepsen worker 9 - jepsen.util 99 :info :write [3 42] :RequestCanceledException INFO [2019-12-17 12:27:53,186] jepsen worker 72 - jepsen.util 162 :invoke :read [24 nil] INFO [2019-12-17 12:27:53,187] jepsen worker 9 - jepsen.util 189 :invoke :read [3 nil] INFO [2019-12-17 12:27:53,187] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:53,198] jepsen worker 63 - jepsen.util 153 :ok :read [21 22] INFO [2019-12-17 12:27:53,199] jepsen worker 63 - jepsen.util 153 :invoke :write [21 17] INFO [2019-12-17 12:27:53,216] jepsen worker 34 - jepsen.util 214 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:27:53,217] jepsen worker 34 - jepsen.util 304 :invoke :cas [11 [35 16]] INFO [2019-12-17 12:27:53,249] jepsen worker 63 - jepsen.util 153 :ok :write [21 17] INFO [2019-12-17 12:27:53,249] jepsen worker 4 - jepsen.util 184 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:27:53,249] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,250] jepsen worker 4 - jepsen.util 274 :invoke :cas [1 [3 28]] INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1487,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:53,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1488,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:53,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:53,261] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:53,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:53,279] jepsen worker 63 - jepsen.util 153 :ok :read [21 17] INFO [2019-12-17 12:27:53,279] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,287] jepsen worker 56 - jepsen.util 146 :info :write [18 32] :RequestCanceledException INFO [2019-12-17 12:27:53,289] jepsen worker 56 - jepsen.util 236 :invoke :write [18 39] INFO [2019-12-17 12:27:53,299] jepsen worker 63 - jepsen.util 153 :ok :read [21 17] INFO [2019-12-17 12:27:53,299] jepsen worker 63 - jepsen.util 153 :invoke :write [21 49] INFO [2019-12-17 12:27:53,306] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [39 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,306] jepsen worker 87 - jepsen.util 177 :invoke :cas [29 [11 47]] INFO [2019-12-17 12:27:53,321] jepsen worker 76 - jepsen.util 166 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:27:53,322] jepsen worker 76 - jepsen.util 256 :invoke :read [25 nil] INFO [2019-12-17 12:27:53,332] jepsen worker 6 - jepsen.util 276 :fail :cas [2 [3 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,332] jepsen worker 6 - jepsen.util 276 :invoke :cas [2 [24 23]] INFO [2019-12-17 12:27:53,352] jepsen worker 54 - jepsen.util 324 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:27:53,352] jepsen worker 78 - jepsen.util 258 :info :write [26 27] :RequestCanceledException INFO [2019-12-17 12:27:53,353] jepsen worker 54 - jepsen.util 414 :invoke :write [18 15] INFO [2019-12-17 12:27:53,353] jepsen worker 78 - jepsen.util 348 :invoke :read [26 nil] INFO [2019-12-17 12:27:53,376] jepsen worker 14 - jepsen.util 284 :fail :cas [4 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,376] jepsen worker 14 - jepsen.util 284 :invoke :write [4 19] INFO [2019-12-17 12:27:53,378] jepsen worker 63 - jepsen.util 153 :ok :write [21 49] INFO [2019-12-17 12:27:53,378] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,385] jepsen worker 17 - jepsen.util 287 :info :cas [5 [7 1]] :RequestCanceledException INFO [2019-12-17 12:27:53,386] jepsen worker 17 - jepsen.util 377 :invoke :read [5 nil] INFO [2019-12-17 12:27:53,394] jepsen worker 25 - jepsen.util 385 :fail :cas [8 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,394] jepsen worker 25 - jepsen.util 385 :invoke :write [8 24] INFO [2019-12-17 12:27:53,404] jepsen worker 62 - jepsen.util 242 :info :write [20 42] :RequestCanceledException INFO [2019-12-17 12:27:53,406] jepsen worker 62 - jepsen.util 332 :invoke :cas [20 [23 31]] INFO [2019-12-17 12:27:53,410] jepsen worker 63 - jepsen.util 153 :ok :read [21 49] INFO [2019-12-17 12:27:53,410] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [21 25]] INFO [2019-12-17 12:27:53,427] jepsen worker 24 - jepsen.util 114 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:27:53,429] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [21 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,429] jepsen worker 24 - jepsen.util 204 :invoke :write [8 22] INFO [2019-12-17 12:27:53,429] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [24 1]] INFO [2019-12-17 12:27:53,442] jepsen worker 15 - jepsen.util 285 :info :write [5 25] :SyncWriteAmbiguous INFO [2019-12-17 12:27:53,444] jepsen worker 15 - jepsen.util 375 :invoke :read [5 nil] INFO [2019-12-17 12:27:53,450] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,450] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [16 33]] INFO [2019-12-17 12:27:53,476] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [16 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,476] jepsen worker 63 - jepsen.util 153 :invoke :write [21 32] INFO [2019-12-17 12:27:53,479] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:53,479] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:53,479] jepsen worker 71 - jepsen.util 341 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,479] jepsen worker 79 - jepsen.util 259 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,479] jepsen worker 71 - jepsen.util 341 :invoke :cas [23 [2 44]] INFO [2019-12-17 12:27:53,479] jepsen worker 79 - jepsen.util 259 :invoke :write [26 9] INFO [2019-12-17 12:27:53,482] jepsen worker 84 - jepsen.util 84 :fail :cas [28 [23 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,483] jepsen worker 84 - jepsen.util 84 :invoke :read [28 nil] INFO [2019-12-17 12:27:53,504] jepsen worker 82 - jepsen.util 352 :info :write [27 14] :RequestCanceledException INFO [2019-12-17 12:27:53,505] jepsen worker 82 - jepsen.util 442 :invoke :read [27 nil] INFO [2019-12-17 12:27:53,509] jepsen worker 4 - jepsen.util 274 :fail :cas [1 [3 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,509] jepsen worker 4 - jepsen.util 274 :invoke :read [1 nil] INFO [2019-12-17 12:27:53,534] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0234375}, :ns_1@172.28.128.126 {:progress 0.02158273381294962}, :ns_1@172.28.128.127 {:progress 0.05447470817120625}, :ns_1@172.28.128.128 {:progress 0.03773584905660377}} INFO [2019-12-17 12:27:53,534] jepsen nemesis - couchbase.util Rebalance check count: 5 INFO [2019-12-17 12:27:53,534] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:53,535] jepsen worker 63 - jepsen.util 153 :ok :write [21 32] INFO [2019-12-17 12:27:53,535] jepsen worker 63 - jepsen.util 153 :invoke :write [21 0] INFO [2019-12-17 12:27:53,546] jepsen worker 22 - jepsen.util 292 :ok :read [7 20] INFO [2019-12-17 12:27:53,546] jepsen worker 22 - jepsen.util 292 :invoke :write [7 2] INFO [2019-12-17 12:27:53,558] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:53,559] jepsen worker 21 - jepsen.util 291 :invoke :cas [7 [33 39]] INFO [2019-12-17 12:27:53,577] jepsen worker 34 - jepsen.util 304 :fail :cas [11 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,577] jepsen worker 34 - jepsen.util 304 :invoke :write [11 44] INFO [2019-12-17 12:27:53,612] jepsen worker 51 - jepsen.util 141 :fail :cas [17 [20 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,612] jepsen worker 51 - jepsen.util 141 :invoke :write [17 21] INFO [2019-12-17 12:27:53,618] jepsen worker 4 - jepsen.util 274 :ok :read [1 23] INFO [2019-12-17 12:27:53,618] jepsen worker 4 - jepsen.util 274 :invoke :cas [1 [46 0]] INFO [2019-12-17 12:27:53,621] jepsen worker 53 - jepsen.util 323 :info :write [17 22] :RequestCanceledException INFO [2019-12-17 12:27:53,621] jepsen worker 68 - jepsen.util 248 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,621] jepsen worker 16 - jepsen.util 106 :info :write [5 23] :RequestCanceledException INFO [2019-12-17 12:27:53,621] jepsen worker 68 - jepsen.util 248 :invoke :write [22 25] INFO [2019-12-17 12:27:53,621] jepsen worker 41 - jepsen.util 221 :info :write [13 43] :RequestCanceledException INFO [2019-12-17 12:27:53,622] jepsen worker 16 - jepsen.util 196 :invoke :write [5 40] INFO [2019-12-17 12:27:53,622] jepsen worker 53 - jepsen.util 413 :invoke :read [17 nil] INFO [2019-12-17 12:27:53,622] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:27:53,629] jepsen worker 76 - jepsen.util 256 :ok :read [25 2] INFO [2019-12-17 12:27:53,629] jepsen worker 76 - jepsen.util 256 :invoke :cas [25 [24 37]] INFO [2019-12-17 12:27:53,646] jepsen worker 63 - jepsen.util 153 :ok :write [21 0] INFO [2019-12-17 12:27:53,646] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [1 6]] INFO [2019-12-17 12:27:53,661] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [1 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,662] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,677] jepsen worker 5 - jepsen.util 5 :ok :read [1 23] INFO [2019-12-17 12:27:53,677] jepsen worker 5 - jepsen.util 5 :invoke :cas [1 [30 45]] INFO [2019-12-17 12:27:53,679] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,680] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [46 39]] INFO [2019-12-17 12:27:53,687] jepsen worker 66 - jepsen.util 246 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,687] jepsen worker 66 - jepsen.util 246 :invoke :write [22 24] INFO [2019-12-17 12:27:53,755] jepsen worker 70 - jepsen.util 340 :info :write [23 19] :RequestCanceledException INFO [2019-12-17 12:27:53,755] jepsen worker 48 - jepsen.util 318 :info :cas [16 [42 7]] :RequestCanceledException INFO [2019-12-17 12:27:53,755] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:53,755] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1544,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:53,756] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:53,756] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1547,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:53,757] jepsen worker 70 - jepsen.util 430 :invoke :write [23 48] INFO [2019-12-17 12:27:53,757] jepsen worker 48 - jepsen.util 408 :invoke :read [16 nil] INFO [2019-12-17 12:27:53,764] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,765] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [33 13]] INFO [2019-12-17 12:27:53,765] jepsen worker 89 - jepsen.util 269 :ok :read [29 5] INFO [2019-12-17 12:27:53,765] jepsen worker 89 - jepsen.util 269 :invoke :write [29 28] INFO [2019-12-17 12:27:53,778] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [33 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,779] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [18 26]] INFO [2019-12-17 12:27:53,801] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [18 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,801] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,812] jepsen worker 76 - jepsen.util 256 :fail :cas [25 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,812] jepsen worker 76 - jepsen.util 256 :invoke :read [25 nil] INFO [2019-12-17 12:27:53,820] jepsen worker 7 - jepsen.util 367 :info :cas [2 [26 34]] :RequestCanceledException INFO [2019-12-17 12:27:53,820] jepsen worker 77 - jepsen.util 257 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:27:53,820] jepsen worker 77 - jepsen.util 257 :invoke :write [25 43] INFO [2019-12-17 12:27:53,821] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [34 37]] INFO [2019-12-17 12:27:53,822] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,822] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [9 35]] INFO [2019-12-17 12:27:53,827] jepsen worker 6 - jepsen.util 276 :fail :cas [2 [24 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,827] jepsen worker 6 - jepsen.util 276 :invoke :write [2 47] INFO [2019-12-17 12:27:53,834] jepsen worker 40 - jepsen.util 310 :info :write [13 17] :RequestCanceledException INFO [2019-12-17 12:27:53,835] jepsen worker 40 - jepsen.util 400 :invoke :write [13 11] INFO [2019-12-17 12:27:53,845] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [9 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,845] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:53,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:53,867] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,867] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:53,884] jepsen worker 63 - jepsen.util 153 :ok :read [21 0] INFO [2019-12-17 12:27:53,884] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [14 11]] INFO [2019-12-17 12:27:53,887] jepsen worker 13 - jepsen.util 193 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:27:53,887] jepsen worker 45 - jepsen.util 315 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:27:53,887] jepsen worker 35 - jepsen.util 35 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:27:53,888] jepsen worker 45 - jepsen.util 405 :invoke :read [15 nil] INFO [2019-12-17 12:27:53,888] jepsen worker 13 - jepsen.util 283 :invoke :cas [4 [11 32]] INFO [2019-12-17 12:27:53,888] jepsen worker 35 - jepsen.util 125 :invoke :write [11 34] INFO [2019-12-17 12:27:53,904] jepsen worker 63 - jepsen.util 153 :fail :cas [21 [14 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,905] jepsen worker 63 - jepsen.util 153 :invoke :write [21 1] INFO [2019-12-17 12:27:53,905] jepsen worker 31 - jepsen.util 391 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:27:53,906] jepsen worker 31 - jepsen.util 481 :invoke :cas [10 [36 48]] INFO [2019-12-17 12:27:53,925] jepsen worker 23 - jepsen.util 113 :ok :read [7 20] INFO [2019-12-17 12:27:53,925] jepsen worker 23 - jepsen.util 113 :invoke :cas [7 [12 44]] INFO [2019-12-17 12:27:53,926] jepsen worker 4 - jepsen.util 274 :fail :cas [1 [46 0]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,926] jepsen worker 4 - jepsen.util 274 :invoke :write [1 29] INFO [2019-12-17 12:27:53,931] jepsen worker 81 - jepsen.util 261 :info :write [27 7] :RequestCanceledException INFO [2019-12-17 12:27:53,931] jepsen worker 2 - jepsen.util 92 :info :write [0 5] :RequestCanceledException INFO [2019-12-17 12:27:53,933] jepsen worker 81 - jepsen.util 351 :invoke :read [27 nil] INFO [2019-12-17 12:27:53,933] jepsen worker 2 - jepsen.util 182 :invoke :write [0 31] INFO [2019-12-17 12:27:53,943] jepsen worker 31 - jepsen.util 481 :fail :cas [10 [36 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,944] jepsen worker 31 - jepsen.util 481 :invoke :write [10 37] INFO [2019-12-17 12:27:53,952] jepsen worker 26 - jepsen.util 386 :info :write [8 33] :RequestCanceledException INFO [2019-12-17 12:27:53,954] jepsen worker 26 - jepsen.util 476 :invoke :read [8 nil] INFO [2019-12-17 12:27:53,962] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [34 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:53,962] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [38 40]] INFO [2019-12-17 12:27:53,964] jepsen worker 20 - jepsen.util 290 :info :write [6 38] :SyncWriteAmbiguous INFO [2019-12-17 12:27:53,965] jepsen worker 20 - jepsen.util 380 :invoke :read [6 nil] INFO [2019-12-17 12:27:53,967] jepsen worker 45 - jepsen.util 405 :ok :read [15 35] INFO [2019-12-17 12:27:53,967] jepsen worker 45 - jepsen.util 405 :invoke :write [15 16] INFO [2019-12-17 12:27:54,007] jepsen worker 53 - jepsen.util 413 :ok :read [17 0] INFO [2019-12-17 12:27:54,007] jepsen worker 53 - jepsen.util 413 :invoke :write [17 34] INFO [2019-12-17 12:27:54,009] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [21 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,009] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [47 17]] INFO [2019-12-17 12:27:54,034] jepsen worker 83 - jepsen.util 353 :info :write [27 33] :RequestCanceledException INFO [2019-12-17 12:27:54,034] jepsen worker 49 - jepsen.util 229 :info :write [16 5] :RequestCanceledException INFO [2019-12-17 12:27:54,035] jepsen worker 1 - jepsen.util 271 :info :cas [0 [46 12]] :RequestCanceledException INFO [2019-12-17 12:27:54,036] jepsen worker 49 - jepsen.util 319 :invoke :cas [16 [12 21]] INFO [2019-12-17 12:27:54,036] jepsen worker 83 - jepsen.util 443 :invoke :read [27 nil] INFO [2019-12-17 12:27:54,036] jepsen worker 1 - jepsen.util 361 :invoke :cas [0 [17 25]] INFO [2019-12-17 12:27:54,043] jepsen worker 78 - jepsen.util 348 :ok :read [26 47] INFO [2019-12-17 12:27:54,043] jepsen worker 78 - jepsen.util 348 :invoke :write [26 19] INFO [2019-12-17 12:27:54,046] jepsen worker 63 - jepsen.util 153 :ok :write [21 1] INFO [2019-12-17 12:27:54,046] jepsen worker 63 - jepsen.util 153 :invoke :write [21 19] INFO [2019-12-17 12:27:54,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:54,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1591,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:54,087] jepsen worker 55 - jepsen.util 325 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:27:54,088] jepsen worker 55 - jepsen.util 415 :invoke :write [18 2] INFO [2019-12-17 12:27:54,099] jepsen worker 62 - jepsen.util 332 :fail :cas [20 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,100] jepsen worker 62 - jepsen.util 332 :invoke :cas [20 [48 25]] INFO [2019-12-17 12:27:54,104] jepsen worker 85 - jepsen.util 265 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,105] jepsen worker 85 - jepsen.util 265 :invoke :cas [28 [48 25]] INFO [2019-12-17 12:27:54,121] jepsen worker 8 - jepsen.util 278 :info :cas [2 [6 45]] :RequestCanceledException INFO [2019-12-17 12:27:54,123] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [38 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,123] jepsen worker 8 - jepsen.util 368 :invoke :write [2 10] INFO [2019-12-17 12:27:54,123] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [44 12]] INFO [2019-12-17 12:27:54,134] jepsen worker 86 - jepsen.util 86 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,134] jepsen worker 33 - jepsen.util 303 :info :write [11 22] :RequestCanceledException INFO [2019-12-17 12:27:54,135] jepsen worker 86 - jepsen.util 86 :invoke :write [28 33] INFO [2019-12-17 12:27:54,136] jepsen worker 33 - jepsen.util 393 :invoke :cas [11 [16 41]] INFO [2019-12-17 12:27:54,151] jepsen worker 63 - jepsen.util 153 :ok :write [21 19] INFO [2019-12-17 12:27:54,151] jepsen worker 63 - jepsen.util 153 :invoke :write [21 14] INFO [2019-12-17 12:27:54,151] jepsen worker 28 - jepsen.util 208 :info :write [9 28] :RequestCanceledException INFO [2019-12-17 12:27:54,153] jepsen worker 28 - jepsen.util 298 :invoke :write [9 44] INFO [2019-12-17 12:27:54,173] jepsen worker 5 - jepsen.util 5 :fail :cas [1 [30 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,173] jepsen worker 5 - jepsen.util 5 :invoke :write [1 2] INFO [2019-12-17 12:27:54,205] jepsen worker 58 - jepsen.util 148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,205] jepsen worker 3 - jepsen.util 183 :info :write [1 42] :RequestCanceledException INFO [2019-12-17 12:27:54,205] jepsen worker 58 - jepsen.util 148 :invoke :read [19 nil] INFO [2019-12-17 12:27:54,206] jepsen worker 3 - jepsen.util 273 :invoke :read [1 nil] INFO [2019-12-17 12:27:54,226] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [44 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,226] jepsen worker 7 - jepsen.util 457 :invoke :read [2 nil] INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1612,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1613,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:54,231] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1614,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:54,234] jepsen worker 39 - jepsen.util 219 :info :write [13 37] :RequestCanceledException INFO [2019-12-17 12:27:54,236] jepsen worker 39 - jepsen.util 309 :invoke :cas [13 [26 33]] INFO [2019-12-17 12:27:54,253] jepsen worker 63 - jepsen.util 153 :ok :write [21 14] INFO [2019-12-17 12:27:54,253] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,267] jepsen worker 87 - jepsen.util 177 :fail :cas [29 [11 47]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,267] jepsen worker 87 - jepsen.util 177 :invoke :write [29 4] INFO [2019-12-17 12:27:54,282] jepsen worker 63 - jepsen.util 153 :ok :read [21 14] INFO [2019-12-17 12:27:54,282] jepsen worker 63 - jepsen.util 153 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:54,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:54,302] jepsen worker 63 - jepsen.util 153 :ok :read [21 14] INFO [2019-12-17 12:27:54,302] jepsen worker 63 - jepsen.util 153 :invoke :write [21 49] INFO [2019-12-17 12:27:54,302] jepsen worker 57 - jepsen.util 327 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,303] jepsen worker 57 - jepsen.util 327 :invoke :read [19 nil] INFO [2019-12-17 12:27:54,305] jepsen worker 32 - jepsen.util 212 :info :write [10 47] :SyncWriteAmbiguous INFO [2019-12-17 12:27:54,306] jepsen worker 32 - jepsen.util 302 :invoke :cas [10 [32 6]] INFO [2019-12-17 12:27:54,311] jepsen worker 21 - jepsen.util 291 :fail :cas [7 [33 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,311] jepsen worker 21 - jepsen.util 291 :invoke :read [7 nil] INFO [2019-12-17 12:27:54,318] jepsen worker 23 - jepsen.util 113 :fail :cas [7 [12 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,318] jepsen worker 23 - jepsen.util 113 :invoke :write [7 2] INFO [2019-12-17 12:27:54,320] jepsen worker 42 - jepsen.util 402 :info :write [14 40] :RequestCanceledException INFO [2019-12-17 12:27:54,321] jepsen worker 47 - jepsen.util 317 :info :write [15 32] :RequestCanceledException INFO [2019-12-17 12:27:54,322] jepsen worker 42 - jepsen.util 492 :invoke :write [14 27] INFO [2019-12-17 12:27:54,322] jepsen worker 47 - jepsen.util 407 :invoke :cas [15 [16 9]] INFO [2019-12-17 12:27:54,341] jepsen worker 72 - jepsen.util 162 :ok :read [24 35] INFO [2019-12-17 12:27:54,341] jepsen worker 72 - jepsen.util 162 :invoke :write [24 21] INFO [2019-12-17 12:27:54,368] jepsen worker 73 - jepsen.util 253 :info :write [24 39] :SyncWriteAmbiguous INFO [2019-12-17 12:27:54,369] jepsen worker 73 - jepsen.util 343 :invoke :cas [24 [7 15]] INFO [2019-12-17 12:27:54,370] jepsen worker 9 - jepsen.util 189 :ok :read [3 14] INFO [2019-12-17 12:27:54,370] jepsen worker 9 - jepsen.util 189 :invoke :write [3 43] INFO [2019-12-17 12:27:54,389] jepsen worker 7 - jepsen.util 457 :ok :read [2 15] INFO [2019-12-17 12:27:54,389] jepsen worker 7 - jepsen.util 457 :invoke :read [2 nil] INFO [2019-12-17 12:27:54,397] jepsen worker 71 - jepsen.util 341 :fail :cas [23 [2 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,398] jepsen worker 71 - jepsen.util 341 :invoke :read [23 nil] INFO [2019-12-17 12:27:54,399] jepsen worker 20 - jepsen.util 380 :ok :read [6 45] INFO [2019-12-17 12:27:54,399] jepsen worker 20 - jepsen.util 380 :invoke :read [6 nil] INFO [2019-12-17 12:27:54,405] jepsen worker 18 - jepsen.util 288 :info :write [6 12] :RequestCanceledException INFO [2019-12-17 12:27:54,406] jepsen worker 18 - jepsen.util 378 :invoke :write [6 10] INFO [2019-12-17 12:27:54,421] jepsen worker 47 - jepsen.util 407 :fail :cas [15 [16 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,421] jepsen worker 10 - jepsen.util 190 :info :write [3 12] :RequestCanceledException INFO [2019-12-17 12:27:54,421] jepsen worker 27 - jepsen.util 27 :info :cas [9 [25 37]] :RequestCanceledException INFO [2019-12-17 12:27:54,421] jepsen worker 65 - jepsen.util 335 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,421] jepsen worker 47 - jepsen.util 407 :invoke :write [15 29] INFO [2019-12-17 12:27:54,421] jepsen worker 65 - jepsen.util 335 :invoke :cas [21 [38 14]] INFO [2019-12-17 12:27:54,422] jepsen worker 10 - jepsen.util 280 :invoke :cas [3 [25 26]] INFO [2019-12-17 12:27:54,422] jepsen worker 27 - jepsen.util 117 :invoke :write [9 18] INFO [2019-12-17 12:27:54,424] jepsen worker 63 - jepsen.util 153 :ok :write [21 49] INFO [2019-12-17 12:27:54,424] jepsen worker 63 - jepsen.util 153 :invoke :cas [21 [38 7]] INFO [2019-12-17 12:27:54,434] jepsen worker 11 - jepsen.util 281 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,434] jepsen worker 11 - jepsen.util 281 :invoke :cas [3 [30 31]] INFO [2019-12-17 12:27:54,442] jepsen worker 76 - jepsen.util 256 :ok :read [25 2] INFO [2019-12-17 12:27:54,443] jepsen worker 76 - jepsen.util 256 :invoke :read [25 nil] INFO [2019-12-17 12:27:54,449] jepsen worker 20 - jepsen.util 380 :ok :read [6 45] INFO [2019-12-17 12:27:54,450] jepsen worker 20 - jepsen.util 380 :invoke :read [6 nil] INFO [2019-12-17 12:27:54,452] jepsen worker 88 - jepsen.util 268 :info :write [29 40] :RequestCanceledException INFO [2019-12-17 12:27:54,453] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [37 46]] INFO [2019-12-17 12:27:54,465] jepsen worker 13 - jepsen.util 283 :fail :cas [4 [11 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,465] jepsen worker 13 - jepsen.util 283 :invoke :read [4 nil] INFO [2019-12-17 12:27:54,502] jepsen worker 62 - jepsen.util 332 :fail :cas [20 [48 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,502] jepsen worker 62 - jepsen.util 332 :invoke :write [20 0] INFO [2019-12-17 12:27:54,510] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:54,510] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1649,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:54,521] jepsen worker 46 - jepsen.util 316 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,521] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:54,523] jepsen worker 26 - jepsen.util 476 :ok :read [8 14] INFO [2019-12-17 12:27:54,524] jepsen worker 26 - jepsen.util 476 :invoke :cas [8 [24 1]] INFO [2019-12-17 12:27:54,526] jepsen worker 13 - jepsen.util 283 :ok :read [4 4] INFO [2019-12-17 12:27:54,526] jepsen worker 13 - jepsen.util 283 :invoke :write [4 46] INFO [2019-12-17 12:27:54,534] jepsen worker 64 - jepsen.util 334 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,535] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [13 49]] INFO [2019-12-17 12:27:54,551] jepsen worker 61 - jepsen.util 331 :info :cas [20 [7 13]] :RequestCanceledException INFO [2019-12-17 12:27:54,553] jepsen worker 61 - jepsen.util 421 :invoke :read [20 nil] INFO [2019-12-17 12:27:54,553] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [13 49]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,554] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [20 16]] INFO [2019-12-17 12:27:54,566] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.02734375}, :ns_1@172.28.128.126 {:progress 0.02517985611510787}, :ns_1@172.28.128.127 {:progress 0.0622568093385214}, :ns_1@172.28.128.128 {:progress 0.05283018867924527}} INFO [2019-12-17 12:27:54,567] jepsen nemesis - couchbase.util Rebalance check count: 6 INFO [2019-12-17 12:27:54,567] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:54,571] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [20 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,571] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,588] jepsen worker 38 - jepsen.util 218 :fail :read [12 nil] :RequestTimeoutException INFO [2019-12-17 12:27:54,588] jepsen worker 38 - jepsen.util 218 :invoke :read [12 nil] INFO [2019-12-17 12:27:54,592] jepsen worker 64 - jepsen.util 334 :ok :read [21 49] INFO [2019-12-17 12:27:54,592] jepsen worker 64 - jepsen.util 334 :invoke :write [21 7] INFO [2019-12-17 12:27:54,597] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [47 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,598] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [47 16]] INFO [2019-12-17 12:27:54,610] jepsen worker 0 - jepsen.util 270 :info :write [0 9] :SyncWriteAmbiguous INFO [2019-12-17 12:27:54,611] jepsen worker 0 - jepsen.util 360 :invoke :write [0 9] INFO [2019-12-17 12:27:54,621] jepsen worker 36 - jepsen.util 306 :info :write [12 5] :RequestCanceledException INFO [2019-12-17 12:27:54,622] jepsen worker 36 - jepsen.util 396 :invoke :read [12 nil] INFO [2019-12-17 12:27:54,625] jepsen worker 59 - jepsen.util 239 :ok :read [19 8] INFO [2019-12-17 12:27:54,625] jepsen worker 59 - jepsen.util 239 :invoke :read [19 nil] INFO [2019-12-17 12:27:54,634] jepsen worker 80 - jepsen.util 440 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:27:54,634] jepsen worker 75 - jepsen.util 255 :info :write [25 13] :RequestCanceledException INFO [2019-12-17 12:27:54,635] jepsen worker 49 - jepsen.util 319 :fail :cas [16 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,635] jepsen worker 49 - jepsen.util 319 :invoke :read [16 nil] INFO [2019-12-17 12:27:54,636] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:54,636] jepsen worker 80 - jepsen.util 530 :invoke :write [26 42] INFO [2019-12-17 12:27:54,652] jepsen worker 64 - jepsen.util 334 :ok :write [21 7] INFO [2019-12-17 12:27:54,652] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [22 17]] INFO [2019-12-17 12:27:54,653] jepsen worker 12 - jepsen.util 192 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:27:54,653] jepsen worker 37 - jepsen.util 127 :info :write [12 36] :RequestCanceledException INFO [2019-12-17 12:27:54,654] jepsen worker 12 - jepsen.util 282 :invoke :read [4 nil] INFO [2019-12-17 12:27:54,654] jepsen worker 37 - jepsen.util 217 :invoke :cas [12 [37 16]] INFO [2019-12-17 12:27:54,674] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [22 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,674] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [26 38]] INFO [2019-12-17 12:27:54,703] jepsen worker 76 - jepsen.util 256 :ok :read [25 2] INFO [2019-12-17 12:27:54,703] jepsen worker 76 - jepsen.util 256 :invoke :write [25 36] INFO [2019-12-17 12:27:54,708] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [26 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,708] jepsen worker 64 - jepsen.util 334 :invoke :write [21 31] INFO [2019-12-17 12:27:54,738] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:54,738] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:54,779] jepsen worker 64 - jepsen.util 334 :ok :write [21 31] INFO [2019-12-17 12:27:54,779] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,787] jepsen worker 52 - jepsen.util 232 :info :write [17 23] :RequestCanceledException INFO [2019-12-17 12:27:54,789] jepsen worker 52 - jepsen.util 322 :invoke :read [17 nil] INFO [2019-12-17 12:27:54,796] jepsen worker 64 - jepsen.util 334 :ok :read [21 31] INFO [2019-12-17 12:27:54,796] jepsen worker 64 - jepsen.util 334 :invoke :write [21 38] INFO [2019-12-17 12:27:54,798] jepsen worker 69 - jepsen.util 339 :fail :cas [23 [12 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,798] jepsen worker 69 - jepsen.util 339 :invoke :write [23 1] INFO [2019-12-17 12:27:54,813] jepsen worker 33 - jepsen.util 393 :fail :cas [11 [16 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,813] jepsen worker 33 - jepsen.util 393 :invoke :read [11 nil] INFO [2019-12-17 12:27:54,821] jepsen worker 67 - jepsen.util 337 :info :write [22 34] :RequestCanceledException INFO [2019-12-17 12:27:54,822] jepsen worker 67 - jepsen.util 427 :invoke :cas [22 [21 24]] INFO [2019-12-17 12:27:54,864] jepsen worker 64 - jepsen.util 334 :ok :write [21 38] INFO [2019-12-17 12:27:54,864] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:54,883] jepsen worker 64 - jepsen.util 334 :ok :read [21 38] INFO [2019-12-17 12:27:54,884] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [18 39]] INFO [2019-12-17 12:27:54,900] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [18 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,900] jepsen worker 64 - jepsen.util 334 :invoke :write [21 47] INFO [2019-12-17 12:27:54,905] jepsen worker 43 - jepsen.util 403 :info :write [14 1] :RequestCanceledException INFO [2019-12-17 12:27:54,906] jepsen worker 43 - jepsen.util 493 :invoke :write [14 48] INFO [2019-12-17 12:27:54,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:54,955] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1701,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:54,956] jepsen worker 64 - jepsen.util 334 :ok :write [21 47] INFO [2019-12-17 12:27:54,956] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [26 10]] INFO [2019-12-17 12:27:54,985] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:54,989] jepsen worker 64 - jepsen.util 334 :invoke :write [21 13] INFO [2019-12-17 12:27:55,005] jepsen worker 60 - jepsen.util 420 :info :write [20 22] :RequestCanceledException INFO [2019-12-17 12:27:55,006] jepsen worker 60 - jepsen.util 510 :invoke :read [20 nil] INFO [2019-12-17 12:27:55,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":1416,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:55,063] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:55,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1711,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:55,064] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:55,064] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:55,086] jepsen worker 3 - jepsen.util 273 :ok :read [1 23] INFO [2019-12-17 12:27:55,086] jepsen worker 3 - jepsen.util 273 :invoke :write [1 34] INFO [2019-12-17 12:27:55,087] jepsen worker 64 - jepsen.util 334 :ok :write [21 13] INFO [2019-12-17 12:27:55,088] jepsen worker 64 - jepsen.util 334 :invoke :write [21 45] INFO [2019-12-17 12:27:55,095] jepsen worker 28 - jepsen.util 298 :ok :write [9 44] INFO [2019-12-17 12:27:55,095] jepsen worker 28 - jepsen.util 298 :invoke :write [9 45] INFO [2019-12-17 12:27:55,111] jepsen worker 7 - jepsen.util 457 :ok :read [2 15] INFO [2019-12-17 12:27:55,111] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [18 15]] INFO [2019-12-17 12:27:55,121] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [47 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,121] jepsen worker 29 - jepsen.util 209 :info :write [9 39] :RequestCanceledException INFO [2019-12-17 12:27:55,121] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:55,122] jepsen worker 29 - jepsen.util 299 :invoke :cas [9 [44 48]] INFO [2019-12-17 12:27:55,151] jepsen worker 19 - jepsen.util 199 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:27:55,151] jepsen worker 30 - jepsen.util 120 :info :cas [10 [28 29]] :RequestCanceledException INFO [2019-12-17 12:27:55,152] jepsen worker 19 - jepsen.util 289 :invoke :read [6 nil] INFO [2019-12-17 12:27:55,152] jepsen worker 30 - jepsen.util 210 :invoke :read [10 nil] INFO [2019-12-17 12:27:55,154] jepsen worker 37 - jepsen.util 217 :fail :cas [12 [37 16]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,154] jepsen worker 37 - jepsen.util 217 :invoke :write [12 34] INFO [2019-12-17 12:27:55,160] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [37 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,160] jepsen worker 88 - jepsen.util 358 :invoke :read [29 nil] INFO [2019-12-17 12:27:55,177] jepsen worker 64 - jepsen.util 334 :ok :write [21 45] INFO [2019-12-17 12:27:55,178] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [23 2]] INFO [2019-12-17 12:27:55,188] jepsen worker 50 - jepsen.util 320 :info :write [16 0] :RequestCanceledException INFO [2019-12-17 12:27:55,188] jepsen worker 28 - jepsen.util 298 :ok :write [9 45] INFO [2019-12-17 12:27:55,188] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:55,189] jepsen worker 50 - jepsen.util 410 :invoke :write [16 17] INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1727,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:55,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:55,203] jepsen worker 74 - jepsen.util 344 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,203] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:55,206] jepsen worker 30 - jepsen.util 210 :ok :read [10 23] INFO [2019-12-17 12:27:55,206] jepsen worker 30 - jepsen.util 210 :invoke :read [10 nil] INFO [2019-12-17 12:27:55,210] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:55,210] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:55,223] jepsen worker 28 - jepsen.util 298 :ok :read [9 45] INFO [2019-12-17 12:27:55,223] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [11 36]] INFO [2019-12-17 12:27:55,231] jepsen worker 19 - jepsen.util 289 :ok :read [6 45] INFO [2019-12-17 12:27:55,231] jepsen worker 19 - jepsen.util 289 :invoke :read [6 nil] INFO [2019-12-17 12:27:55,237] jepsen worker 30 - jepsen.util 210 :ok :read [10 23] INFO [2019-12-17 12:27:55,237] jepsen worker 30 - jepsen.util 210 :invoke :write [10 21] INFO [2019-12-17 12:27:55,255] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [11 36]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,256] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:55,270] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [23 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,270] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [1 44]] INFO [2019-12-17 12:27:55,272] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1735,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1738,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:55,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1739,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:55,278] jepsen worker 19 - jepsen.util 289 :ok :read [6 45] INFO [2019-12-17 12:27:55,278] jepsen worker 19 - jepsen.util 289 :invoke :read [6 nil] INFO [2019-12-17 12:27:55,291] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [1 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,291] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [18 12]] INFO [2019-12-17 12:27:55,308] jepsen worker 19 - jepsen.util 289 :ok :read [6 45] INFO [2019-12-17 12:27:55,308] jepsen worker 19 - jepsen.util 289 :invoke :cas [6 [14 30]] INFO [2019-12-17 12:27:55,310] jepsen worker 12 - jepsen.util 282 :ok :read [4 4] INFO [2019-12-17 12:27:55,310] jepsen worker 12 - jepsen.util 282 :invoke :cas [4 [12 28]] INFO [2019-12-17 12:27:55,323] jepsen worker 61 - jepsen.util 421 :ok :read [20 7] INFO [2019-12-17 12:27:55,324] jepsen worker 61 - jepsen.util 421 :invoke :write [20 38] INFO [2019-12-17 12:27:55,329] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [18 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,329] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:55,348] jepsen worker 19 - jepsen.util 289 :fail :cas [6 [14 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,348] jepsen worker 19 - jepsen.util 289 :invoke :write [6 15] INFO [2019-12-17 12:27:55,352] jepsen worker 56 - jepsen.util 236 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:27:55,353] jepsen worker 56 - jepsen.util 326 :invoke :write [18 26] INFO [2019-12-17 12:27:55,388] jepsen worker 54 - jepsen.util 414 :info :write [18 15] :RequestCanceledException INFO [2019-12-17 12:27:55,389] jepsen worker 54 - jepsen.util 504 :invoke :cas [18 [16 33]] INFO [2019-12-17 12:27:55,396] jepsen worker 64 - jepsen.util 334 :ok :read [21 45] INFO [2019-12-17 12:27:55,396] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [2 18]] INFO [2019-12-17 12:27:55,420] jepsen worker 14 - jepsen.util 284 :info :write [4 19] :RequestCanceledException INFO [2019-12-17 12:27:55,420] jepsen worker 17 - jepsen.util 377 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,421] jepsen worker 17 - jepsen.util 377 :invoke :cas [5 [40 17]] INFO [2019-12-17 12:27:55,422] jepsen worker 14 - jepsen.util 374 :invoke :write [4 18] INFO [2019-12-17 12:27:55,425] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [2 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,425] jepsen worker 64 - jepsen.util 334 :invoke :write [21 28] INFO [2019-12-17 12:27:55,450] jepsen worker 24 - jepsen.util 204 :info :write [8 22] :RequestCanceledException INFO [2019-12-17 12:27:55,452] jepsen worker 24 - jepsen.util 294 :invoke :cas [8 [26 2]] INFO [2019-12-17 12:27:55,465] jepsen worker 52 - jepsen.util 322 :ok :read [17 0] INFO [2019-12-17 12:27:55,465] jepsen worker 52 - jepsen.util 322 :invoke :read [17 nil] INFO [2019-12-17 12:27:55,475] jepsen worker 17 - jepsen.util 377 :fail :cas [5 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,476] jepsen worker 17 - jepsen.util 377 :invoke :cas [5 [34 17]] INFO [2019-12-17 12:27:55,488] jepsen worker 25 - jepsen.util 385 :info :write [8 24] :RequestCanceledException INFO [2019-12-17 12:27:55,489] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,490] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:55,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1760,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:55,491] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:55,491] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1763,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:55,492] jepsen worker 17 - jepsen.util 377 :fail :cas [5 [34 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,492] jepsen worker 17 - jepsen.util 377 :invoke :write [5 22] INFO [2019-12-17 12:27:55,494] jepsen worker 64 - jepsen.util 334 :ok :write [21 28] INFO [2019-12-17 12:27:55,494] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [40 17]] INFO [2019-12-17 12:27:55,520] jepsen worker 15 - jepsen.util 375 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,520] jepsen worker 84 - jepsen.util 84 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,520] jepsen worker 15 - jepsen.util 375 :invoke :read [5 nil] INFO [2019-12-17 12:27:55,520] jepsen worker 84 - jepsen.util 84 :invoke :write [28 13] INFO [2019-12-17 12:27:55,530] jepsen worker 26 - jepsen.util 476 :fail :cas [8 [24 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,530] jepsen worker 26 - jepsen.util 476 :invoke :cas [8 [12 34]] INFO [2019-12-17 12:27:55,533] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [40 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,534] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [23 8]] INFO [2019-12-17 12:27:55,541] jepsen worker 21 - jepsen.util 291 :ok :read [7 20] INFO [2019-12-17 12:27:55,541] jepsen worker 21 - jepsen.util 291 :invoke :write [7 48] INFO [2019-12-17 12:27:55,553] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [23 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,553] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [3 4]] INFO [2019-12-17 12:27:55,571] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [3 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,572] jepsen worker 88 - jepsen.util 358 :ok :read [29 5] INFO [2019-12-17 12:27:55,572] jepsen worker 64 - jepsen.util 334 :invoke :write [21 37] INFO [2019-12-17 12:27:55,572] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [32 34]] INFO [2019-12-17 12:27:55,573] jepsen worker 71 - jepsen.util 341 :ok :read [23 29] INFO [2019-12-17 12:27:55,573] jepsen worker 71 - jepsen.util 341 :invoke :read [23 nil] INFO [2019-12-17 12:27:55,588] jepsen worker 82 - jepsen.util 442 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,588] jepsen worker 79 - jepsen.util 259 :info :write [26 9] :RequestCanceledException INFO [2019-12-17 12:27:55,588] jepsen worker 82 - jepsen.util 442 :invoke :write [27 36] INFO [2019-12-17 12:27:55,589] jepsen worker 79 - jepsen.util 349 :invoke :write [26 13] INFO [2019-12-17 12:27:55,593] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.03515625}, :ns_1@172.28.128.126 {:progress 0.03237410071942448}, :ns_1@172.28.128.127 {:progress 0.0622568093385214}, :ns_1@172.28.128.128 {:progress 0.06037735849056602}} INFO [2019-12-17 12:27:55,593] jepsen nemesis - couchbase.util Rebalance check count: 7 INFO [2019-12-17 12:27:55,593] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:55,615] jepsen worker 64 - jepsen.util 334 :ok :write [21 37] INFO [2019-12-17 12:27:55,615] jepsen worker 64 - jepsen.util 334 :invoke :write [21 11] INFO [2019-12-17 12:27:55,620] jepsen worker 22 - jepsen.util 292 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:27:55,621] jepsen worker 34 - jepsen.util 304 :info :write [11 44] :RequestCanceledException INFO [2019-12-17 12:27:55,621] jepsen worker 33 - jepsen.util 393 :ok :read [11 7] INFO [2019-12-17 12:27:55,622] jepsen worker 22 - jepsen.util 382 :invoke :read [7 nil] INFO [2019-12-17 12:27:55,622] jepsen worker 33 - jepsen.util 393 :invoke :cas [11 [46 5]] INFO [2019-12-17 12:27:55,623] jepsen worker 34 - jepsen.util 394 :invoke :cas [11 [2 6]] INFO [2019-12-17 12:27:55,635] jepsen worker 41 - jepsen.util 311 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,635] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:27:55,684] jepsen worker 64 - jepsen.util 334 :ok :write [21 11] INFO [2019-12-17 12:27:55,684] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [29 35]] INFO [2019-12-17 12:27:55,694] jepsen worker 52 - jepsen.util 322 :ok :read [17 0] INFO [2019-12-17 12:27:55,694] jepsen worker 52 - jepsen.util 322 :invoke :read [17 nil] INFO [2019-12-17 12:27:55,705] jepsen worker 16 - jepsen.util 196 :info :write [5 40] :RequestCanceledException INFO [2019-12-17 12:27:55,705] jepsen worker 51 - jepsen.util 141 :info :write [17 21] :RequestCanceledException INFO [2019-12-17 12:27:55,706] jepsen worker 51 - jepsen.util 231 :invoke :read [17 nil] INFO [2019-12-17 12:27:55,706] jepsen worker 16 - jepsen.util 286 :invoke :cas [5 [22 24]] INFO [2019-12-17 12:27:55,714] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [29 35]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,714] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:55,721] jepsen worker 68 - jepsen.util 248 :info :write [22 25] :RequestCanceledException INFO [2019-12-17 12:27:55,721] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:55,722] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1794,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:55,722] jepsen worker 68 - jepsen.util 338 :invoke :read [22 nil] INFO [2019-12-17 12:27:55,733] jepsen worker 38 - jepsen.util 218 :ok :read [12 12] INFO [2019-12-17 12:27:55,733] jepsen worker 38 - jepsen.util 218 :invoke :cas [12 [37 18]] INFO [2019-12-17 12:27:55,743] jepsen worker 64 - jepsen.util 334 :ok :read [21 11] INFO [2019-12-17 12:27:55,743] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [23 11]] INFO [2019-12-17 12:27:55,763] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [23 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,763] jepsen worker 64 - jepsen.util 334 :invoke :write [21 4] INFO [2019-12-17 12:27:55,772] jepsen worker 26 - jepsen.util 476 :fail :cas [8 [12 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,772] jepsen worker 26 - jepsen.util 476 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,781] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:55,781] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:55,786] jepsen worker 12 - jepsen.util 282 :fail :cas [4 [12 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,786] jepsen worker 12 - jepsen.util 282 :invoke :cas [4 [26 17]] INFO [2019-12-17 12:27:55,787] jepsen worker 66 - jepsen.util 246 :info :write [22 24] :RequestCanceledException INFO [2019-12-17 12:27:55,787] jepsen worker 66 - jepsen.util 336 :invoke :cas [22 [4 34]] INFO [2019-12-17 12:27:55,806] jepsen worker 52 - jepsen.util 322 :ok :read [17 0] INFO [2019-12-17 12:27:55,806] jepsen worker 52 - jepsen.util 322 :invoke :cas [17 [15 32]] INFO [2019-12-17 12:27:55,816] jepsen worker 32 - jepsen.util 302 :fail :cas [10 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,817] jepsen worker 32 - jepsen.util 302 :invoke :write [10 33] INFO [2019-12-17 12:27:55,821] jepsen worker 48 - jepsen.util 408 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,822] jepsen worker 48 - jepsen.util 408 :invoke :read [16 nil] INFO [2019-12-17 12:27:55,831] jepsen worker 45 - jepsen.util 405 :info :write [15 16] :SyncWriteAmbiguous INFO [2019-12-17 12:27:55,833] jepsen worker 45 - jepsen.util 495 :invoke :write [15 40] INFO [2019-12-17 12:27:55,852] jepsen worker 89 - jepsen.util 269 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:27:55,852] jepsen worker 70 - jepsen.util 430 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:27:55,853] jepsen worker 89 - jepsen.util 359 :invoke :read [29 nil] INFO [2019-12-17 12:27:55,853] jepsen worker 70 - jepsen.util 520 :invoke :cas [23 [36 33]] INFO [2019-12-17 12:27:55,858] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:55,858] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,859] jepsen worker 65 - jepsen.util 335 :fail :cas [21 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,859] jepsen worker 65 - jepsen.util 335 :invoke :cas [21 [4 25]] INFO [2019-12-17 12:27:55,872] jepsen worker 54 - jepsen.util 504 :fail :cas [18 [16 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,872] jepsen worker 54 - jepsen.util 504 :invoke :read [18 nil] INFO [2019-12-17 12:27:55,883] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [32 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,883] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [18 14]] INFO [2019-12-17 12:27:55,889] jepsen worker 46 - jepsen.util 316 :ok :read [15 35] INFO [2019-12-17 12:27:55,889] jepsen worker 46 - jepsen.util 316 :invoke :cas [15 [18 10]] INFO [2019-12-17 12:27:55,894] jepsen worker 26 - jepsen.util 476 :ok :read [8 14] INFO [2019-12-17 12:27:55,894] jepsen worker 26 - jepsen.util 476 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,904] jepsen worker 40 - jepsen.util 400 :info :write [13 11] :RequestCanceledException INFO [2019-12-17 12:27:55,906] jepsen worker 29 - jepsen.util 299 :fail :cas [9 [44 48]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,906] jepsen worker 40 - jepsen.util 490 :invoke :read [13 nil] INFO [2019-12-17 12:27:55,906] jepsen worker 29 - jepsen.util 299 :invoke :read [9 nil] INFO [2019-12-17 12:27:55,907] jepsen worker 52 - jepsen.util 322 :fail :cas [17 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,907] jepsen worker 52 - jepsen.util 322 :invoke :write [17 19] INFO [2019-12-17 12:27:55,916] jepsen worker 65 - jepsen.util 335 :fail :cas [21 [4 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,917] jepsen worker 65 - jepsen.util 335 :invoke :write [21 42] INFO [2019-12-17 12:27:55,920] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [18 15]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,920] jepsen worker 7 - jepsen.util 457 :invoke :read [2 nil] INFO [2019-12-17 12:27:55,920] jepsen worker 77 - jepsen.util 257 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:27:55,921] jepsen worker 6 - jepsen.util 276 :info :write [2 47] :RequestCanceledException INFO [2019-12-17 12:27:55,921] jepsen worker 77 - jepsen.util 347 :invoke :write [25 29] INFO [2019-12-17 12:27:55,922] jepsen worker 6 - jepsen.util 366 :invoke :write [2 15] INFO [2019-12-17 12:27:55,935] jepsen worker 4 - jepsen.util 274 :info :write [1 29] :RequestCanceledException INFO [2019-12-17 12:27:55,937] jepsen worker 4 - jepsen.util 364 :invoke :cas [1 [44 2]] INFO [2019-12-17 12:27:55,937] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:55,937] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:55,939] jepsen worker 46 - jepsen.util 316 :fail :cas [15 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:55,940] jepsen worker 46 - jepsen.util 316 :invoke :read [15 nil] INFO [2019-12-17 12:27:55,949] jepsen worker 35 - jepsen.util 125 :info :write [11 34] :RequestCanceledException INFO [2019-12-17 12:27:55,950] jepsen worker 35 - jepsen.util 215 :invoke :read [11 nil] INFO [2019-12-17 12:27:55,957] jepsen worker 60 - jepsen.util 510 :ok :read [20 7] INFO [2019-12-17 12:27:55,957] jepsen worker 60 - jepsen.util 510 :invoke :write [20 24] INFO [2019-12-17 12:27:55,976] jepsen worker 29 - jepsen.util 299 :ok :read [9 45] INFO [2019-12-17 12:27:55,976] jepsen worker 29 - jepsen.util 299 :invoke :write [9 47] INFO [2019-12-17 12:27:55,987] jepsen worker 81 - jepsen.util 351 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:27:55,987] jepsen worker 81 - jepsen.util 351 :invoke :write [27 23] INFO [2019-12-17 12:27:56,005] jepsen worker 2 - jepsen.util 182 :info :write [0 31] :RequestCanceledException INFO [2019-12-17 12:27:56,006] jepsen worker 2 - jepsen.util 272 :invoke :read [0 nil] INFO [2019-12-17 12:27:56,012] jepsen worker 64 - jepsen.util 334 :ok :write [21 4] INFO [2019-12-17 12:27:56,012] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [33 17]] INFO [2019-12-17 12:27:56,014] jepsen worker 27 - jepsen.util 117 :ok :write [9 18] INFO [2019-12-17 12:27:56,014] jepsen worker 27 - jepsen.util 117 :invoke :write [9 42] INFO [2019-12-17 12:27:56,021] jepsen worker 53 - jepsen.util 413 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:27:56,021] jepsen worker 31 - jepsen.util 481 :info :write [10 37] :RequestCanceledException INFO [2019-12-17 12:27:56,023] jepsen worker 53 - jepsen.util 503 :invoke :read [17 nil] INFO [2019-12-17 12:27:56,023] jepsen worker 31 - jepsen.util 571 :invoke :write [10 38] INFO [2019-12-17 12:27:56,031] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:56,031] jepsen worker 75 - jepsen.util 345 :invoke :cas [25 [28 39]] INFO [2019-12-17 12:27:56,035] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,035] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [45 14]] INFO [2019-12-17 12:27:56,047] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:56,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:56,048] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:56,048] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:56,057] jepsen worker 36 - jepsen.util 396 :ok :read [12 12] INFO [2019-12-17 12:27:56,057] jepsen worker 36 - jepsen.util 396 :invoke :write [12 12] INFO [2019-12-17 12:27:56,072] jepsen worker 65 - jepsen.util 335 :ok :write [21 42] INFO [2019-12-17 12:27:56,073] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,077] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [45 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,078] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,087] jepsen worker 1 - jepsen.util 361 :info :cas [0 [17 25]] :RequestCanceledException INFO [2019-12-17 12:27:56,088] jepsen worker 1 - jepsen.util 451 :invoke :cas [0 [35 31]] INFO [2019-12-17 12:27:56,105] jepsen worker 65 - jepsen.util 335 :ok :read [21 42] INFO [2019-12-17 12:27:56,105] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,105] jepsen worker 55 - jepsen.util 415 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:27:56,105] jepsen worker 78 - jepsen.util 348 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:27:56,106] jepsen worker 78 - jepsen.util 438 :invoke :read [26 nil] INFO [2019-12-17 12:27:56,106] jepsen worker 55 - jepsen.util 505 :invoke :write [18 10] INFO [2019-12-17 12:27:56,116] jepsen worker 51 - jepsen.util 231 :ok :read [17 0] INFO [2019-12-17 12:27:56,116] jepsen worker 51 - jepsen.util 231 :invoke :cas [17 [47 32]] INFO [2019-12-17 12:27:56,120] jepsen worker 83 - jepsen.util 443 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,121] jepsen worker 83 - jepsen.util 443 :invoke :cas [27 [30 23]] INFO [2019-12-17 12:27:56,125] jepsen worker 64 - jepsen.util 334 :ok :read [21 42] INFO [2019-12-17 12:27:56,126] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,127] jepsen worker 34 - jepsen.util 394 :fail :cas [11 [2 6]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,127] jepsen worker 34 - jepsen.util 394 :invoke :write [11 3] INFO [2019-12-17 12:27:56,166] jepsen worker 89 - jepsen.util 359 :ok :read [29 5] INFO [2019-12-17 12:27:56,166] jepsen worker 89 - jepsen.util 359 :invoke :write [29 5] INFO [2019-12-17 12:27:56,175] jepsen worker 65 - jepsen.util 335 :ok :read [21 42] INFO [2019-12-17 12:27:56,175] jepsen worker 65 - jepsen.util 335 :invoke :cas [21 [36 39]] INFO [2019-12-17 12:27:56,180] jepsen worker 64 - jepsen.util 334 :ok :read [21 42] INFO [2019-12-17 12:27:56,181] jepsen worker 64 - jepsen.util 334 :invoke :write [21 4] INFO [2019-12-17 12:27:56,182] jepsen worker 33 - jepsen.util 393 :fail :cas [11 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,182] jepsen worker 33 - jepsen.util 393 :invoke :cas [11 [8 40]] INFO [2019-12-17 12:27:56,191] jepsen worker 65 - jepsen.util 335 :fail :cas [21 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,191] jepsen worker 65 - jepsen.util 335 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,198] jepsen worker 85 - jepsen.util 265 :info :cas [28 [48 25]] :RequestCanceledException INFO [2019-12-17 12:27:56,200] jepsen worker 85 - jepsen.util 355 :invoke :write [28 25] INFO [2019-12-17 12:27:56,205] jepsen worker 54 - jepsen.util 504 :ok :read [18 47] INFO [2019-12-17 12:27:56,205] jepsen worker 54 - jepsen.util 504 :invoke :read [18 nil] INFO [2019-12-17 12:27:56,206] jepsen worker 12 - jepsen.util 282 :fail :cas [4 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,206] jepsen worker 12 - jepsen.util 282 :invoke :cas [4 [35 18]] INFO [2019-12-17 12:27:56,221] jepsen worker 8 - jepsen.util 368 :info :write [2 10] :RequestCanceledException INFO [2019-12-17 12:27:56,222] jepsen worker 8 - jepsen.util 458 :invoke :write [2 11] INFO [2019-12-17 12:27:56,225] jepsen worker 22 - jepsen.util 382 :ok :read [7 20] INFO [2019-12-17 12:27:56,226] jepsen worker 22 - jepsen.util 382 :invoke :write [7 9] INFO [2019-12-17 12:27:56,234] jepsen worker 86 - jepsen.util 86 :info :write [28 33] :RequestCanceledException INFO [2019-12-17 12:27:56,234] jepsen worker 65 - jepsen.util 335 :ok :read [21 42] INFO [2019-12-17 12:27:56,235] jepsen worker 70 - jepsen.util 520 :fail :cas [23 [36 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,235] jepsen worker 65 - jepsen.util 335 :invoke :write [21 22] INFO [2019-12-17 12:27:56,235] jepsen worker 70 - jepsen.util 520 :invoke :read [23 nil] INFO [2019-12-17 12:27:56,235] jepsen worker 86 - jepsen.util 176 :invoke :write [28 12] INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1876,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1879,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:56,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:56,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:56,254] jepsen worker 72 - jepsen.util 162 :info :write [24 21] :SyncWriteAmbiguous INFO [2019-12-17 12:27:56,255] jepsen worker 72 - jepsen.util 252 :invoke :cas [24 [13 29]] INFO [2019-12-17 12:27:56,263] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:56,263] jepsen worker 25 - jepsen.util 475 :invoke :cas [8 [40 43]] INFO [2019-12-17 12:27:56,264] jepsen worker 26 - jepsen.util 476 :ok :read [8 14] INFO [2019-12-17 12:27:56,265] jepsen worker 26 - jepsen.util 476 :invoke :cas [8 [20 45]] INFO [2019-12-17 12:27:56,278] jepsen worker 53 - jepsen.util 503 :ok :read [17 0] INFO [2019-12-17 12:27:56,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:56,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1868,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:56,278] jepsen worker 53 - jepsen.util 503 :invoke :cas [17 [21 3]] INFO [2019-12-17 12:27:56,279] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:56,279] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:56,287] jepsen worker 39 - jepsen.util 309 :info :cas [13 [26 33]] :RequestCanceledException INFO [2019-12-17 12:27:56,287] jepsen worker 87 - jepsen.util 177 :info :write [29 4] :RequestCanceledException INFO [2019-12-17 12:27:56,287] jepsen worker 5 - jepsen.util 5 :info :write [1 2] :RequestCanceledException INFO [2019-12-17 12:27:56,288] jepsen worker 87 - jepsen.util 267 :invoke :write [29 29] INFO [2019-12-17 12:27:56,288] jepsen worker 5 - jepsen.util 95 :invoke :write [1 11] INFO [2019-12-17 12:27:56,288] jepsen worker 39 - jepsen.util 399 :invoke :write [13 31] INFO [2019-12-17 12:27:56,303] jepsen worker 58 - jepsen.util 148 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,303] jepsen worker 58 - jepsen.util 148 :invoke :write [19 33] INFO [2019-12-17 12:27:56,329] jepsen worker 72 - jepsen.util 252 :fail :cas [24 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,329] jepsen worker 72 - jepsen.util 252 :invoke :write [24 25] INFO [2019-12-17 12:27:56,334] jepsen worker 23 - jepsen.util 113 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:27:56,336] jepsen worker 23 - jepsen.util 203 :invoke :cas [7 [29 14]] INFO [2019-12-17 12:27:56,339] jepsen worker 7 - jepsen.util 457 :ok :read [2 15] INFO [2019-12-17 12:27:56,340] jepsen worker 7 - jepsen.util 457 :invoke :cas [2 [22 38]] INFO [2019-12-17 12:27:56,369] jepsen worker 64 - jepsen.util 334 :ok :write [21 4] INFO [2019-12-17 12:27:56,369] jepsen worker 64 - jepsen.util 334 :invoke :write [21 16] INFO [2019-12-17 12:27:56,378] jepsen worker 26 - jepsen.util 476 :fail :cas [8 [20 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,378] jepsen worker 26 - jepsen.util 476 :invoke :write [8 0] INFO [2019-12-17 12:27:56,387] jepsen worker 9 - jepsen.util 189 :info :write [3 43] :RequestCanceledException INFO [2019-12-17 12:27:56,388] jepsen worker 9 - jepsen.util 279 :invoke :cas [3 [32 42]] INFO [2019-12-17 12:27:56,402] jepsen worker 53 - jepsen.util 503 :fail :cas [17 [21 3]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,402] jepsen worker 53 - jepsen.util 503 :invoke :read [17 nil] INFO [2019-12-17 12:27:56,404] jepsen worker 57 - jepsen.util 327 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,404] jepsen worker 57 - jepsen.util 327 :invoke :write [19 18] INFO [2019-12-17 12:27:56,420] jepsen worker 18 - jepsen.util 378 :info :write [6 10] :RequestCanceledException INFO [2019-12-17 12:27:56,421] jepsen worker 42 - jepsen.util 492 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:27:56,422] jepsen worker 18 - jepsen.util 468 :invoke :read [6 nil] INFO [2019-12-17 12:27:56,422] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [18 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,422] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [33 12]] INFO [2019-12-17 12:27:56,422] jepsen worker 42 - jepsen.util 582 :invoke :cas [14 [29 14]] INFO [2019-12-17 12:27:56,435] jepsen worker 11 - jepsen.util 281 :info :cas [3 [30 31]] :RequestCanceledException INFO [2019-12-17 12:27:56,436] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,451] jepsen worker 64 - jepsen.util 334 :ok :write [21 16] INFO [2019-12-17 12:27:56,452] jepsen worker 73 - jepsen.util 343 :info :cas [24 [7 15]] :RequestCanceledException INFO [2019-12-17 12:27:56,452] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [41 23]] INFO [2019-12-17 12:27:56,453] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:56,453] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1910,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:56,453] jepsen worker 73 - jepsen.util 433 :invoke :read [24 nil] INFO [2019-12-17 12:27:56,459] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,460] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [19 46]] INFO [2019-12-17 12:27:56,487] jepsen worker 10 - jepsen.util 280 :info :cas [3 [25 26]] :RequestCanceledException INFO [2019-12-17 12:27:56,488] jepsen worker 10 - jepsen.util 370 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,491] jepsen worker 0 - jepsen.util 360 :info :write [0 9] :SyncWriteAmbiguous INFO [2019-12-17 12:27:56,492] jepsen worker 0 - jepsen.util 450 :invoke :cas [0 [29 25]] INFO [2019-12-17 12:27:56,495] jepsen worker 75 - jepsen.util 345 :fail :cas [25 [28 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,496] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:56,496] jepsen worker 73 - jepsen.util 433 :ok :read [24 35] INFO [2019-12-17 12:27:56,496] jepsen worker 73 - jepsen.util 433 :invoke :cas [24 [35 31]] INFO [2019-12-17 12:27:56,501] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [19 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,501] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [41 23]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,501] jepsen worker 64 - jepsen.util 334 :invoke :write [21 10] INFO [2019-12-17 12:27:56,501] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [8 37]] INFO [2019-12-17 12:27:56,520] jepsen worker 63 - jepsen.util 153 :info :cas [21 [38 7]] :RequestCanceledException INFO [2019-12-17 12:27:56,522] jepsen worker 63 - jepsen.util 243 :invoke :write [21 19] INFO [2019-12-17 12:27:56,535] jepsen worker 47 - jepsen.util 407 :info :write [15 29] :RequestCanceledException INFO [2019-12-17 12:27:56,535] jepsen worker 20 - jepsen.util 380 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:27:56,535] jepsen worker 62 - jepsen.util 332 :info :write [20 0] :RequestCanceledException INFO [2019-12-17 12:27:56,535] jepsen worker 20 - jepsen.util 380 :invoke :write [6 1] INFO [2019-12-17 12:27:56,536] jepsen worker 47 - jepsen.util 497 :invoke :write [15 37] INFO [2019-12-17 12:27:56,536] jepsen worker 62 - jepsen.util 422 :invoke :write [20 36] INFO [2019-12-17 12:27:56,566] jepsen worker 59 - jepsen.util 239 :ok :read [19 8] INFO [2019-12-17 12:27:56,567] jepsen worker 59 - jepsen.util 239 :invoke :write [19 9] INFO [2019-12-17 12:27:56,586] jepsen worker 24 - jepsen.util 294 :fail :cas [8 [26 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,586] jepsen worker 24 - jepsen.util 294 :invoke :write [8 42] INFO [2019-12-17 12:27:56,592] jepsen worker 10 - jepsen.util 370 :ok :read [3 14] INFO [2019-12-17 12:27:56,592] jepsen worker 10 - jepsen.util 370 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,593] jepsen worker 49 - jepsen.util 319 :ok :read [16 8] INFO [2019-12-17 12:27:56,593] jepsen worker 49 - jepsen.util 319 :invoke :read [16 nil] INFO [2019-12-17 12:27:56,635] jepsen worker 13 - jepsen.util 283 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:27:56,636] jepsen worker 13 - jepsen.util 373 :invoke :read [4 nil] INFO [2019-12-17 12:27:56,640] jepsen worker 35 - jepsen.util 215 :ok :read [11 7] INFO [2019-12-17 12:27:56,640] jepsen worker 35 - jepsen.util 215 :invoke :cas [11 [35 39]] INFO [2019-12-17 12:27:56,675] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.03515625}, :ns_1@172.28.128.126 {:progress 0.03237410071942448}, :ns_1@172.28.128.127 {:progress 0.08560311284046696}, :ns_1@172.28.128.128 {:progress 0.07547169811320753}} INFO [2019-12-17 12:27:56,675] jepsen nemesis - couchbase.util Rebalance check count: 8 INFO [2019-12-17 12:27:56,675] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:56,678] jepsen worker 7 - jepsen.util 457 :fail :cas [2 [22 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,678] jepsen worker 7 - jepsen.util 457 :invoke :write [2 20] INFO [2019-12-17 12:27:56,679] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [8 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,680] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [25 40]] INFO [2019-12-17 12:27:56,698] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:56,698] jepsen worker 80 - jepsen.util 530 :info :write [26 42] :RequestCanceledException INFO [2019-12-17 12:27:56,698] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1938,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:56,699] jepsen worker 80 - jepsen.util 620 :invoke :cas [26 [9 28]] INFO [2019-12-17 12:27:56,720] jepsen worker 76 - jepsen.util 256 :info :write [25 36] :RequestCanceledException INFO [2019-12-17 12:27:56,722] jepsen worker 76 - jepsen.util 346 :invoke :write [25 19] INFO [2019-12-17 12:27:56,727] jepsen worker 53 - jepsen.util 503 :ok :read [17 0] INFO [2019-12-17 12:27:56,727] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,727] jepsen worker 53 - jepsen.util 503 :invoke :read [17 nil] INFO [2019-12-17 12:27:56,727] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [23 31]] INFO [2019-12-17 12:27:56,749] jepsen worker 12 - jepsen.util 282 :fail :cas [4 [35 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,749] jepsen worker 12 - jepsen.util 282 :invoke :write [4 36] INFO [2019-12-17 12:27:56,759] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [23 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,759] jepsen worker 11 - jepsen.util 371 :invoke :cas [3 [13 39]] INFO [2019-12-17 12:27:56,775] jepsen worker 64 - jepsen.util 334 :ok :write [21 10] INFO [2019-12-17 12:27:56,775] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,802] jepsen worker 71 - jepsen.util 341 :ok :read [23 29] INFO [2019-12-17 12:27:56,802] jepsen worker 71 - jepsen.util 341 :invoke :write [23 5] INFO [2019-12-17 12:27:56,806] jepsen worker 11 - jepsen.util 371 :fail :cas [3 [13 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,806] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,825] jepsen worker 64 - jepsen.util 334 :ok :read [21 10] INFO [2019-12-17 12:27:56,825] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [15 19]] INFO [2019-12-17 12:27:56,844] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,844] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,857] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [15 19]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,857] jepsen worker 64 - jepsen.util 334 :invoke :write [21 0] INFO [2019-12-17 12:27:56,884] jepsen worker 51 - jepsen.util 231 :fail :cas [17 [47 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,884] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,884] jepsen worker 51 - jepsen.util 231 :invoke :cas [17 [20 34]] INFO [2019-12-17 12:27:56,884] jepsen worker 11 - jepsen.util 371 :invoke :read [3 nil] INFO [2019-12-17 12:27:56,887] jepsen worker 69 - jepsen.util 339 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:27:56,889] jepsen worker 69 - jepsen.util 429 :invoke :write [23 1] INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:56,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1958,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:56,918] jepsen worker 80 - jepsen.util 620 :fail :cas [26 [9 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,918] jepsen worker 80 - jepsen.util 620 :invoke :write [26 19] INFO [2019-12-17 12:27:56,920] jepsen worker 67 - jepsen.util 427 :info :cas [22 [21 24]] :RequestCanceledException INFO [2019-12-17 12:27:56,921] jepsen worker 67 - jepsen.util 517 :invoke :write [22 45] INFO [2019-12-17 12:27:56,931] jepsen worker 11 - jepsen.util 371 :ok :read [3 14] INFO [2019-12-17 12:27:56,931] jepsen worker 11 - jepsen.util 371 :invoke :write [3 48] INFO [2019-12-17 12:27:56,948] jepsen worker 64 - jepsen.util 334 :ok :write [21 0] INFO [2019-12-17 12:27:56,948] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,970] jepsen worker 42 - jepsen.util 582 :fail :cas [14 [29 14]] :ValueNotSwapFrom INFO [2019-12-17 12:27:56,971] jepsen worker 42 - jepsen.util 582 :invoke :write [14 49] INFO [2019-12-17 12:27:56,987] jepsen worker 43 - jepsen.util 493 :info :write [14 48] :RequestCanceledException INFO [2019-12-17 12:27:56,988] jepsen worker 43 - jepsen.util 583 :invoke :cas [14 [13 29]] INFO [2019-12-17 12:27:56,990] jepsen worker 64 - jepsen.util 334 :ok :read [21 0] INFO [2019-12-17 12:27:56,990] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:56,995] jepsen worker 53 - jepsen.util 503 :ok :read [17 0] INFO [2019-12-17 12:27:56,995] jepsen worker 53 - jepsen.util 503 :invoke :cas [17 [16 40]] INFO [2019-12-17 12:27:56,996] jepsen worker 54 - jepsen.util 504 :ok :read [18 47] INFO [2019-12-17 12:27:56,996] jepsen worker 54 - jepsen.util 504 :invoke :read [18 nil] INFO [2019-12-17 12:27:57,016] jepsen worker 64 - jepsen.util 334 :ok :read [21 0] INFO [2019-12-17 12:27:57,016] jepsen worker 64 - jepsen.util 334 :invoke :write [21 14] INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1969,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,018] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1971,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,069] jepsen worker 25 - jepsen.util 475 :fail :cas [8 [40 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,070] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:57,094] jepsen worker 64 - jepsen.util 334 :ok :write [21 14] INFO [2019-12-17 12:27:57,095] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [24 45]] INFO [2019-12-17 12:27:57,109] jepsen worker 30 - jepsen.util 210 :info :write [10 21] :SyncWriteAmbiguous INFO [2019-12-17 12:27:57,110] jepsen worker 30 - jepsen.util 300 :invoke :write [10 9] INFO [2019-12-17 12:27:57,114] jepsen worker 3 - jepsen.util 273 :info :write [1 34] :RequestCanceledException INFO [2019-12-17 12:27:57,116] jepsen worker 3 - jepsen.util 363 :invoke :write [1 27] INFO [2019-12-17 12:27:57,116] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [24 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,116] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [25 5]] INFO [2019-12-17 12:27:57,125] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:57,125] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:57,134] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [25 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,134] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [34 9]] INFO [2019-12-17 12:27:57,153] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [34 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,154] jepsen worker 64 - jepsen.util 334 :invoke :write [21 45] INFO [2019-12-17 12:27:57,183] jepsen worker 38 - jepsen.util 218 :fail :cas [12 [37 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,184] jepsen worker 38 - jepsen.util 218 :invoke :cas [12 [13 40]] INFO [2019-12-17 12:27:57,210] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:57,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1996,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:57,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:57,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:57,212] jepsen worker 50 - jepsen.util 410 :info :write [16 17] :RequestCanceledException INFO [2019-12-17 12:27:57,213] jepsen worker 70 - jepsen.util 520 :ok :read [23 29] INFO [2019-12-17 12:27:57,213] jepsen worker 70 - jepsen.util 520 :invoke :cas [23 [20 17]] INFO [2019-12-17 12:27:57,214] jepsen worker 50 - jepsen.util 500 :invoke :cas [16 [1 43]] INFO [2019-12-17 12:27:57,228] jepsen worker 64 - jepsen.util 334 :ok :write [21 45] INFO [2019-12-17 12:27:57,228] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:57,234] jepsen worker 37 - jepsen.util 217 :info :write [12 34] :RequestCanceledException INFO [2019-12-17 12:27:57,236] jepsen worker 37 - jepsen.util 307 :invoke :write [12 21] INFO [2019-12-17 12:27:57,245] jepsen worker 64 - jepsen.util 334 :ok :read [21 45] INFO [2019-12-17 12:27:57,245] jepsen worker 64 - jepsen.util 334 :invoke :write [21 36] INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1990,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:57,251] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":1991,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:57,264] jepsen worker 54 - jepsen.util 504 :ok :read [18 47] INFO [2019-12-17 12:27:57,264] jepsen worker 54 - jepsen.util 504 :invoke :cas [18 [47 22]] INFO [2019-12-17 12:27:57,276] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:57,277] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:57,283] jepsen worker 4 - jepsen.util 364 :fail :cas [1 [44 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,283] jepsen worker 4 - jepsen.util 364 :invoke :read [1 nil] INFO [2019-12-17 12:27:57,286] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,286] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [45 1]] INFO [2019-12-17 12:27:57,305] jepsen worker 74 - jepsen.util 344 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,305] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:57,324] jepsen worker 35 - jepsen.util 215 :fail :cas [11 [35 39]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,324] jepsen worker 35 - jepsen.util 215 :invoke :write [11 49] INFO [2019-12-17 12:27:57,330] jepsen worker 74 - jepsen.util 344 :ok :read [24 35] INFO [2019-12-17 12:27:57,330] jepsen worker 74 - jepsen.util 344 :invoke :cas [24 [21 46]] INFO [2019-12-17 12:27:57,351] jepsen worker 28 - jepsen.util 298 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,351] jepsen worker 28 - jepsen.util 298 :invoke :write [9 27] INFO [2019-12-17 12:27:57,355] jepsen worker 17 - jepsen.util 377 :info :write [5 22] :SyncWriteAmbiguous INFO [2019-12-17 12:27:57,357] jepsen worker 17 - jepsen.util 467 :invoke :write [5 9] INFO [2019-12-17 12:27:57,369] jepsen worker 74 - jepsen.util 344 :fail :cas [24 [21 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,369] jepsen worker 74 - jepsen.util 344 :invoke :read [24 nil] INFO [2019-12-17 12:27:57,379] jepsen worker 2 - jepsen.util 272 :ok :read [0 14] INFO [2019-12-17 12:27:57,379] jepsen worker 2 - jepsen.util 272 :invoke :write [0 2] INFO [2019-12-17 12:27:57,387] jepsen worker 56 - jepsen.util 326 :info :write [18 26] :RequestCanceledException INFO [2019-12-17 12:27:57,389] jepsen worker 56 - jepsen.util 416 :invoke :cas [18 [7 42]] INFO [2019-12-17 12:27:57,390] jepsen worker 74 - jepsen.util 344 :ok :read [24 35] INFO [2019-12-17 12:27:57,390] jepsen worker 74 - jepsen.util 344 :invoke :cas [24 [3 32]] INFO [2019-12-17 12:27:57,403] jepsen worker 78 - jepsen.util 438 :ok :read [26 47] INFO [2019-12-17 12:27:57,404] jepsen worker 78 - jepsen.util 438 :invoke :write [26 45] INFO [2019-12-17 12:27:57,418] jepsen worker 64 - jepsen.util 334 :ok :write [21 36] INFO [2019-12-17 12:27:57,418] jepsen worker 64 - jepsen.util 334 :invoke :write [21 14] INFO [2019-12-17 12:27:57,428] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:57,428] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2023,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:57,428] jepsen worker 74 - jepsen.util 344 :fail :cas [24 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,428] jepsen worker 74 - jepsen.util 344 :invoke :write [24 21] INFO [2019-12-17 12:27:57,435] jepsen worker 19 - jepsen.util 289 :info :write [6 15] :RequestCanceledException INFO [2019-12-17 12:27:57,435] jepsen worker 61 - jepsen.util 421 :info :write [20 38] :RequestCanceledException INFO [2019-12-17 12:27:57,436] jepsen worker 19 - jepsen.util 379 :invoke :write [6 18] INFO [2019-12-17 12:27:57,436] jepsen worker 61 - jepsen.util 511 :invoke :write [20 25] INFO [2019-12-17 12:27:57,453] jepsen worker 53 - jepsen.util 503 :fail :cas [17 [16 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,453] jepsen worker 53 - jepsen.util 503 :invoke :cas [17 [24 9]] INFO [2019-12-17 12:27:57,493] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:57,493] jepsen worker 25 - jepsen.util 475 :invoke :read [8 nil] INFO [2019-12-17 12:27:57,512] jepsen worker 64 - jepsen.util 334 :ok :write [21 14] INFO [2019-12-17 12:27:57,512] jepsen worker 64 - jepsen.util 334 :invoke :write [21 34] INFO [2019-12-17 12:27:57,521] jepsen worker 14 - jepsen.util 374 :info :write [4 18] :RequestCanceledException INFO [2019-12-17 12:27:57,522] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:57,569] jepsen worker 64 - jepsen.util 334 :ok :write [21 34] INFO [2019-12-17 12:27:57,569] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [17 44]] INFO [2019-12-17 12:27:57,584] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [17 44]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,584] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [8 2]] INFO [2019-12-17 12:27:57,586] jepsen worker 38 - jepsen.util 218 :fail :cas [12 [13 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,587] jepsen worker 38 - jepsen.util 218 :invoke :write [12 4] INFO [2019-12-17 12:27:57,605] jepsen worker 21 - jepsen.util 291 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:27:57,605] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [8 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,605] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [42 2]] INFO [2019-12-17 12:27:57,606] jepsen worker 21 - jepsen.util 381 :invoke :read [7 nil] INFO [2019-12-17 12:27:57,620] jepsen worker 15 - jepsen.util 375 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,620] jepsen worker 79 - jepsen.util 349 :info :write [26 13] :RequestCanceledException INFO [2019-12-17 12:27:57,620] jepsen worker 84 - jepsen.util 84 :info :write [28 13] :RequestCanceledException INFO [2019-12-17 12:27:57,621] jepsen worker 15 - jepsen.util 375 :invoke :write [5 44] INFO [2019-12-17 12:27:57,622] jepsen worker 79 - jepsen.util 439 :invoke :cas [26 [25 18]] INFO [2019-12-17 12:27:57,622] jepsen worker 84 - jepsen.util 174 :invoke :write [28 42] INFO [2019-12-17 12:27:57,631] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [42 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,631] jepsen worker 64 - jepsen.util 334 :invoke :write [21 16] INFO [2019-12-17 12:27:57,639] jepsen worker 13 - jepsen.util 373 :ok :read [4 4] INFO [2019-12-17 12:27:57,639] jepsen worker 13 - jepsen.util 373 :invoke :write [4 29] INFO [2019-12-17 12:27:57,673] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":1708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,673] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:57,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2044,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:57,674] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:57,674] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2047,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:57,687] jepsen worker 82 - jepsen.util 442 :info :write [27 36] :RequestCanceledException INFO [2019-12-17 12:27:57,687] jepsen worker 64 - jepsen.util 334 :ok :write [21 16] INFO [2019-12-17 12:27:57,687] jepsen worker 64 - jepsen.util 334 :invoke :write [21 46] INFO [2019-12-17 12:27:57,688] jepsen worker 82 - jepsen.util 532 :invoke :cas [27 [25 46]] INFO [2019-12-17 12:27:57,693] jepsen worker 4 - jepsen.util 364 :ok :read [1 23] INFO [2019-12-17 12:27:57,693] jepsen worker 4 - jepsen.util 364 :invoke :write [1 33] INFO [2019-12-17 12:27:57,693] jepsen worker 21 - jepsen.util 381 :ok :read [7 20] INFO [2019-12-17 12:27:57,693] jepsen worker 21 - jepsen.util 381 :invoke :cas [7 [49 17]] INFO [2019-12-17 12:27:57,716] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.046875}, :ns_1@172.28.128.126 {:progress 0.04316546762589923}, :ns_1@172.28.128.127 {:progress 0.08560311284046696}, :ns_1@172.28.128.128 {:progress 0.1056603773584905}} INFO [2019-12-17 12:27:57,716] jepsen nemesis - couchbase.util Rebalance check count: 9 INFO [2019-12-17 12:27:57,716] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:57,735] jepsen worker 41 - jepsen.util 311 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,735] jepsen worker 16 - jepsen.util 286 :info :cas [5 [22 24]] :RequestCanceledException INFO [2019-12-17 12:27:57,735] jepsen worker 41 - jepsen.util 311 :invoke :cas [13 [37 13]] INFO [2019-12-17 12:27:57,736] jepsen worker 16 - jepsen.util 376 :invoke :write [5 45] INFO [2019-12-17 12:27:57,744] jepsen worker 21 - jepsen.util 381 :fail :cas [7 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,744] jepsen worker 21 - jepsen.util 381 :invoke :write [7 7] INFO [2019-12-17 12:27:57,751] jepsen worker 68 - jepsen.util 338 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,752] jepsen worker 68 - jepsen.util 338 :invoke :read [22 nil] INFO [2019-12-17 12:27:57,768] jepsen worker 64 - jepsen.util 334 :ok :write [21 46] INFO [2019-12-17 12:27:57,769] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:57,789] jepsen worker 64 - jepsen.util 334 :ok :read [21 46] INFO [2019-12-17 12:27:57,789] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:57,821] jepsen worker 66 - jepsen.util 336 :info :cas [22 [4 34]] :RequestCanceledException INFO [2019-12-17 12:27:57,823] jepsen worker 51 - jepsen.util 231 :fail :cas [17 [20 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,823] jepsen worker 51 - jepsen.util 231 :invoke :write [17 40] INFO [2019-12-17 12:27:57,823] jepsen worker 66 - jepsen.util 426 :invoke :read [22 nil] INFO [2019-12-17 12:27:57,835] jepsen worker 64 - jepsen.util 334 :ok :read [21 46] INFO [2019-12-17 12:27:57,836] jepsen worker 64 - jepsen.util 334 :invoke :write [21 35] INFO [2019-12-17 12:27:57,887] jepsen worker 45 - jepsen.util 495 :info :write [15 40] :RequestCanceledException INFO [2019-12-17 12:27:57,888] jepsen worker 45 - jepsen.util 585 :invoke :write [15 26] INFO [2019-12-17 12:27:57,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:57,889] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:57,889] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:57,890] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2067,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:57,906] jepsen worker 32 - jepsen.util 302 :info :write [10 33] :RequestCanceledException INFO [2019-12-17 12:27:57,907] jepsen worker 32 - jepsen.util 392 :invoke :cas [10 [46 49]] INFO [2019-12-17 12:27:57,913] jepsen worker 64 - jepsen.util 334 :ok :write [21 35] INFO [2019-12-17 12:27:57,913] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [32 28]] INFO [2019-12-17 12:27:57,920] jepsen worker 48 - jepsen.util 408 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,921] jepsen worker 48 - jepsen.util 408 :invoke :cas [16 [7 33]] INFO [2019-12-17 12:27:57,924] jepsen worker 53 - jepsen.util 503 :fail :cas [17 [24 9]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,925] jepsen worker 53 - jepsen.util 503 :invoke :write [17 40] INFO [2019-12-17 12:27:57,929] jepsen worker 40 - jepsen.util 490 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:27:57,929] jepsen worker 40 - jepsen.util 490 :invoke :cas [13 [27 47]] INFO [2019-12-17 12:27:57,934] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [32 28]] :ValueNotSwapFrom INFO [2019-12-17 12:27:57,934] jepsen worker 64 - jepsen.util 334 :invoke :write [21 39] INFO [2019-12-17 12:27:57,948] jepsen worker 52 - jepsen.util 322 :info :write [17 19] :RequestCanceledException INFO [2019-12-17 12:27:57,948] jepsen worker 77 - jepsen.util 347 :info :write [25 29] :RequestCanceledException INFO [2019-12-17 12:27:57,950] jepsen worker 52 - jepsen.util 412 :invoke :read [17 nil] INFO [2019-12-17 12:27:57,950] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [25 40]] INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2081,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:57,999] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2083,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:58,003] jepsen worker 60 - jepsen.util 510 :info :write [20 24] :RequestCanceledException INFO [2019-12-17 12:27:58,004] jepsen worker 60 - jepsen.util 600 :invoke :read [20 nil] INFO [2019-12-17 12:27:58,014] jepsen worker 64 - jepsen.util 334 :ok :write [21 39] INFO [2019-12-17 12:27:58,014] jepsen worker 64 - jepsen.util 334 :invoke :write [21 6] INFO [2019-12-17 12:27:58,021] jepsen worker 6 - jepsen.util 366 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:27:58,021] jepsen worker 29 - jepsen.util 299 :info :write [9 47] :RequestCanceledException INFO [2019-12-17 12:27:58,021] jepsen worker 46 - jepsen.util 316 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:27:58,021] jepsen worker 46 - jepsen.util 316 :invoke :cas [15 [36 46]] INFO [2019-12-17 12:27:58,022] jepsen worker 6 - jepsen.util 456 :invoke :write [2 3] INFO [2019-12-17 12:27:58,022] jepsen worker 29 - jepsen.util 389 :invoke :write [9 43] INFO [2019-12-17 12:27:58,047] jepsen worker 31 - jepsen.util 571 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:27:58,048] jepsen worker 31 - jepsen.util 661 :invoke :cas [10 [9 48]] INFO [2019-12-17 12:27:58,053] jepsen worker 70 - jepsen.util 520 :fail :cas [23 [20 17]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,053] jepsen worker 25 - jepsen.util 475 :ok :read [8 14] INFO [2019-12-17 12:27:58,053] jepsen worker 70 - jepsen.util 520 :invoke :write [23 49] INFO [2019-12-17 12:27:58,053] jepsen worker 25 - jepsen.util 475 :invoke :cas [8 [44 42]] INFO [2019-12-17 12:27:58,085] jepsen worker 81 - jepsen.util 351 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:27:58,087] jepsen worker 81 - jepsen.util 441 :invoke :read [27 nil] INFO [2019-12-17 12:27:58,099] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,100] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:58,104] jepsen worker 27 - jepsen.util 117 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:27:58,105] jepsen worker 27 - jepsen.util 207 :invoke :cas [9 [11 33]] INFO [2019-12-17 12:27:58,106] jepsen worker 64 - jepsen.util 334 :ok :write [21 6] INFO [2019-12-17 12:27:58,107] jepsen worker 64 - jepsen.util 334 :invoke :write [21 24] INFO [2019-12-17 12:27:58,121] jepsen worker 36 - jepsen.util 396 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:27:58,121] jepsen worker 1 - jepsen.util 451 :info :cas [0 [35 31]] :RequestCanceledException INFO [2019-12-17 12:27:58,123] jepsen worker 36 - jepsen.util 486 :invoke :read [12 nil] INFO [2019-12-17 12:27:58,123] jepsen worker 1 - jepsen.util 541 :invoke :cas [0 [41 12]] INFO [2019-12-17 12:27:58,141] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [45 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,141] jepsen worker 88 - jepsen.util 358 :invoke :read [29 nil] INFO [2019-12-17 12:27:58,179] jepsen worker 72 - jepsen.util 252 :info :write [24 25] :SyncWriteAmbiguous INFO [2019-12-17 12:27:58,181] jepsen worker 72 - jepsen.util 342 :invoke :write [24 29] INFO [2019-12-17 12:27:58,196] jepsen worker 18 - jepsen.util 468 :ok :read [6 45] INFO [2019-12-17 12:27:58,196] jepsen worker 18 - jepsen.util 468 :invoke :write [6 49] INFO [2019-12-17 12:27:58,199] jepsen worker 55 - jepsen.util 505 :info :write [18 10] :RequestCanceledException INFO [2019-12-17 12:27:58,199] jepsen worker 34 - jepsen.util 394 :info :write [11 3] :RequestCanceledException INFO [2019-12-17 12:27:58,199] jepsen worker 89 - jepsen.util 359 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:27:58,200] jepsen worker 34 - jepsen.util 484 :invoke :cas [11 [15 1]] INFO [2019-12-17 12:27:58,200] jepsen worker 89 - jepsen.util 449 :invoke :write [29 5] INFO [2019-12-17 12:27:58,200] jepsen worker 55 - jepsen.util 595 :invoke :write [18 11] INFO [2019-12-17 12:27:58,212] jepsen worker 64 - jepsen.util 334 :ok :write [21 24] INFO [2019-12-17 12:27:58,212] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,221] jepsen worker 85 - jepsen.util 355 :info :write [28 25] :RequestCanceledException INFO [2019-12-17 12:27:58,221] jepsen worker 83 - jepsen.util 443 :info :cas [27 [30 23]] :RequestCanceledException INFO [2019-12-17 12:27:58,222] jepsen worker 85 - jepsen.util 445 :invoke :cas [28 [49 6]] INFO [2019-12-17 12:27:58,222] jepsen worker 83 - jepsen.util 533 :invoke :cas [27 [41 37]] INFO [2019-12-17 12:27:58,234] jepsen worker 64 - jepsen.util 334 :ok :read [21 24] INFO [2019-12-17 12:27:58,234] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [16 13]] INFO [2019-12-17 12:27:58,234] jepsen worker 33 - jepsen.util 393 :info :cas [11 [8 40]] :RequestCanceledException INFO [2019-12-17 12:27:58,235] jepsen worker 33 - jepsen.util 483 :invoke :cas [11 [25 18]] INFO [2019-12-17 12:27:58,247] jepsen worker 86 - jepsen.util 176 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:27:58,248] jepsen worker 86 - jepsen.util 266 :invoke :read [28 nil] INFO [2019-12-17 12:27:58,252] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [16 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,252] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [29 25]] INFO [2019-12-17 12:27:58,254] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2100,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:58,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:58,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2101,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:58,269] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [29 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,269] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [4 26]] INFO [2019-12-17 12:27:58,283] jepsen worker 22 - jepsen.util 382 :info :write [7 9] :RequestCanceledException INFO [2019-12-17 12:27:58,284] jepsen worker 22 - jepsen.util 472 :invoke :cas [7 [41 46]] INFO [2019-12-17 12:27:58,286] jepsen worker 63 - jepsen.util 243 :ok :write [21 19] INFO [2019-12-17 12:27:58,286] jepsen worker 63 - jepsen.util 243 :invoke :write [21 30] INFO [2019-12-17 12:27:58,293] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [4 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,293] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:58,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2107,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:58,298] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:58,298] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2111,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:58,301] jepsen worker 87 - jepsen.util 267 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:27:58,302] jepsen worker 87 - jepsen.util 357 :invoke :read [29 nil] INFO [2019-12-17 12:27:58,319] jepsen worker 39 - jepsen.util 399 :info :write [13 31] :RequestCanceledException INFO [2019-12-17 12:27:58,319] jepsen worker 65 - jepsen.util 335 :info :write [21 22] :RequestCanceledException INFO [2019-12-17 12:27:58,320] jepsen worker 65 - jepsen.util 425 :invoke :cas [21 [29 21]] INFO [2019-12-17 12:27:58,320] jepsen worker 39 - jepsen.util 489 :invoke :write [13 15] INFO [2019-12-17 12:27:58,335] jepsen worker 8 - jepsen.util 458 :info :write [2 11] :RequestCanceledException INFO [2019-12-17 12:27:58,337] jepsen worker 8 - jepsen.util 548 :invoke :cas [2 [29 41]] INFO [2019-12-17 12:27:58,348] jepsen worker 43 - jepsen.util 583 :fail :cas [14 [13 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,348] jepsen worker 43 - jepsen.util 583 :invoke :write [14 37] INFO [2019-12-17 12:27:58,381] jepsen worker 64 - jepsen.util 334 :ok :read [21 19] INFO [2019-12-17 12:27:58,381] jepsen worker 64 - jepsen.util 334 :invoke :write [21 36] INFO [2019-12-17 12:27:58,386] jepsen worker 5 - jepsen.util 95 :info :write [1 11] :RequestCanceledException INFO [2019-12-17 12:27:58,387] jepsen worker 5 - jepsen.util 185 :invoke :read [1 nil] INFO [2019-12-17 12:27:58,388] jepsen worker 65 - jepsen.util 425 :fail :cas [21 [29 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,388] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,403] jepsen worker 58 - jepsen.util 148 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:27:58,405] jepsen worker 58 - jepsen.util 238 :invoke :cas [19 [45 39]] INFO [2019-12-17 12:27:58,410] jepsen worker 52 - jepsen.util 412 :ok :read [17 0] INFO [2019-12-17 12:27:58,410] jepsen worker 52 - jepsen.util 412 :invoke :cas [17 [17 26]] INFO [2019-12-17 12:27:58,418] jepsen worker 26 - jepsen.util 476 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:27:58,420] jepsen worker 26 - jepsen.util 566 :invoke :write [8 30] INFO [2019-12-17 12:27:58,422] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [25 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,422] jepsen worker 77 - jepsen.util 437 :invoke :read [25 nil] INFO [2019-12-17 12:27:58,435] jepsen worker 23 - jepsen.util 203 :info :cas [7 [29 14]] :RequestCanceledException INFO [2019-12-17 12:27:58,437] jepsen worker 23 - jepsen.util 293 :invoke :read [7 nil] INFO [2019-12-17 12:27:58,438] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,438] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:58,446] jepsen worker 9 - jepsen.util 279 :info :cas [3 [32 42]] :RequestCanceledException INFO [2019-12-17 12:27:58,447] jepsen worker 9 - jepsen.util 369 :invoke :write [3 39] INFO [2019-12-17 12:27:58,456] jepsen worker 64 - jepsen.util 334 :ok :write [21 36] INFO [2019-12-17 12:27:58,456] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,472] jepsen worker 65 - jepsen.util 425 :ok :read [21 36] INFO [2019-12-17 12:27:58,472] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,491] jepsen worker 64 - jepsen.util 334 :ok :read [21 36] INFO [2019-12-17 12:27:58,491] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,499] jepsen worker 65 - jepsen.util 425 :ok :read [21 36] INFO [2019-12-17 12:27:58,499] jepsen worker 65 - jepsen.util 425 :invoke :write [21 12] INFO [2019-12-17 12:27:58,503] jepsen worker 57 - jepsen.util 327 :info :write [19 18] :RequestCanceledException INFO [2019-12-17 12:27:58,505] jepsen worker 57 - jepsen.util 417 :invoke :write [19 26] INFO [2019-12-17 12:27:58,516] jepsen worker 64 - jepsen.util 334 :ok :read [21 36] INFO [2019-12-17 12:27:58,517] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [17 4]] INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:58,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2143,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:58,551] jepsen worker 0 - jepsen.util 450 :info :cas [0 [29 25]] :RequestCanceledException INFO [2019-12-17 12:27:58,553] jepsen worker 0 - jepsen.util 540 :invoke :cas [0 [48 1]] INFO [2019-12-17 12:27:58,573] jepsen worker 36 - jepsen.util 486 :ok :read [12 12] INFO [2019-12-17 12:27:58,574] jepsen worker 36 - jepsen.util 486 :invoke :cas [12 [20 10]] INFO [2019-12-17 12:27:58,574] jepsen worker 79 - jepsen.util 439 :fail :cas [26 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,575] jepsen worker 79 - jepsen.util 439 :invoke :cas [26 [21 37]] INFO [2019-12-17 12:27:58,584] jepsen worker 65 - jepsen.util 425 :ok :write [21 12] INFO [2019-12-17 12:27:58,584] jepsen worker 65 - jepsen.util 425 :invoke :write [21 23] INFO [2019-12-17 12:27:58,586] jepsen worker 47 - jepsen.util 497 :info :write [15 37] :RequestCanceledException INFO [2019-12-17 12:27:58,587] jepsen worker 47 - jepsen.util 587 :invoke :cas [15 [25 34]] INFO [2019-12-17 12:27:58,598] jepsen worker 62 - jepsen.util 422 :info :write [20 36] :RequestCanceledException INFO [2019-12-17 12:27:58,600] jepsen worker 62 - jepsen.util 512 :invoke :cas [20 [33 2]] INFO [2019-12-17 12:27:58,603] jepsen worker 33 - jepsen.util 483 :fail :cas [11 [25 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,603] jepsen worker 33 - jepsen.util 483 :invoke :cas [11 [8 21]] INFO [2019-12-17 12:27:58,618] jepsen worker 73 - jepsen.util 433 :info :cas [24 [35 31]] :RequestCanceledException INFO [2019-12-17 12:27:58,620] jepsen worker 73 - jepsen.util 523 :invoke :cas [24 [49 0]] INFO [2019-12-17 12:27:58,634] jepsen worker 24 - jepsen.util 294 :info :write [8 42] :RequestCanceledException INFO [2019-12-17 12:27:58,634] jepsen worker 20 - jepsen.util 380 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:27:58,634] jepsen worker 10 - jepsen.util 370 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:27:58,635] jepsen worker 10 - jepsen.util 370 :invoke :cas [3 [4 45]] INFO [2019-12-17 12:27:58,636] jepsen worker 24 - jepsen.util 384 :invoke :cas [8 [8 29]] INFO [2019-12-17 12:27:58,636] jepsen worker 20 - jepsen.util 470 :invoke :write [6 48] INFO [2019-12-17 12:27:58,641] jepsen worker 64 - jepsen.util 334 :fail :cas [21 [17 4]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,641] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,652] jepsen worker 59 - jepsen.util 239 :info :write [19 9] :RequestCanceledException INFO [2019-12-17 12:27:58,652] jepsen worker 49 - jepsen.util 319 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:27:58,652] jepsen worker 49 - jepsen.util 319 :invoke :cas [16 [17 23]] INFO [2019-12-17 12:27:58,653] jepsen worker 59 - jepsen.util 329 :invoke :cas [19 [30 37]] INFO [2019-12-17 12:27:58,656] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:58,657] jepsen worker 44 - jepsen.util 224 :invoke :read [14 nil] INFO [2019-12-17 12:27:58,661] jepsen worker 47 - jepsen.util 587 :fail :cas [15 [25 34]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,661] jepsen worker 47 - jepsen.util 587 :invoke :write [15 19] INFO [2019-12-17 12:27:58,670] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:58,670] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:58,696] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,697] jepsen worker 14 - jepsen.util 464 :invoke :read [4 nil] INFO [2019-12-17 12:27:58,722] jepsen worker 65 - jepsen.util 425 :ok :write [21 23] INFO [2019-12-17 12:27:58,722] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,726] jepsen worker 36 - jepsen.util 486 :fail :cas [12 [20 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,726] jepsen worker 36 - jepsen.util 486 :invoke :read [12 nil] INFO [2019-12-17 12:27:58,737] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.046875}, :ns_1@172.28.128.126 {:progress 0.04316546762589923}, :ns_1@172.28.128.127 {:progress 0.0933852140077821}, :ns_1@172.28.128.128 {:progress 0.1283018867924528}} INFO [2019-12-17 12:27:58,737] jepsen nemesis - couchbase.util Rebalance check count: 10 INFO [2019-12-17 12:27:58,737] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:58,740] jepsen worker 25 - jepsen.util 475 :fail :cas [8 [44 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,740] jepsen worker 25 - jepsen.util 475 :invoke :write [8 3] INFO [2019-12-17 12:27:58,748] jepsen worker 76 - jepsen.util 346 :info :write [25 19] :RequestCanceledException INFO [2019-12-17 12:27:58,750] jepsen worker 76 - jepsen.util 436 :invoke :write [25 38] INFO [2019-12-17 12:27:58,762] jepsen worker 10 - jepsen.util 370 :fail :cas [3 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,762] jepsen worker 10 - jepsen.util 370 :invoke :cas [3 [20 37]] INFO [2019-12-17 12:27:58,787] jepsen worker 7 - jepsen.util 457 :info :write [2 20] :RequestCanceledException INFO [2019-12-17 12:27:58,789] jepsen worker 7 - jepsen.util 547 :invoke :write [2 24] INFO [2019-12-17 12:27:58,789] jepsen worker 64 - jepsen.util 334 :ok :read [21 23] INFO [2019-12-17 12:27:58,789] jepsen worker 64 - jepsen.util 334 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,811] jepsen worker 65 - jepsen.util 425 :ok :read [21 23] INFO [2019-12-17 12:27:58,811] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,821] jepsen worker 12 - jepsen.util 282 :info :write [4 36] :RequestCanceledException INFO [2019-12-17 12:27:58,822] jepsen worker 12 - jepsen.util 372 :invoke :write [4 9] INFO [2019-12-17 12:27:58,845] jepsen worker 14 - jepsen.util 464 :ok :read [4 4] INFO [2019-12-17 12:27:58,845] jepsen worker 14 - jepsen.util 464 :invoke :write [4 12] INFO [2019-12-17 12:27:58,848] jepsen worker 10 - jepsen.util 370 :fail :cas [3 [20 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,848] jepsen worker 10 - jepsen.util 370 :invoke :write [3 40] INFO [2019-12-17 12:27:58,856] jepsen worker 60 - jepsen.util 600 :ok :read [20 7] INFO [2019-12-17 12:27:58,856] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [22 5]] INFO [2019-12-17 12:27:58,861] jepsen worker 64 - jepsen.util 334 :ok :read [21 23] INFO [2019-12-17 12:27:58,862] jepsen worker 64 - jepsen.util 334 :invoke :cas [21 [21 43]] INFO [2019-12-17 12:27:58,874] jepsen worker 65 - jepsen.util 425 :ok :read [21 23] INFO [2019-12-17 12:27:58,875] jepsen worker 65 - jepsen.util 425 :invoke :read [21 nil] INFO [2019-12-17 12:27:58,881] jepsen worker 71 - jepsen.util 341 :info :write [23 5] :RequestCanceledException INFO [2019-12-17 12:27:58,882] jepsen worker 88 - jepsen.util 358 :ok :read [29 5] INFO [2019-12-17 12:27:58,882] jepsen worker 71 - jepsen.util 431 :invoke :cas [23 [46 33]] INFO [2019-12-17 12:27:58,882] jepsen worker 88 - jepsen.util 358 :invoke :cas [29 [22 0]] INFO [2019-12-17 12:27:58,903] jepsen worker 52 - jepsen.util 412 :fail :cas [17 [17 26]] :ValueNotSwapFrom INFO [2019-12-17 12:27:58,904] jepsen worker 52 - jepsen.util 412 :invoke :read [17 nil] INFO [2019-12-17 12:27:58,914] jepsen worker 23 - jepsen.util 293 :ok :read [7 20] INFO [2019-12-17 12:27:58,915] jepsen worker 23 - jepsen.util 293 :invoke :write [7 29] INFO [2019-12-17 12:27:58,921] jepsen worker 69 - jepsen.util 429 :info :write [23 1] :RequestCanceledException INFO [2019-12-17 12:27:58,922] jepsen worker 69 - jepsen.util 519 :invoke :write [23 48] INFO [2019-12-17 12:27:58,934] jepsen worker 80 - jepsen.util 620 :info :write [26 19] :RequestCanceledException INFO [2019-12-17 12:27:58,936] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:27:58,942] jepsen worker 36 - jepsen.util 486 :ok :read [12 12] INFO [2019-12-17 12:27:58,942] jepsen worker 36 - jepsen.util 486 :invoke :read [12 nil] INFO [2019-12-17 12:27:58,963] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:58,963] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2195,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:58,968] jepsen worker 77 - jepsen.util 437 :ok :read [25 2] INFO [2019-12-17 12:27:58,968] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [13 30]] INFO [2019-12-17 12:27:58,975] jepsen worker 28 - jepsen.util 298 :ok :write [9 27] INFO [2019-12-17 12:27:58,975] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:58,988] jepsen worker 42 - jepsen.util 582 :info :write [14 49] :RequestCanceledException INFO [2019-12-17 12:27:58,988] jepsen worker 67 - jepsen.util 517 :info :write [22 45] :RequestCanceledException INFO [2019-12-17 12:27:58,989] jepsen worker 67 - jepsen.util 607 :invoke :write [22 43] INFO [2019-12-17 12:27:58,989] jepsen worker 42 - jepsen.util 672 :invoke :write [14 36] INFO [2019-12-17 12:27:59,002] jepsen worker 28 - jepsen.util 298 :ok :read [9 27] INFO [2019-12-17 12:27:59,002] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,004] jepsen worker 11 - jepsen.util 371 :info :write [3 48] :RequestCanceledException INFO [2019-12-17 12:27:59,006] jepsen worker 11 - jepsen.util 461 :invoke :write [3 26] INFO [2019-12-17 12:27:59,006] jepsen worker 52 - jepsen.util 412 :ok :read [17 0] INFO [2019-12-17 12:27:59,007] jepsen worker 52 - jepsen.util 412 :invoke :read [17 nil] INFO [2019-12-17 12:27:59,035] jepsen worker 28 - jepsen.util 298 :ok :read [9 27] INFO [2019-12-17 12:27:59,035] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:27:59,049] jepsen worker 22 - jepsen.util 472 :fail :cas [7 [41 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,050] jepsen worker 22 - jepsen.util 472 :invoke :write [7 36] INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:27:59,073] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2212,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:59,085] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:27:59,086] jepsen worker 28 - jepsen.util 298 :invoke :write [9 47] INFO [2019-12-17 12:27:59,111] jepsen worker 62 - jepsen.util 512 :fail :cas [20 [33 2]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,112] jepsen worker 62 - jepsen.util 512 :invoke :write [20 2] INFO [2019-12-17 12:27:59,121] jepsen worker 44 - jepsen.util 224 :ok :read [14 27] INFO [2019-12-17 12:27:59,121] jepsen worker 44 - jepsen.util 224 :invoke :cas [14 [42 11]] INFO [2019-12-17 12:27:59,134] jepsen worker 3 - jepsen.util 363 :info :write [1 27] :RequestCanceledException INFO [2019-12-17 12:27:59,134] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:27:59,194] jepsen worker 28 - jepsen.util 298 :ok :write [9 47] INFO [2019-12-17 12:27:59,194] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [22 30]] INFO [2019-12-17 12:27:59,214] jepsen worker 8 - jepsen.util 548 :fail :cas [2 [29 41]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,215] jepsen worker 8 - jepsen.util 548 :invoke :cas [2 [36 43]] INFO [2019-12-17 12:27:59,215] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [22 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,216] jepsen worker 28 - jepsen.util 298 :invoke :write [9 18] INFO [2019-12-17 12:27:59,221] jepsen worker 30 - jepsen.util 300 :info :write [10 9] :RequestCanceledException INFO [2019-12-17 12:27:59,223] jepsen worker 30 - jepsen.util 390 :invoke :cas [10 [35 12]] INFO [2019-12-17 12:27:59,239] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:27:59,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2226,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:27:59,246] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [13 30]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,246] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [10 29]] INFO [2019-12-17 12:27:59,250] jepsen worker 34 - jepsen.util 484 :fail :cas [11 [15 1]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,251] jepsen worker 34 - jepsen.util 484 :invoke :cas [11 [12 18]] INFO [2019-12-17 12:27:59,252] jepsen worker 50 - jepsen.util 500 :info :cas [16 [1 43]] :RequestCanceledException INFO [2019-12-17 12:27:59,253] jepsen worker 50 - jepsen.util 590 :invoke :read [16 nil] INFO [2019-12-17 12:27:59,266] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:27:59,267] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [8 10]] INFO [2019-12-17 12:27:59,270] jepsen worker 5 - jepsen.util 185 :ok :read [1 23] INFO [2019-12-17 12:27:59,270] jepsen worker 5 - jepsen.util 185 :invoke :cas [1 [0 8]] INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:27:59,283] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2237,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:27:59,289] jepsen worker 36 - jepsen.util 486 :ok :read [12 12] INFO [2019-12-17 12:27:59,289] jepsen worker 36 - jepsen.util 486 :invoke :cas [12 [43 13]] INFO [2019-12-17 12:27:59,295] jepsen worker 28 - jepsen.util 298 :ok :write [9 18] INFO [2019-12-17 12:27:59,295] jepsen worker 28 - jepsen.util 298 :invoke :write [9 16] INFO [2019-12-17 12:27:59,301] jepsen worker 19 - jepsen.util 379 :info :write [6 18] :SyncWriteAmbiguous INFO [2019-12-17 12:27:59,302] jepsen worker 19 - jepsen.util 469 :invoke :cas [6 [28 42]] INFO [2019-12-17 12:27:59,318] jepsen worker 37 - jepsen.util 307 :info :write [12 21] :RequestCanceledException INFO [2019-12-17 12:27:59,320] jepsen worker 37 - jepsen.util 397 :invoke :write [12 16] INFO [2019-12-17 12:27:59,321] jepsen worker 56 - jepsen.util 416 :fail :cas [18 [7 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,321] jepsen worker 56 - jepsen.util 416 :invoke :read [18 nil] INFO [2019-12-17 12:27:59,332] jepsen worker 44 - jepsen.util 224 :fail :cas [14 [42 11]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,332] jepsen worker 19 - jepsen.util 469 :fail :cas [6 [28 42]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,332] jepsen worker 19 - jepsen.util 469 :invoke :write [6 34] INFO [2019-12-17 12:27:59,332] jepsen worker 44 - jepsen.util 224 :invoke :write [14 25] INFO [2019-12-17 12:27:59,333] jepsen worker 46 - jepsen.util 316 :fail :cas [15 [36 46]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,333] jepsen worker 46 - jepsen.util 316 :invoke :write [15 24] INFO [2019-12-17 12:27:59,341] jepsen worker 79 - jepsen.util 439 :fail :cas [26 [21 37]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,341] jepsen worker 79 - jepsen.util 439 :invoke :read [26 nil] INFO [2019-12-17 12:27:59,349] jepsen worker 27 - jepsen.util 207 :fail :cas [9 [11 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,350] jepsen worker 27 - jepsen.util 207 :invoke :write [9 42] INFO [2019-12-17 12:27:59,365] jepsen worker 33 - jepsen.util 483 :fail :cas [11 [8 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,365] jepsen worker 33 - jepsen.util 483 :invoke :read [11 nil] INFO [2019-12-17 12:27:59,376] jepsen worker 24 - jepsen.util 384 :fail :cas [8 [8 29]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,376] jepsen worker 24 - jepsen.util 384 :invoke :read [8 nil] INFO [2019-12-17 12:27:59,378] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [22 5]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,378] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [3 31]] INFO [2019-12-17 12:27:59,386] jepsen worker 17 - jepsen.util 467 :info :write [5 9] :RequestCanceledException INFO [2019-12-17 12:27:59,386] jepsen worker 8 - jepsen.util 548 :fail :cas [2 [36 43]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,387] jepsen worker 8 - jepsen.util 548 :invoke :read [2 nil] INFO [2019-12-17 12:27:59,387] jepsen worker 17 - jepsen.util 557 :invoke :read [5 nil] INFO [2019-12-17 12:27:59,403] jepsen worker 35 - jepsen.util 215 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:27:59,405] jepsen worker 35 - jepsen.util 305 :invoke :cas [11 [34 27]] INFO [2019-12-17 12:27:59,413] jepsen worker 71 - jepsen.util 431 :fail :cas [23 [46 33]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,413] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:27:59,414] jepsen worker 28 - jepsen.util 298 :ok :write [9 16] INFO [2019-12-17 12:27:59,414] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [6 21]] INFO [2019-12-17 12:27:59,421] jepsen worker 2 - jepsen.util 272 :info :write [0 2] :RequestCanceledException INFO [2019-12-17 12:27:59,423] jepsen worker 2 - jepsen.util 362 :invoke :cas [0 [45 22]] INFO [2019-12-17 12:27:59,429] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:59,429] jepsen worker 75 - jepsen.util 345 :invoke :read [25 nil] INFO [2019-12-17 12:27:59,436] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [6 21]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,436] jepsen worker 28 - jepsen.util 298 :invoke :write [9 48] INFO [2019-12-17 12:27:59,473] jepsen worker 52 - jepsen.util 412 :ok :read [17 0] INFO [2019-12-17 12:27:59,473] jepsen worker 52 - jepsen.util 412 :invoke :cas [17 [17 31]] INFO [2019-12-17 12:27:59,474] jepsen worker 8 - jepsen.util 548 :ok :read [2 15] INFO [2019-12-17 12:27:59,475] jepsen worker 8 - jepsen.util 548 :invoke :read [2 nil] INFO [2019-12-17 12:27:59,487] jepsen worker 78 - jepsen.util 438 :info :write [26 45] :RequestCanceledException INFO [2019-12-17 12:27:59,488] jepsen worker 78 - jepsen.util 528 :invoke :read [26 nil] INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2265,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:27:59,503] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2268,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:27:59,505] jepsen worker 74 - jepsen.util 344 :info :write [24 21] :RequestCanceledException INFO [2019-12-17 12:27:59,507] jepsen worker 74 - jepsen.util 434 :invoke :write [24 18] INFO [2019-12-17 12:27:59,518] jepsen worker 28 - jepsen.util 298 :ok :write [9 48] INFO [2019-12-17 12:27:59,518] jepsen worker 28 - jepsen.util 298 :invoke :write [9 1] INFO [2019-12-17 12:27:59,526] jepsen worker 56 - jepsen.util 416 :ok :read [18 47] INFO [2019-12-17 12:27:59,527] jepsen worker 56 - jepsen.util 416 :invoke :read [18 nil] INFO [2019-12-17 12:27:59,534] jepsen worker 61 - jepsen.util 511 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:27:59,536] jepsen worker 61 - jepsen.util 601 :invoke :read [20 nil] INFO [2019-12-17 12:27:59,578] jepsen worker 24 - jepsen.util 384 :ok :read [8 14] INFO [2019-12-17 12:27:59,578] jepsen worker 24 - jepsen.util 384 :invoke :read [8 nil] INFO [2019-12-17 12:27:59,608] jepsen worker 34 - jepsen.util 484 :fail :cas [11 [12 18]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,608] jepsen worker 34 - jepsen.util 484 :invoke :write [11 27] INFO [2019-12-17 12:27:59,619] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":1970,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:27:59,624] jepsen worker 87 - jepsen.util 357 :ok :read [29 5] INFO [2019-12-17 12:27:59,624] jepsen worker 87 - jepsen.util 357 :invoke :cas [29 [49 23]] INFO [2019-12-17 12:27:59,635] jepsen worker 28 - jepsen.util 298 :ok :write [9 1] INFO [2019-12-17 12:27:59,635] jepsen worker 28 - jepsen.util 298 :invoke :write [9 10] INFO [2019-12-17 12:27:59,648] jepsen worker 33 - jepsen.util 483 :ok :read [11 7] INFO [2019-12-17 12:27:59,648] jepsen worker 33 - jepsen.util 483 :invoke :write [11 33] INFO [2019-12-17 12:27:59,666] jepsen worker 41 - jepsen.util 311 :fail :cas [13 [37 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,667] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:27:59,687] jepsen worker 38 - jepsen.util 218 :info :write [12 4] :RequestCanceledException INFO [2019-12-17 12:27:59,688] jepsen worker 38 - jepsen.util 308 :invoke :cas [12 [47 40]] INFO [2019-12-17 12:27:59,702] jepsen worker 28 - jepsen.util 298 :ok :write [9 10] INFO [2019-12-17 12:27:59,703] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,704] jepsen worker 84 - jepsen.util 174 :info :write [28 42] :RequestCanceledException INFO [2019-12-17 12:27:59,706] jepsen worker 84 - jepsen.util 264 :invoke :write [28 1] INFO [2019-12-17 12:27:59,710] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [8 10]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,710] jepsen worker 8 - jepsen.util 548 :ok :read [2 15] INFO [2019-12-17 12:27:59,710] jepsen worker 8 - jepsen.util 548 :invoke :read [2 nil] INFO [2019-12-17 12:27:59,710] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:27:59,713] jepsen worker 82 - jepsen.util 532 :info :cas [27 [25 46]] :RequestCanceledException INFO [2019-12-17 12:27:59,713] jepsen worker 13 - jepsen.util 373 :info :write [4 29] :RequestCanceledException INFO [2019-12-17 12:27:59,713] jepsen worker 15 - jepsen.util 375 :info :write [5 44] :RequestCanceledException INFO [2019-12-17 12:27:59,714] jepsen worker 82 - jepsen.util 622 :invoke :read [27 nil] INFO [2019-12-17 12:27:59,714] jepsen worker 15 - jepsen.util 465 :invoke :cas [5 [40 15]] INFO [2019-12-17 12:27:59,714] jepsen worker 13 - jepsen.util 463 :invoke :read [4 nil] INFO [2019-12-17 12:27:59,723] jepsen worker 28 - jepsen.util 298 :ok :read [9 10] INFO [2019-12-17 12:27:59,724] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [20 38]] INFO [2019-12-17 12:27:59,735] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:27:59,735] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2288,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:27:59,741] jepsen worker 36 - jepsen.util 486 :fail :cas [12 [43 13]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,741] jepsen worker 36 - jepsen.util 486 :invoke :write [12 0] INFO [2019-12-17 12:27:59,745] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [20 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,745] jepsen worker 28 - jepsen.util 298 :invoke :write [9 2] INFO [2019-12-17 12:27:59,749] jepsen worker 75 - jepsen.util 345 :ok :read [25 2] INFO [2019-12-17 12:27:59,749] jepsen worker 35 - jepsen.util 305 :fail :cas [11 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,749] jepsen worker 75 - jepsen.util 345 :invoke :write [25 25] INFO [2019-12-17 12:27:59,749] jepsen worker 35 - jepsen.util 305 :invoke :write [11 38] INFO [2019-12-17 12:27:59,752] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:27:59,752] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:27:59,756] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.1011673151750972}, :ns_1@172.28.128.128 {:progress 0.1735849056603773}} INFO [2019-12-17 12:27:59,756] jepsen nemesis - couchbase.util Rebalance check count: 11 INFO [2019-12-17 12:27:59,756] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:27:59,773] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:27:59,773] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:27:59,787] jepsen worker 4 - jepsen.util 364 :info :write [1 33] :RequestCanceledException INFO [2019-12-17 12:27:59,788] jepsen worker 4 - jepsen.util 454 :invoke :write [1 21] INFO [2019-12-17 12:27:59,811] jepsen worker 28 - jepsen.util 298 :ok :write [9 2] INFO [2019-12-17 12:27:59,811] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,821] jepsen worker 21 - jepsen.util 381 :info :write [7 7] :RequestCanceledException INFO [2019-12-17 12:27:59,822] jepsen worker 21 - jepsen.util 471 :invoke :write [7 38] INFO [2019-12-17 12:27:59,830] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,830] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [12 8]] INFO [2019-12-17 12:27:59,834] jepsen worker 16 - jepsen.util 376 :info :write [5 45] :RequestCanceledException INFO [2019-12-17 12:27:59,835] jepsen worker 16 - jepsen.util 466 :invoke :write [5 37] INFO [2019-12-17 12:27:59,849] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,849] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [36 38]] INFO [2019-12-17 12:27:59,851] jepsen worker 66 - jepsen.util 426 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:59,851] jepsen worker 68 - jepsen.util 338 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:27:59,851] jepsen worker 66 - jepsen.util 426 :invoke :read [22 nil] INFO [2019-12-17 12:27:59,851] jepsen worker 68 - jepsen.util 338 :invoke :read [22 nil] INFO [2019-12-17 12:27:59,865] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [36 38]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,866] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,876] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,877] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,890] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,890] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,900] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,900] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [46 25]] INFO [2019-12-17 12:27:59,909] jepsen worker 5 - jepsen.util 185 :fail :cas [1 [0 8]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,909] jepsen worker 5 - jepsen.util 185 :invoke :cas [1 [12 20]] INFO [2019-12-17 12:27:59,913] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [46 25]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,913] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [18 31]] INFO [2019-12-17 12:27:59,919] jepsen worker 51 - jepsen.util 231 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:27:59,919] jepsen worker 45 - jepsen.util 585 :info :write [15 26] :RequestCanceledException INFO [2019-12-17 12:27:59,920] jepsen worker 45 - jepsen.util 675 :invoke :read [15 nil] INFO [2019-12-17 12:27:59,920] jepsen worker 51 - jepsen.util 321 :invoke :write [17 30] INFO [2019-12-17 12:27:59,935] jepsen worker 53 - jepsen.util 503 :info :write [17 40] :RequestCanceledException INFO [2019-12-17 12:27:59,936] jepsen worker 53 - jepsen.util 593 :invoke :read [17 nil] INFO [2019-12-17 12:27:59,944] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [18 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,944] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2316,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2319,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:27:59,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2320,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:27:59,957] jepsen worker 28 - jepsen.util 298 :ok :read [9 2] INFO [2019-12-17 12:27:59,957] jepsen worker 52 - jepsen.util 412 :fail :cas [17 [17 31]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,957] jepsen worker 28 - jepsen.util 298 :invoke :write [9 21] INFO [2019-12-17 12:27:59,957] jepsen worker 52 - jepsen.util 412 :invoke :write [17 0] INFO [2019-12-17 12:27:59,979] jepsen worker 38 - jepsen.util 308 :fail :cas [12 [47 40]] :ValueNotSwapFrom INFO [2019-12-17 12:27:59,979] jepsen worker 38 - jepsen.util 308 :invoke :write [12 20] INFO [2019-12-17 12:27:59,981] jepsen worker 56 - jepsen.util 416 :ok :read [18 47] INFO [2019-12-17 12:27:59,981] jepsen worker 56 - jepsen.util 416 :invoke :read [18 nil] INFO [2019-12-17 12:27:59,987] jepsen worker 32 - jepsen.util 392 :info :cas [10 [46 49]] :RequestCanceledException INFO [2019-12-17 12:27:59,989] jepsen worker 32 - jepsen.util 482 :invoke :cas [10 [4 24]] INFO [2019-12-17 12:28:00,010] jepsen worker 79 - jepsen.util 439 :ok :read [26 47] INFO [2019-12-17 12:28:00,010] jepsen worker 79 - jepsen.util 439 :invoke :read [26 nil] INFO [2019-12-17 12:28:00,011] jepsen worker 32 - jepsen.util 482 :fail :cas [10 [4 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,011] jepsen worker 32 - jepsen.util 482 :invoke :cas [10 [29 22]] INFO [2019-12-17 12:28:00,020] jepsen worker 48 - jepsen.util 408 :info :cas [16 [7 33]] :RequestCanceledException INFO [2019-12-17 12:28:00,022] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [10 36]] INFO [2019-12-17 12:28:00,025] jepsen worker 28 - jepsen.util 298 :ok :write [9 21] INFO [2019-12-17 12:28:00,026] jepsen worker 28 - jepsen.util 298 :invoke :write [9 26] INFO [2019-12-17 12:28:00,029] jepsen worker 32 - jepsen.util 482 :fail :cas [10 [29 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,029] jepsen worker 32 - jepsen.util 482 :invoke :cas [10 [9 22]] INFO [2019-12-17 12:28:00,034] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [10 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,034] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [42 6]] INFO [2019-12-17 12:28:00,034] jepsen worker 40 - jepsen.util 490 :info :cas [13 [27 47]] :RequestCanceledException INFO [2019-12-17 12:28:00,036] jepsen worker 40 - jepsen.util 580 :invoke :write [13 2] INFO [2019-12-17 12:28:00,051] jepsen worker 32 - jepsen.util 482 :fail :cas [10 [9 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,051] jepsen worker 13 - jepsen.util 463 :ok :read [4 4] INFO [2019-12-17 12:28:00,051] jepsen worker 32 - jepsen.util 482 :invoke :write [10 45] INFO [2019-12-17 12:28:00,051] jepsen worker 13 - jepsen.util 463 :invoke :write [4 17] INFO [2019-12-17 12:28:00,051] jepsen worker 6 - jepsen.util 456 :info :write [2 3] :RequestCanceledException INFO [2019-12-17 12:28:00,053] jepsen worker 6 - jepsen.util 546 :invoke :cas [2 [29 46]] INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2335,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:00,058] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2336,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:00,080] jepsen worker 78 - jepsen.util 528 :ok :read [26 47] INFO [2019-12-17 12:28:00,080] jepsen worker 78 - jepsen.util 528 :invoke :cas [26 [2 3]] INFO [2019-12-17 12:28:00,088] jepsen worker 31 - jepsen.util 661 :info :cas [10 [9 48]] :RequestCanceledException INFO [2019-12-17 12:28:00,089] jepsen worker 31 - jepsen.util 751 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,103] jepsen worker 28 - jepsen.util 298 :ok :write [9 26] INFO [2019-12-17 12:28:00,103] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [0 47]] INFO [2019-12-17 12:28:00,106] jepsen worker 81 - jepsen.util 441 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:00,106] jepsen worker 81 - jepsen.util 441 :invoke :cas [27 [6 17]] INFO [2019-12-17 12:28:00,120] jepsen worker 29 - jepsen.util 389 :info :write [9 43] :RequestCanceledException INFO [2019-12-17 12:28:00,121] jepsen worker 24 - jepsen.util 384 :ok :read [8 14] INFO [2019-12-17 12:28:00,121] jepsen worker 70 - jepsen.util 520 :info :write [23 49] :RequestCanceledException INFO [2019-12-17 12:28:00,121] jepsen worker 24 - jepsen.util 384 :invoke :cas [8 [37 7]] INFO [2019-12-17 12:28:00,122] jepsen worker 29 - jepsen.util 479 :invoke :write [9 45] INFO [2019-12-17 12:28:00,122] jepsen worker 70 - jepsen.util 610 :invoke :write [23 32] INFO [2019-12-17 12:28:00,128] jepsen worker 1 - jepsen.util 541 :info :cas [0 [41 12]] :RequestCanceledException INFO [2019-12-17 12:28:00,130] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:00,133] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [0 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,133] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [45 33]] INFO [2019-12-17 12:28:00,155] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [45 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,155] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:00,172] jepsen worker 72 - jepsen.util 342 :info :write [24 29] :SyncWriteAmbiguous INFO [2019-12-17 12:28:00,173] jepsen worker 72 - jepsen.util 432 :invoke :write [24 39] INFO [2019-12-17 12:28:00,178] jepsen worker 28 - jepsen.util 298 :ok :read [9 26] INFO [2019-12-17 12:28:00,178] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [5 40]] INFO [2019-12-17 12:28:00,199] jepsen worker 63 - jepsen.util 243 :ok :write [21 30] INFO [2019-12-17 12:28:00,200] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [1 29]] INFO [2019-12-17 12:28:00,221] jepsen worker 55 - jepsen.util 595 :info :write [18 11] :RequestCanceledException INFO [2019-12-17 12:28:00,221] jepsen worker 18 - jepsen.util 468 :info :write [6 49] :RequestCanceledException INFO [2019-12-17 12:28:00,222] jepsen worker 55 - jepsen.util 685 :invoke :read [18 nil] INFO [2019-12-17 12:28:00,222] jepsen worker 18 - jepsen.util 558 :invoke :read [6 nil] INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2364,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:00,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:00,229] jepsen worker 85 - jepsen.util 445 :info :cas [28 [49 6]] :RequestCanceledException INFO [2019-12-17 12:28:00,231] jepsen worker 85 - jepsen.util 535 :invoke :write [28 9] INFO [2019-12-17 12:28:00,233] jepsen worker 30 - jepsen.util 390 :fail :cas [10 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,234] jepsen worker 30 - jepsen.util 390 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,240] jepsen worker 45 - jepsen.util 675 :ok :read [15 35] INFO [2019-12-17 12:28:00,240] jepsen worker 45 - jepsen.util 675 :invoke :read [15 nil] INFO [2019-12-17 12:28:00,242] jepsen worker 53 - jepsen.util 593 :ok :read [17 0] INFO [2019-12-17 12:28:00,242] jepsen worker 53 - jepsen.util 593 :invoke :cas [17 [40 2]] INFO [2019-12-17 12:28:00,250] jepsen worker 83 - jepsen.util 533 :info :cas [27 [41 37]] :RequestCanceledException INFO [2019-12-17 12:28:00,252] jepsen worker 83 - jepsen.util 623 :invoke :read [27 nil] INFO [2019-12-17 12:28:00,254] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:28:00,254] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [19 25]] INFO [2019-12-17 12:28:00,254] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [1 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,254] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:00,261] jepsen worker 30 - jepsen.util 390 :ok :read [10 23] INFO [2019-12-17 12:28:00,261] jepsen worker 30 - jepsen.util 390 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,261] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2374,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2375,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:00,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2376,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:00,270] jepsen worker 45 - jepsen.util 675 :ok :read [15 35] INFO [2019-12-17 12:28:00,270] jepsen worker 45 - jepsen.util 675 :invoke :cas [15 [23 15]] INFO [2019-12-17 12:28:00,279] jepsen worker 63 - jepsen.util 243 :ok :read [21 30] INFO [2019-12-17 12:28:00,279] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:00,285] jepsen worker 86 - jepsen.util 266 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:00,285] jepsen worker 86 - jepsen.util 266 :invoke :cas [28 [48 7]] INFO [2019-12-17 12:28:00,287] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:00,287] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:28:00,291] jepsen worker 30 - jepsen.util 390 :ok :read [10 23] INFO [2019-12-17 12:28:00,291] jepsen worker 30 - jepsen.util 390 :invoke :read [10 nil] INFO [2019-12-17 12:28:00,298] jepsen worker 45 - jepsen.util 675 :fail :cas [15 [23 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,299] jepsen worker 45 - jepsen.util 675 :invoke :write [15 49] INFO [2019-12-17 12:28:00,306] jepsen worker 89 - jepsen.util 449 :info :write [29 5] :RequestCanceledException INFO [2019-12-17 12:28:00,307] jepsen worker 89 - jepsen.util 539 :invoke :cas [29 [46 27]] INFO [2019-12-17 12:28:00,312] jepsen worker 63 - jepsen.util 243 :ok :read [21 30] INFO [2019-12-17 12:28:00,312] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:00,314] jepsen worker 8 - jepsen.util 548 :ok :read [2 15] INFO [2019-12-17 12:28:00,315] jepsen worker 8 - jepsen.util 548 :invoke :cas [2 [23 13]] INFO [2019-12-17 12:28:00,327] jepsen worker 89 - jepsen.util 539 :fail :cas [29 [46 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,328] jepsen worker 89 - jepsen.util 539 :invoke :write [29 28] INFO [2019-12-17 12:28:00,328] jepsen worker 30 - jepsen.util 390 :ok :read [10 23] INFO [2019-12-17 12:28:00,328] jepsen worker 30 - jepsen.util 390 :invoke :write [10 26] INFO [2019-12-17 12:28:00,352] jepsen worker 39 - jepsen.util 489 :info :write [13 15] :RequestCanceledException INFO [2019-12-17 12:28:00,353] jepsen worker 39 - jepsen.util 579 :invoke :read [13 nil] INFO [2019-12-17 12:28:00,362] jepsen worker 88 - jepsen.util 358 :fail :cas [29 [22 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,362] jepsen worker 88 - jepsen.util 358 :invoke :read [29 nil] INFO [2019-12-17 12:28:00,391] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [3 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,391] jepsen worker 60 - jepsen.util 600 :invoke :read [20 nil] INFO [2019-12-17 12:28:00,398] jepsen worker 56 - jepsen.util 416 :ok :read [18 47] INFO [2019-12-17 12:28:00,398] jepsen worker 56 - jepsen.util 416 :invoke :write [18 44] INFO [2019-12-17 12:28:00,406] jepsen worker 43 - jepsen.util 583 :info :write [14 37] :RequestCanceledException INFO [2019-12-17 12:28:00,406] jepsen worker 54 - jepsen.util 504 :info :cas [18 [47 22]] :RequestCanceledException INFO [2019-12-17 12:28:00,407] jepsen worker 43 - jepsen.util 673 :invoke :write [14 23] INFO [2019-12-17 12:28:00,407] jepsen worker 54 - jepsen.util 594 :invoke :read [18 nil] INFO [2019-12-17 12:28:00,480] jepsen worker 5 - jepsen.util 185 :fail :cas [1 [12 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,480] jepsen worker 5 - jepsen.util 185 :invoke :write [1 4] INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":2068,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2405,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:00,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2409,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:00,505] jepsen worker 58 - jepsen.util 238 :info :cas [19 [45 39]] :RequestCanceledException INFO [2019-12-17 12:28:00,506] jepsen worker 58 - jepsen.util 328 :invoke :cas [19 [29 8]] INFO [2019-12-17 12:28:00,521] jepsen worker 57 - jepsen.util 417 :info :write [19 26] :RequestCanceledException INFO [2019-12-17 12:28:00,521] jepsen worker 26 - jepsen.util 566 :info :write [8 30] :RequestCanceledException INFO [2019-12-17 12:28:00,522] jepsen worker 57 - jepsen.util 507 :invoke :write [19 31] INFO [2019-12-17 12:28:00,522] jepsen worker 26 - jepsen.util 656 :invoke :write [8 4] INFO [2019-12-17 12:28:00,549] jepsen worker 61 - jepsen.util 601 :ok :read [20 7] INFO [2019-12-17 12:28:00,550] jepsen worker 61 - jepsen.util 601 :invoke :write [20 31] INFO [2019-12-17 12:28:00,550] jepsen worker 87 - jepsen.util 357 :fail :cas [29 [49 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,550] jepsen worker 87 - jepsen.util 357 :invoke :cas [29 [24 13]] INFO [2019-12-17 12:28:00,552] jepsen worker 9 - jepsen.util 369 :info :write [3 39] :RequestCanceledException INFO [2019-12-17 12:28:00,553] jepsen worker 9 - jepsen.util 459 :invoke :write [3 18] INFO [2019-12-17 12:28:00,572] jepsen worker 24 - jepsen.util 384 :fail :cas [8 [37 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,573] jepsen worker 24 - jepsen.util 384 :invoke :write [8 0] INFO [2019-12-17 12:28:00,588] jepsen worker 0 - jepsen.util 540 :info :cas [0 [48 1]] :RequestCanceledException INFO [2019-12-17 12:28:00,589] jepsen worker 0 - jepsen.util 630 :invoke :write [0 39] INFO [2019-12-17 12:28:00,606] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":2079,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:00,617] jepsen worker 55 - jepsen.util 685 :ok :read [18 47] INFO [2019-12-17 12:28:00,618] jepsen worker 55 - jepsen.util 685 :invoke :cas [18 [40 20]] INFO [2019-12-17 12:28:00,620] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [42 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,620] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [41 42]] INFO [2019-12-17 12:28:00,635] jepsen worker 73 - jepsen.util 523 :info :cas [24 [49 0]] :RequestCanceledException INFO [2019-12-17 12:28:00,636] jepsen worker 73 - jepsen.util 613 :invoke :read [24 nil] INFO [2019-12-17 12:28:00,687] jepsen worker 59 - jepsen.util 329 :info :cas [19 [30 37]] :RequestCanceledException INFO [2019-12-17 12:28:00,688] jepsen worker 59 - jepsen.util 419 :invoke :cas [19 [36 15]] INFO [2019-12-17 12:28:00,696] jepsen worker 55 - jepsen.util 685 :fail :cas [18 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,696] jepsen worker 55 - jepsen.util 685 :invoke :cas [18 [13 13]] INFO [2019-12-17 12:28:00,705] jepsen worker 47 - jepsen.util 587 :info :write [15 19] :RequestCanceledException INFO [2019-12-17 12:28:00,706] jepsen worker 47 - jepsen.util 677 :invoke :write [15 4] INFO [2019-12-17 12:28:00,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:00,717] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2425,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:00,717] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:00,718] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2427,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:00,721] jepsen worker 20 - jepsen.util 470 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:28:00,722] jepsen worker 20 - jepsen.util 560 :invoke :cas [6 [6 46]] INFO [2019-12-17 12:28:00,726] jepsen worker 10 - jepsen.util 370 :info :write [3 40] :SyncWriteAmbiguous INFO [2019-12-17 12:28:00,727] jepsen worker 10 - jepsen.util 460 :invoke :write [3 15] INFO [2019-12-17 12:28:00,729] jepsen worker 8 - jepsen.util 548 :fail :cas [2 [23 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,729] jepsen worker 8 - jepsen.util 548 :invoke :write [2 13] INFO [2019-12-17 12:28:00,752] jepsen worker 49 - jepsen.util 319 :info :cas [16 [17 23]] :RequestCanceledException INFO [2019-12-17 12:28:00,753] jepsen worker 49 - jepsen.util 409 :invoke :read [16 nil] INFO [2019-12-17 12:28:00,776] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.1089494163424124}, :ns_1@172.28.128.128 {:progress 0.2339622641509433}} INFO [2019-12-17 12:28:00,776] jepsen nemesis - couchbase.util Rebalance check count: 12 INFO [2019-12-17 12:28:00,776] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:28:00,805] jepsen worker 76 - jepsen.util 436 :info :write [25 38] :RequestCanceledException INFO [2019-12-17 12:28:00,806] jepsen worker 76 - jepsen.util 526 :invoke :cas [25 [16 5]] INFO [2019-12-17 12:28:00,820] jepsen worker 7 - jepsen.util 547 :info :write [2 24] :RequestCanceledException INFO [2019-12-17 12:28:00,821] jepsen worker 7 - jepsen.util 637 :invoke :cas [2 [38 46]] INFO [2019-12-17 12:28:00,836] jepsen worker 87 - jepsen.util 357 :fail :cas [29 [24 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,837] jepsen worker 87 - jepsen.util 357 :invoke :write [29 29] INFO [2019-12-17 12:28:00,849] jepsen worker 25 - jepsen.util 475 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:28:00,850] jepsen worker 25 - jepsen.util 565 :invoke :write [8 28] INFO [2019-12-17 12:28:00,867] jepsen worker 78 - jepsen.util 528 :fail :cas [26 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,867] jepsen worker 78 - jepsen.util 528 :invoke :cas [26 [27 30]] INFO [2019-12-17 12:28:00,896] jepsen worker 6 - jepsen.util 546 :fail :cas [2 [29 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,896] jepsen worker 6 - jepsen.util 546 :invoke :read [2 nil] INFO [2019-12-17 12:28:00,899] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:28:00,900] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:28:00,917] jepsen worker 7 - jepsen.util 637 :fail :cas [2 [38 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:00,918] jepsen worker 7 - jepsen.util 637 :invoke :write [2 13] INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:00,922] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2442,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:00,935] jepsen worker 65 - jepsen.util 425 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:00,935] jepsen worker 12 - jepsen.util 372 :info :write [4 9] :RequestCanceledException INFO [2019-12-17 12:28:00,935] jepsen worker 64 - jepsen.util 334 :info :cas [21 [21 43]] :RequestCanceledException INFO [2019-12-17 12:28:00,935] jepsen worker 23 - jepsen.util 293 :info :write [7 29] :RequestCanceledException INFO [2019-12-17 12:28:00,935] jepsen worker 65 - jepsen.util 425 :invoke :cas [21 [43 17]] INFO [2019-12-17 12:28:00,936] jepsen worker 23 - jepsen.util 383 :invoke :read [7 nil] INFO [2019-12-17 12:28:00,936] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [1 20]] INFO [2019-12-17 12:28:00,936] jepsen worker 12 - jepsen.util 462 :invoke :read [4 nil] INFO [2019-12-17 12:28:00,952] jepsen worker 14 - jepsen.util 464 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:28:00,953] jepsen worker 14 - jepsen.util 554 :invoke :read [4 nil] INFO [2019-12-17 12:28:01,002] jepsen worker 67 - jepsen.util 607 :info :write [22 43] :RequestCanceledException INFO [2019-12-17 12:28:01,003] jepsen worker 67 - jepsen.util 697 :invoke :cas [22 [49 30]] INFO [2019-12-17 12:28:01,020] jepsen worker 69 - jepsen.util 519 :info :write [23 48] :RequestCanceledException INFO [2019-12-17 12:28:01,020] jepsen worker 42 - jepsen.util 672 :info :write [14 36] :RequestCanceledException INFO [2019-12-17 12:28:01,021] jepsen worker 69 - jepsen.util 609 :invoke :cas [23 [26 17]] INFO [2019-12-17 12:28:01,021] jepsen worker 42 - jepsen.util 762 :invoke :write [14 41] INFO [2019-12-17 12:28:01,028] jepsen worker 79 - jepsen.util 439 :ok :read [26 47] INFO [2019-12-17 12:28:01,028] jepsen worker 79 - jepsen.util 439 :invoke :cas [26 [1 10]] INFO [2019-12-17 12:28:01,032] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":2141,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:01,035] jepsen worker 11 - jepsen.util 461 :info :write [3 26] :RequestCanceledException INFO [2019-12-17 12:28:01,036] jepsen worker 11 - jepsen.util 551 :invoke :write [3 9] INFO [2019-12-17 12:28:01,041] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:01,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2459,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:01,096] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:01,096] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:28:01,103] jepsen worker 22 - jepsen.util 472 :info :write [7 36] :RequestCanceledException INFO [2019-12-17 12:28:01,103] jepsen worker 22 - jepsen.util 562 :invoke :read [7 nil] INFO [2019-12-17 12:28:01,119] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [19 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,119] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:01,121] jepsen worker 62 - jepsen.util 512 :info :write [20 2] :RequestCanceledException INFO [2019-12-17 12:28:01,123] jepsen worker 62 - jepsen.util 602 :invoke :cas [20 [44 15]] INFO [2019-12-17 12:28:01,182] jepsen worker 54 - jepsen.util 594 :ok :read [18 47] INFO [2019-12-17 12:28:01,182] jepsen worker 77 - jepsen.util 437 :fail :cas [25 [41 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,182] jepsen worker 54 - jepsen.util 594 :invoke :read [18 nil] INFO [2019-12-17 12:28:01,182] jepsen worker 77 - jepsen.util 437 :invoke :cas [25 [2 14]] INFO [2019-12-17 12:28:01,192] jepsen worker 22 - jepsen.util 562 :ok :read [7 20] INFO [2019-12-17 12:28:01,192] jepsen worker 22 - jepsen.util 562 :invoke :read [7 nil] INFO [2019-12-17 12:28:01,204] jepsen worker 55 - jepsen.util 685 :fail :cas [18 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,204] jepsen worker 55 - jepsen.util 685 :invoke :write [18 2] INFO [2019-12-17 12:28:01,206] jepsen worker 53 - jepsen.util 593 :fail :cas [17 [40 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,206] jepsen worker 19 - jepsen.util 469 :info :write [6 34] :SyncWriteAmbiguous INFO [2019-12-17 12:28:01,206] jepsen worker 53 - jepsen.util 593 :invoke :cas [17 [34 27]] INFO [2019-12-17 12:28:01,207] jepsen worker 19 - jepsen.util 559 :invoke :write [6 27] INFO [2019-12-17 12:28:01,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:01,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2472,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:01,208] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:01,208] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2473,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:01,240] jepsen worker 76 - jepsen.util 526 :fail :cas [25 [16 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,241] jepsen worker 76 - jepsen.util 526 :invoke :cas [25 [45 16]] INFO [2019-12-17 12:28:01,244] jepsen worker 12 - jepsen.util 462 :ok :read [4 4] INFO [2019-12-17 12:28:01,244] jepsen worker 12 - jepsen.util 462 :invoke :read [4 nil] INFO [2019-12-17 12:28:01,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:01,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2479,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:01,282] jepsen worker 50 - jepsen.util 590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,283] jepsen worker 50 - jepsen.util 590 :invoke :read [16 nil] INFO [2019-12-17 12:28:01,287] jepsen worker 60 - jepsen.util 600 :ok :read [20 7] INFO [2019-12-17 12:28:01,287] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [19 39]] INFO [2019-12-17 12:28:01,290] jepsen worker 29 - jepsen.util 479 :ok :write [9 45] INFO [2019-12-17 12:28:01,290] jepsen worker 29 - jepsen.util 479 :invoke :read [9 nil] INFO [2019-12-17 12:28:01,371] jepsen worker 62 - jepsen.util 602 :fail :cas [20 [44 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,371] jepsen worker 62 - jepsen.util 602 :invoke :cas [20 [26 7]] INFO [2019-12-17 12:28:01,385] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:28:01,385] jepsen worker 71 - jepsen.util 431 :invoke :cas [23 [13 47]] INFO [2019-12-17 12:28:01,421] jepsen worker 27 - jepsen.util 207 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:28:01,421] jepsen worker 37 - jepsen.util 397 :info :write [12 16] :RequestCanceledException INFO [2019-12-17 12:28:01,421] jepsen worker 46 - jepsen.util 316 :info :write [15 24] :RequestCanceledException INFO [2019-12-17 12:28:01,422] jepsen worker 27 - jepsen.util 297 :invoke :cas [9 [27 27]] INFO [2019-12-17 12:28:01,422] jepsen worker 46 - jepsen.util 406 :invoke :write [15 2] INFO [2019-12-17 12:28:01,423] jepsen worker 37 - jepsen.util 487 :invoke :cas [12 [23 19]] INFO [2019-12-17 12:28:01,429] jepsen worker 14 - jepsen.util 554 :ok :read [4 4] INFO [2019-12-17 12:28:01,429] jepsen worker 14 - jepsen.util 554 :invoke :write [4 43] INFO [2019-12-17 12:28:01,433] jepsen worker 76 - jepsen.util 526 :fail :cas [25 [45 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,433] jepsen worker 76 - jepsen.util 526 :invoke :cas [25 [44 32]] INFO [2019-12-17 12:28:01,434] jepsen worker 44 - jepsen.util 224 :info :write [14 25] :RequestCanceledException INFO [2019-12-17 12:28:01,434] jepsen worker 17 - jepsen.util 557 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,435] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [3 14]] INFO [2019-12-17 12:28:01,436] jepsen worker 44 - jepsen.util 314 :invoke :cas [14 [0 36]] INFO [2019-12-17 12:28:01,457] jepsen worker 78 - jepsen.util 528 :fail :cas [26 [27 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,457] jepsen worker 78 - jepsen.util 528 :invoke :write [26 38] INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2497,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:01,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2500,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:01,470] jepsen worker 62 - jepsen.util 602 :fail :cas [20 [26 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,471] jepsen worker 62 - jepsen.util 602 :invoke :cas [20 [35 46]] INFO [2019-12-17 12:28:01,483] jepsen worker 79 - jepsen.util 439 :fail :cas [26 [1 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,483] jepsen worker 79 - jepsen.util 439 :invoke :write [26 30] INFO [2019-12-17 12:28:01,520] jepsen worker 2 - jepsen.util 362 :info :cas [0 [45 22]] :RequestCanceledException INFO [2019-12-17 12:28:01,521] jepsen worker 2 - jepsen.util 452 :invoke :write [0 40] INFO [2019-12-17 12:28:01,541] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:01,541] jepsen worker 80 - jepsen.util 710 :invoke :read [26 nil] INFO [2019-12-17 12:28:01,547] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:28:01,548] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:01,554] jepsen worker 41 - jepsen.util 311 :ok :read [13 9] INFO [2019-12-17 12:28:01,554] jepsen worker 41 - jepsen.util 311 :invoke :cas [13 [39 14]] INFO [2019-12-17 12:28:01,556] jepsen worker 23 - jepsen.util 383 :ok :read [7 20] INFO [2019-12-17 12:28:01,556] jepsen worker 23 - jepsen.util 383 :invoke :write [7 44] INFO [2019-12-17 12:28:01,575] jepsen worker 48 - jepsen.util 498 :fail :cas [16 [10 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,576] jepsen worker 48 - jepsen.util 498 :invoke :write [16 42] INFO [2019-12-17 12:28:01,577] jepsen worker 54 - jepsen.util 594 :ok :read [18 47] INFO [2019-12-17 12:28:01,577] jepsen worker 54 - jepsen.util 594 :invoke :cas [18 [4 2]] INFO [2019-12-17 12:28:01,577] jepsen worker 76 - jepsen.util 526 :fail :cas [25 [44 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,578] jepsen worker 76 - jepsen.util 526 :invoke :write [25 49] INFO [2019-12-17 12:28:01,579] jepsen worker 71 - jepsen.util 431 :fail :cas [23 [13 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,580] jepsen worker 71 - jepsen.util 431 :invoke :read [23 nil] INFO [2019-12-17 12:28:01,582] jepsen worker 1 - jepsen.util 631 :ok :read [0 14] INFO [2019-12-17 12:28:01,582] jepsen worker 1 - jepsen.util 631 :invoke :write [0 24] INFO [2019-12-17 12:28:01,601] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [3 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,602] jepsen worker 17 - jepsen.util 557 :invoke :write [5 13] INFO [2019-12-17 12:28:01,604] jepsen worker 74 - jepsen.util 434 :info :write [24 18] :RequestCanceledException INFO [2019-12-17 12:28:01,605] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [47 27]] INFO [2019-12-17 12:28:01,624] jepsen worker 41 - jepsen.util 311 :fail :cas [13 [39 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,625] jepsen worker 41 - jepsen.util 311 :invoke :read [13 nil] INFO [2019-12-17 12:28:01,641] jepsen worker 37 - jepsen.util 487 :fail :cas [12 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,641] jepsen worker 37 - jepsen.util 487 :invoke :read [12 nil] INFO [2019-12-17 12:28:01,664] jepsen worker 62 - jepsen.util 602 :fail :cas [20 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,665] jepsen worker 62 - jepsen.util 602 :invoke :write [20 25] INFO [2019-12-17 12:28:01,687] jepsen worker 34 - jepsen.util 484 :info :write [11 27] :RequestCanceledException INFO [2019-12-17 12:28:01,688] jepsen worker 34 - jepsen.util 574 :invoke :write [11 47] INFO [2019-12-17 12:28:01,704] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:01,704] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2527,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:01,720] jepsen worker 33 - jepsen.util 483 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:28:01,721] jepsen worker 33 - jepsen.util 573 :invoke :read [11 nil] INFO [2019-12-17 12:28:01,752] jepsen worker 84 - jepsen.util 264 :info :write [28 1] :RequestCanceledException INFO [2019-12-17 12:28:01,753] jepsen worker 84 - jepsen.util 354 :invoke :read [28 nil] INFO [2019-12-17 12:28:01,772] jepsen worker 3 - jepsen.util 453 :ok :read [1 23] INFO [2019-12-17 12:28:01,772] jepsen worker 3 - jepsen.util 453 :invoke :write [1 37] INFO [2019-12-17 12:28:01,840] jepsen worker 4 - jepsen.util 454 :info :write [1 21] :RequestCanceledException INFO [2019-12-17 12:28:01,840] jepsen worker 21 - jepsen.util 471 :info :write [7 38] :RequestCanceledException INFO [2019-12-17 12:28:01,840] jepsen worker 82 - jepsen.util 622 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,840] jepsen worker 82 - jepsen.util 622 :invoke :cas [27 [11 39]] INFO [2019-12-17 12:28:01,840] jepsen worker 15 - jepsen.util 465 :info :cas [5 [40 15]] :RequestCanceledException INFO [2019-12-17 12:28:01,840] jepsen worker 75 - jepsen.util 345 :info :write [25 25] :RequestCanceledException INFO [2019-12-17 12:28:01,841] jepsen worker 4 - jepsen.util 544 :invoke :read [1 nil] INFO [2019-12-17 12:28:01,841] jepsen worker 21 - jepsen.util 561 :invoke :write [7 27] INFO [2019-12-17 12:28:01,841] jepsen worker 15 - jepsen.util 555 :invoke :write [5 1] INFO [2019-12-17 12:28:01,841] jepsen worker 75 - jepsen.util 435 :invoke :cas [25 [6 37]] INFO [2019-12-17 12:28:01,845] jepsen worker 33 - jepsen.util 573 :ok :read [11 7] INFO [2019-12-17 12:28:01,845] jepsen worker 33 - jepsen.util 573 :invoke :cas [11 [24 49]] INFO [2019-12-17 12:28:01,848] jepsen worker 36 - jepsen.util 486 :info :write [12 0] :RequestCanceledException INFO [2019-12-17 12:28:01,848] jepsen worker 35 - jepsen.util 305 :info :write [11 38] :RequestCanceledException INFO [2019-12-17 12:28:01,848] jepsen worker 16 - jepsen.util 466 :info :write [5 37] :RequestCanceledException INFO [2019-12-17 12:28:01,848] jepsen worker 31 - jepsen.util 751 :ok :read [10 23] INFO [2019-12-17 12:28:01,848] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [49 20]] INFO [2019-12-17 12:28:01,849] jepsen worker 35 - jepsen.util 395 :invoke :read [11 nil] INFO [2019-12-17 12:28:01,849] jepsen worker 16 - jepsen.util 556 :invoke :cas [5 [32 26]] INFO [2019-12-17 12:28:01,849] jepsen worker 36 - jepsen.util 576 :invoke :write [12 22] INFO [2019-12-17 12:28:01,855] jepsen worker 18 - jepsen.util 558 :ok :read [6 45] INFO [2019-12-17 12:28:01,855] jepsen worker 18 - jepsen.util 558 :invoke :cas [6 [46 31]] INFO [2019-12-17 12:28:01,855] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [19 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,856] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [31 1]] INFO [2019-12-17 12:28:01,863] jepsen worker 73 - jepsen.util 613 :ok :read [24 35] INFO [2019-12-17 12:28:01,863] jepsen worker 73 - jepsen.util 613 :invoke :cas [24 [9 3]] INFO [2019-12-17 12:28:01,877] jepsen worker 31 - jepsen.util 751 :fail :cas [10 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,877] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [40 13]] INFO [2019-12-17 12:28:01,883] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.132295719844358}, :ns_1@172.28.128.128 {:progress 0.279245283018868}} INFO [2019-12-17 12:28:01,883] jepsen nemesis - couchbase.util Rebalance check count: 13 INFO [2019-12-17 12:28:01,883] jepsen nemesis - couchbase.util Rebalance stuck count: 0 INFO [2019-12-17 12:28:01,891] jepsen worker 18 - jepsen.util 558 :fail :cas [6 [46 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,892] jepsen worker 18 - jepsen.util 558 :invoke :cas [6 [7 47]] INFO [2019-12-17 12:28:01,903] jepsen worker 73 - jepsen.util 613 :fail :cas [24 [9 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,904] jepsen worker 73 - jepsen.util 613 :invoke :write [24 1] INFO [2019-12-17 12:28:01,917] jepsen worker 31 - jepsen.util 751 :fail :cas [10 [40 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,917] jepsen worker 31 - jepsen.util 751 :invoke :read [10 nil] INFO [2019-12-17 12:28:01,918] jepsen worker 32 - jepsen.util 482 :info :write [10 45] :SyncWriteAmbiguous INFO [2019-12-17 12:28:01,919] jepsen worker 32 - jepsen.util 572 :invoke :write [10 1] INFO [2019-12-17 12:28:01,946] jepsen worker 18 - jepsen.util 558 :fail :cas [6 [7 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:01,946] jepsen worker 18 - jepsen.util 558 :invoke :write [6 23] INFO [2019-12-17 12:28:01,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:01,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2546,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:01,951] jepsen worker 68 - jepsen.util 338 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,951] jepsen worker 66 - jepsen.util 426 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:01,951] jepsen worker 66 - jepsen.util 426 :invoke :cas [22 [19 33]] INFO [2019-12-17 12:28:01,951] jepsen worker 68 - jepsen.util 338 :invoke :cas [22 [4 42]] INFO [2019-12-17 12:28:01,973] jepsen worker 88 - jepsen.util 358 :ok :read [29 5] INFO [2019-12-17 12:28:01,973] jepsen worker 88 - jepsen.util 358 :invoke :write [29 37] INFO [2019-12-17 12:28:01,985] jepsen worker 51 - jepsen.util 321 :info :write [17 30] :RequestCanceledException INFO [2019-12-17 12:28:01,987] jepsen worker 51 - jepsen.util 411 :invoke :write [17 15] INFO [2019-12-17 12:28:02,002] jepsen worker 52 - jepsen.util 412 :info :write [17 0] :RequestCanceledException INFO [2019-12-17 12:28:02,003] jepsen worker 52 - jepsen.util 502 :invoke :read [17 nil] INFO [2019-12-17 12:28:02,017] jepsen worker 31 - jepsen.util 751 :ok :read [10 23] INFO [2019-12-17 12:28:02,017] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [1 5]] INFO [2019-12-17 12:28:02,024] jepsen worker 72 - jepsen.util 432 :info :write [24 39] :SyncWriteAmbiguous INFO [2019-12-17 12:28:02,025] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [11 29]] INFO [2019-12-17 12:28:02,031] jepsen worker 12 - jepsen.util 462 :ok :read [4 4] INFO [2019-12-17 12:28:02,031] jepsen worker 12 - jepsen.util 462 :invoke :write [4 41] INFO [2019-12-17 12:28:02,035] jepsen worker 38 - jepsen.util 308 :info :write [12 20] :RequestCanceledException INFO [2019-12-17 12:28:02,036] jepsen worker 38 - jepsen.util 398 :invoke :write [12 37] INFO [2019-12-17 12:28:02,043] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [5 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,043] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:28:02,052] jepsen worker 40 - jepsen.util 580 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:28:02,053] jepsen worker 40 - jepsen.util 670 :invoke :cas [13 [36 40]] INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2566,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:02,059] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2568,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:02,078] jepsen worker 33 - jepsen.util 573 :fail :cas [11 [24 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,079] jepsen worker 33 - jepsen.util 573 :invoke :cas [11 [23 33]] INFO [2019-12-17 12:28:02,080] jepsen worker 54 - jepsen.util 594 :fail :cas [18 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,081] jepsen worker 54 - jepsen.util 594 :invoke :write [18 44] INFO [2019-12-17 12:28:02,084] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [1 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,084] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [5 24]] INFO [2019-12-17 12:28:02,105] jepsen worker 13 - jepsen.util 463 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:28:02,106] jepsen worker 13 - jepsen.util 553 :invoke :write [4 28] INFO [2019-12-17 12:28:02,117] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:28:02,117] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:02,130] jepsen worker 35 - jepsen.util 395 :ok :read [11 7] INFO [2019-12-17 12:28:02,131] jepsen worker 35 - jepsen.util 395 :invoke :write [11 5] INFO [2019-12-17 12:28:02,146] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [5 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,147] jepsen worker 64 - jepsen.util 424 :invoke :write [21 17] INFO [2019-12-17 12:28:02,147] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [31 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,147] jepsen worker 60 - jepsen.util 600 :invoke :read [20 nil] INFO [2019-12-17 12:28:02,161] jepsen worker 4 - jepsen.util 544 :ok :read [1 23] INFO [2019-12-17 12:28:02,162] jepsen worker 4 - jepsen.util 544 :invoke :cas [1 [35 15]] INFO [2019-12-17 12:28:02,174] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:02,174] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:02,187] jepsen worker 37 - jepsen.util 487 :ok :read [12 12] INFO [2019-12-17 12:28:02,188] jepsen worker 37 - jepsen.util 487 :invoke :write [12 11] INFO [2019-12-17 12:28:02,205] jepsen worker 81 - jepsen.util 441 :info :cas [27 [6 17]] :RequestCanceledException INFO [2019-12-17 12:28:02,206] jepsen worker 81 - jepsen.util 531 :invoke :cas [27 [47 8]] INFO [2019-12-17 12:28:02,212] jepsen worker 64 - jepsen.util 424 :ok :write [21 17] INFO [2019-12-17 12:28:02,213] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,214] jepsen worker 71 - jepsen.util 431 :ok :read [23 29] INFO [2019-12-17 12:28:02,214] jepsen worker 71 - jepsen.util 431 :invoke :write [23 17] INFO [2019-12-17 12:28:02,221] jepsen worker 70 - jepsen.util 610 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:28:02,222] jepsen worker 70 - jepsen.util 700 :invoke :write [23 39] INFO [2019-12-17 12:28:02,235] jepsen worker 64 - jepsen.util 424 :ok :read [21 17] INFO [2019-12-17 12:28:02,235] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [27 25]] INFO [2019-12-17 12:28:02,251] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [27 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,251] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [15 37]] INFO [2019-12-17 12:28:02,261] jepsen worker 75 - jepsen.util 435 :fail :cas [25 [6 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,261] jepsen worker 75 - jepsen.util 435 :invoke :cas [25 [4 45]] INFO [2019-12-17 12:28:02,265] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [15 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,265] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2595,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:02,278] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2599,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:02,282] jepsen worker 64 - jepsen.util 424 :ok :read [21 17] INFO [2019-12-17 12:28:02,282] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2587,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:02,296] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2589,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:02,297] jepsen worker 64 - jepsen.util 424 :ok :read [21 17] INFO [2019-12-17 12:28:02,297] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [3 25]] INFO [2019-12-17 12:28:02,304] jepsen worker 85 - jepsen.util 535 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:28:02,305] jepsen worker 85 - jepsen.util 625 :invoke :cas [28 [33 19]] INFO [2019-12-17 12:28:02,316] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [3 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,316] jepsen worker 64 - jepsen.util 424 :invoke :write [21 41] INFO [2019-12-17 12:28:02,320] jepsen worker 83 - jepsen.util 623 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,320] jepsen worker 45 - jepsen.util 675 :info :write [15 49] :RequestCanceledException INFO [2019-12-17 12:28:02,321] jepsen worker 83 - jepsen.util 623 :invoke :write [27 23] INFO [2019-12-17 12:28:02,322] jepsen worker 45 - jepsen.util 765 :invoke :read [15 nil] INFO [2019-12-17 12:28:02,384] jepsen worker 64 - jepsen.util 424 :ok :write [21 41] INFO [2019-12-17 12:28:02,384] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [21 34]] INFO [2019-12-17 12:28:02,387] jepsen worker 86 - jepsen.util 266 :info :cas [28 [48 7]] :RequestCanceledException INFO [2019-12-17 12:28:02,388] jepsen worker 86 - jepsen.util 356 :invoke :cas [28 [28 32]] INFO [2019-12-17 12:28:02,398] jepsen worker 45 - jepsen.util 765 :ok :read [15 35] INFO [2019-12-17 12:28:02,399] jepsen worker 45 - jepsen.util 765 :invoke :write [15 10] INFO [2019-12-17 12:28:02,410] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [21 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,410] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,420] jepsen worker 30 - jepsen.util 390 :info :write [10 26] :RequestCanceledException INFO [2019-12-17 12:28:02,420] jepsen worker 63 - jepsen.util 243 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,420] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,421] jepsen worker 30 - jepsen.util 480 :invoke :write [10 38] INFO [2019-12-17 12:28:02,435] jepsen worker 89 - jepsen.util 539 :info :write [29 28] :RequestCanceledException INFO [2019-12-17 12:28:02,437] jepsen worker 89 - jepsen.util 629 :invoke :write [29 39] INFO [2019-12-17 12:28:02,442] jepsen worker 63 - jepsen.util 243 :ok :read [21 41] INFO [2019-12-17 12:28:02,443] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [39 26]] INFO [2019-12-17 12:28:02,451] jepsen worker 39 - jepsen.util 579 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,452] jepsen worker 39 - jepsen.util 579 :invoke :write [13 40] INFO [2019-12-17 12:28:02,454] jepsen worker 48 - jepsen.util 498 :ok :write [16 42] INFO [2019-12-17 12:28:02,454] jepsen worker 48 - jepsen.util 498 :invoke :write [16 27] INFO [2019-12-17 12:28:02,457] jepsen worker 33 - jepsen.util 573 :fail :cas [11 [23 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,457] jepsen worker 33 - jepsen.util 573 :invoke :write [11 30] INFO [2019-12-17 12:28:02,459] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [39 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,459] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,477] jepsen worker 41 - jepsen.util 311 :ok :read [13 9] INFO [2019-12-17 12:28:02,477] jepsen worker 41 - jepsen.util 311 :invoke :write [13 36] INFO [2019-12-17 12:28:02,479] jepsen worker 63 - jepsen.util 243 :ok :read [21 41] INFO [2019-12-17 12:28:02,479] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [10 32]] INFO [2019-12-17 12:28:02,497] jepsen worker 80 - jepsen.util 710 :ok :read [26 47] INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2621,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:02,497] jepsen worker 80 - jepsen.util 710 :invoke :write [26 18] INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:02,497] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2625,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:02,499] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [10 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,499] jepsen worker 63 - jepsen.util 243 :invoke :cas [21 [28 29]] INFO [2019-12-17 12:28:02,499] jepsen worker 56 - jepsen.util 416 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:28:02,501] jepsen worker 56 - jepsen.util 506 :invoke :cas [18 [48 22]] INFO [2019-12-17 12:28:02,520] jepsen worker 43 - jepsen.util 673 :info :write [14 23] :RequestCanceledException INFO [2019-12-17 12:28:02,521] jepsen worker 43 - jepsen.util 763 :invoke :read [14 nil] INFO [2019-12-17 12:28:02,522] jepsen worker 63 - jepsen.util 243 :fail :cas [21 [28 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,523] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,544] jepsen worker 63 - jepsen.util 243 :ok :read [21 41] INFO [2019-12-17 12:28:02,545] jepsen worker 63 - jepsen.util 243 :invoke :write [21 9] INFO [2019-12-17 12:28:02,561] jepsen worker 75 - jepsen.util 435 :fail :cas [25 [4 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,562] jepsen worker 75 - jepsen.util 435 :invoke :write [25 17] INFO [2019-12-17 12:28:02,573] jepsen worker 20 - jepsen.util 560 :fail :cas [6 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,573] jepsen worker 20 - jepsen.util 560 :invoke :read [6 nil] INFO [2019-12-17 12:28:02,583] jepsen worker 5 - jepsen.util 185 :info :write [1 4] :RequestCanceledException INFO [2019-12-17 12:28:02,584] jepsen worker 5 - jepsen.util 275 :invoke :read [1 nil] INFO [2019-12-17 12:28:02,596] jepsen worker 65 - jepsen.util 425 :fail :cas [21 [43 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,596] jepsen worker 65 - jepsen.util 425 :invoke :write [21 6] INFO [2019-12-17 12:28:02,604] jepsen worker 58 - jepsen.util 328 :info :cas [19 [29 8]] :RequestCanceledException INFO [2019-12-17 12:28:02,605] jepsen worker 58 - jepsen.util 418 :invoke :cas [19 [2 2]] INFO [2019-12-17 12:28:02,611] jepsen worker 1 - jepsen.util 631 :ok :write [0 24] INFO [2019-12-17 12:28:02,611] jepsen worker 1 - jepsen.util 631 :invoke :cas [0 [44 10]] INFO [2019-12-17 12:28:02,620] jepsen worker 61 - jepsen.util 601 :info :write [20 31] :RequestCanceledException INFO [2019-12-17 12:28:02,620] jepsen worker 57 - jepsen.util 507 :info :write [19 31] :RequestCanceledException INFO [2019-12-17 12:28:02,621] jepsen worker 0 - jepsen.util 630 :info :write [0 39] :RequestCanceledException INFO [2019-12-17 12:28:02,622] jepsen worker 61 - jepsen.util 691 :invoke :read [20 nil] INFO [2019-12-17 12:28:02,622] jepsen worker 57 - jepsen.util 597 :invoke :cas [19 [26 10]] INFO [2019-12-17 12:28:02,622] jepsen worker 0 - jepsen.util 720 :invoke :read [0 nil] INFO [2019-12-17 12:28:02,628] jepsen worker 6 - jepsen.util 546 :ok :read [2 15] INFO [2019-12-17 12:28:02,628] jepsen worker 6 - jepsen.util 546 :invoke :read [2 nil] INFO [2019-12-17 12:28:02,635] jepsen worker 26 - jepsen.util 656 :info :write [8 4] :RequestCanceledException INFO [2019-12-17 12:28:02,636] jepsen worker 26 - jepsen.util 746 :invoke :read [8 nil] INFO [2019-12-17 12:28:02,652] jepsen worker 24 - jepsen.util 384 :info :write [8 0] :RequestCanceledException INFO [2019-12-17 12:28:02,652] jepsen worker 9 - jepsen.util 459 :info :write [3 18] :RequestCanceledException INFO [2019-12-17 12:28:02,653] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [12 45]] INFO [2019-12-17 12:28:02,653] jepsen worker 9 - jepsen.util 549 :invoke :read [3 nil] INFO [2019-12-17 12:28:02,677] jepsen worker 3 - jepsen.util 453 :ok :write [1 37] INFO [2019-12-17 12:28:02,677] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [20 36]] INFO [2019-12-17 12:28:02,686] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [11 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,686] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [43 10]] INFO [2019-12-17 12:28:02,700] jepsen worker 20 - jepsen.util 560 :ok :read [6 45] INFO [2019-12-17 12:28:02,700] jepsen worker 20 - jepsen.util 560 :invoke :read [6 nil] INFO [2019-12-17 12:28:02,705] jepsen worker 59 - jepsen.util 419 :info :cas [19 [36 15]] :RequestCanceledException INFO [2019-12-17 12:28:02,707] jepsen worker 59 - jepsen.util 509 :invoke :cas [19 [26 27]] INFO [2019-12-17 12:28:02,712] jepsen worker 53 - jepsen.util 593 :fail :cas [17 [34 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,712] jepsen worker 53 - jepsen.util 593 :invoke :read [17 nil] INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2652,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:02,716] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2656,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:02,717] jepsen worker 4 - jepsen.util 544 :fail :cas [1 [35 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,717] jepsen worker 4 - jepsen.util 544 :invoke :read [1 nil] INFO [2019-12-17 12:28:02,721] jepsen worker 47 - jepsen.util 677 :info :write [15 4] :RequestCanceledException INFO [2019-12-17 12:28:02,722] jepsen worker 47 - jepsen.util 767 :invoke :read [15 nil] INFO [2019-12-17 12:28:02,727] jepsen worker 63 - jepsen.util 243 :ok :write [21 9] INFO [2019-12-17 12:28:02,728] jepsen worker 63 - jepsen.util 243 :invoke :write [21 35] INFO [2019-12-17 12:28:02,745] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [43 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,745] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [20 7]] INFO [2019-12-17 12:28:02,752] jepsen worker 10 - jepsen.util 460 :info :write [3 15] :RequestCanceledException INFO [2019-12-17 12:28:02,753] jepsen worker 10 - jepsen.util 550 :invoke :cas [3 [42 23]] INFO [2019-12-17 12:28:02,773] jepsen worker 52 - jepsen.util 502 :ok :read [17 0] INFO [2019-12-17 12:28:02,773] jepsen worker 52 - jepsen.util 502 :invoke :write [17 34] INFO [2019-12-17 12:28:02,791] jepsen worker 20 - jepsen.util 560 :ok :read [6 45] INFO [2019-12-17 12:28:02,791] jepsen worker 20 - jepsen.util 560 :invoke :cas [6 [0 48]] INFO [2019-12-17 12:28:02,796] jepsen worker 9 - jepsen.util 549 :ok :read [3 14] INFO [2019-12-17 12:28:02,796] jepsen worker 9 - jepsen.util 549 :invoke :cas [3 [37 3]] INFO [2019-12-17 12:28:02,821] jepsen worker 49 - jepsen.util 409 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:02,821] jepsen worker 8 - jepsen.util 548 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:28:02,821] jepsen worker 49 - jepsen.util 409 :invoke :read [16 nil] INFO [2019-12-17 12:28:02,822] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [49 7]] INFO [2019-12-17 12:28:02,824] jepsen worker 60 - jepsen.util 600 :ok :read [20 7] INFO [2019-12-17 12:28:02,825] jepsen worker 60 - jepsen.util 600 :invoke :cas [20 [32 31]] INFO [2019-12-17 12:28:02,827] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [20 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,827] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [20 14]] INFO [2019-12-17 12:28:02,843] jepsen worker 20 - jepsen.util 560 :fail :cas [6 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,843] jepsen worker 20 - jepsen.util 560 :invoke :write [6 25] INFO [2019-12-17 12:28:02,867] jepsen worker 72 - jepsen.util 522 :fail :cas [24 [20 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,867] jepsen worker 72 - jepsen.util 522 :invoke :cas [24 [17 17]] INFO [2019-12-17 12:28:02,902] jepsen worker 63 - jepsen.util 243 :ok :write [21 35] INFO [2019-12-17 12:28:02,902] jepsen worker 63 - jepsen.util 243 :invoke :read [21 nil] INFO [2019-12-17 12:28:02,908] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.05859375}, :ns_1@172.28.128.126 {:progress 0.0539568345323741}, :ns_1@172.28.128.127 {:progress 0.132295719844358}, :ns_1@172.28.128.128 {:progress 0.279245283018868}} INFO [2019-12-17 12:28:02,908] jepsen nemesis - couchbase.util Rebalance check count: 14 INFO [2019-12-17 12:28:02,908] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:02,917] jepsen worker 9 - jepsen.util 549 :fail :cas [3 [37 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,918] jepsen worker 9 - jepsen.util 549 :invoke :write [3 23] INFO [2019-12-17 12:28:02,921] jepsen worker 25 - jepsen.util 565 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:28:02,921] jepsen worker 87 - jepsen.util 357 :info :write [29 29] :RequestCanceledException INFO [2019-12-17 12:28:02,923] jepsen worker 25 - jepsen.util 655 :invoke :write [8 18] INFO [2019-12-17 12:28:02,923] jepsen worker 87 - jepsen.util 447 :invoke :cas [29 [29 15]] INFO [2019-12-17 12:28:02,926] jepsen worker 63 - jepsen.util 243 :ok :read [21 35] INFO [2019-12-17 12:28:02,926] jepsen worker 63 - jepsen.util 243 :invoke :write [21 38] INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2681,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:02,934] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:02,941] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [12 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,941] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [19 29]] INFO [2019-12-17 12:28:02,955] jepsen worker 17 - jepsen.util 557 :ok :write [5 13] INFO [2019-12-17 12:28:02,955] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [38 45]] INFO [2019-12-17 12:28:02,956] jepsen worker 8 - jepsen.util 638 :fail :cas [2 [49 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:02,957] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [33 38]] INFO [2019-12-17 12:28:03,001] jepsen worker 60 - jepsen.util 600 :fail :cas [20 [32 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,001] jepsen worker 60 - jepsen.util 600 :invoke :write [20 18] INFO [2019-12-17 12:28:03,005] jepsen worker 7 - jepsen.util 637 :info :write [2 13] :RequestCanceledException INFO [2019-12-17 12:28:03,007] jepsen worker 7 - jepsen.util 727 :invoke :cas [2 [16 38]] INFO [2019-12-17 12:28:03,042] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:03,042] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2695,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2698,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:03,043] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2699,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:03,048] jepsen worker 11 - jepsen.util 551 :info :write [3 9] :RequestCanceledException INFO [2019-12-17 12:28:03,049] jepsen worker 11 - jepsen.util 641 :invoke :cas [3 [19 27]] INFO [2019-12-17 12:28:03,073] jepsen worker 53 - jepsen.util 593 :ok :read [17 0] INFO [2019-12-17 12:28:03,073] jepsen worker 53 - jepsen.util 593 :invoke :write [17 43] INFO [2019-12-17 12:28:03,084] jepsen worker 4 - jepsen.util 544 :ok :read [1 37] INFO [2019-12-17 12:28:03,084] jepsen worker 4 - jepsen.util 544 :invoke :cas [1 [6 6]] INFO [2019-12-17 12:28:03,105] jepsen worker 67 - jepsen.util 697 :info :cas [22 [49 30]] :RequestCanceledException INFO [2019-12-17 12:28:03,106] jepsen worker 67 - jepsen.util 787 :invoke :cas [22 [12 1]] INFO [2019-12-17 12:28:03,121] jepsen worker 19 - jepsen.util 559 :info :write [6 27] :SyncWriteAmbiguous INFO [2019-12-17 12:28:03,121] jepsen worker 69 - jepsen.util 609 :info :cas [23 [26 17]] :RequestCanceledException INFO [2019-12-17 12:28:03,122] jepsen worker 19 - jepsen.util 649 :invoke :read [6 nil] INFO [2019-12-17 12:28:03,122] jepsen worker 69 - jepsen.util 699 :invoke :cas [23 [12 8]] INFO [2019-12-17 12:28:03,135] jepsen worker 42 - jepsen.util 762 :info :write [14 41] :RequestCanceledException INFO [2019-12-17 12:28:03,136] jepsen worker 42 - jepsen.util 852 :invoke :read [14 nil] INFO [2019-12-17 12:28:03,154] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [47 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,155] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [8 44]] INFO [2019-12-17 12:28:03,170] jepsen worker 19 - jepsen.util 649 :ok :read [6 45] INFO [2019-12-17 12:28:03,170] jepsen worker 19 - jepsen.util 649 :invoke :cas [6 [43 1]] INFO [2019-12-17 12:28:03,173] jepsen worker 47 - jepsen.util 767 :ok :read [15 35] INFO [2019-12-17 12:28:03,173] jepsen worker 47 - jepsen.util 767 :invoke :write [15 20] INFO [2019-12-17 12:28:03,188] jepsen worker 11 - jepsen.util 641 :fail :cas [3 [19 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,189] jepsen worker 11 - jepsen.util 641 :invoke :write [3 27] INFO [2019-12-17 12:28:03,203] jepsen worker 43 - jepsen.util 763 :ok :read [14 27] INFO [2019-12-17 12:28:03,203] jepsen worker 43 - jepsen.util 763 :invoke :cas [14 [4 0]] INFO [2019-12-17 12:28:03,214] jepsen worker 19 - jepsen.util 649 :fail :cas [6 [43 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,214] jepsen worker 19 - jepsen.util 649 :invoke :write [6 49] INFO [2019-12-17 12:28:03,217] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [8 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,217] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [14 5]] INFO [2019-12-17 12:28:03,218] jepsen worker 22 - jepsen.util 562 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,218] jepsen worker 55 - jepsen.util 685 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:28:03,219] jepsen worker 22 - jepsen.util 562 :invoke :cas [7 [4 39]] INFO [2019-12-17 12:28:03,220] jepsen worker 55 - jepsen.util 775 :invoke :write [18 7] INFO [2019-12-17 12:28:03,228] jepsen worker 26 - jepsen.util 746 :ok :read [8 14] INFO [2019-12-17 12:28:03,228] jepsen worker 26 - jepsen.util 746 :invoke :cas [8 [46 11]] INFO [2019-12-17 12:28:03,244] jepsen worker 87 - jepsen.util 447 :fail :cas [29 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,244] jepsen worker 87 - jepsen.util 447 :invoke :read [29 nil] INFO [2019-12-17 12:28:03,262] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:03,262] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2724,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:03,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2725,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:03,264] jepsen worker 69 - jepsen.util 699 :fail :cas [23 [12 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,264] jepsen worker 69 - jepsen.util 699 :invoke :read [23 nil] INFO [2019-12-17 12:28:03,270] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [14 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,270] jepsen worker 74 - jepsen.util 524 :invoke :read [24 nil] INFO [2019-12-17 12:28:03,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:03,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2712,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:03,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:03,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:03,296] jepsen worker 74 - jepsen.util 524 :ok :read [24 35] INFO [2019-12-17 12:28:03,296] jepsen worker 74 - jepsen.util 524 :invoke :write [24 39] INFO [2019-12-17 12:28:03,296] jepsen worker 61 - jepsen.util 691 :ok :read [20 7] INFO [2019-12-17 12:28:03,296] jepsen worker 61 - jepsen.util 691 :invoke :read [20 nil] INFO [2019-12-17 12:28:03,305] jepsen worker 29 - jepsen.util 479 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,305] jepsen worker 29 - jepsen.util 479 :invoke :write [9 2] INFO [2019-12-17 12:28:03,307] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [19 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,307] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [49 31]] INFO [2019-12-17 12:28:03,372] jepsen worker 56 - jepsen.util 506 :fail :cas [18 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,372] jepsen worker 56 - jepsen.util 506 :invoke :cas [18 [35 16]] INFO [2019-12-17 12:28:03,387] jepsen worker 50 - jepsen.util 590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,387] jepsen worker 50 - jepsen.util 590 :invoke :read [16 nil] INFO [2019-12-17 12:28:03,391] jepsen worker 87 - jepsen.util 447 :ok :read [29 5] INFO [2019-12-17 12:28:03,391] jepsen worker 87 - jepsen.util 447 :invoke :write [29 17] INFO [2019-12-17 12:28:03,428] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [49 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,429] jepsen worker 24 - jepsen.util 474 :invoke :cas [8 [42 30]] INFO [2019-12-17 12:28:03,434] jepsen worker 46 - jepsen.util 406 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:28:03,435] jepsen worker 46 - jepsen.util 496 :invoke :cas [15 [43 40]] INFO [2019-12-17 12:28:03,436] jepsen worker 56 - jepsen.util 506 :fail :cas [18 [35 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,436] jepsen worker 56 - jepsen.util 506 :invoke :write [18 31] INFO [2019-12-17 12:28:03,482] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:03,483] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2745,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:03,487] jepsen worker 78 - jepsen.util 528 :info :write [26 38] :RequestCanceledException INFO [2019-12-17 12:28:03,487] jepsen worker 14 - jepsen.util 554 :info :write [4 43] :RequestCanceledException INFO [2019-12-17 12:28:03,488] jepsen worker 14 - jepsen.util 644 :invoke :write [4 34] INFO [2019-12-17 12:28:03,488] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,503] jepsen worker 5 - jepsen.util 275 :ok :read [1 37] INFO [2019-12-17 12:28:03,503] jepsen worker 5 - jepsen.util 275 :invoke :write [1 16] INFO [2019-12-17 12:28:03,519] jepsen worker 4 - jepsen.util 544 :fail :cas [1 [6 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,519] jepsen worker 4 - jepsen.util 544 :invoke :write [1 18] INFO [2019-12-17 12:28:03,521] jepsen worker 27 - jepsen.util 297 :info :cas [9 [27 27]] :RequestCanceledException INFO [2019-12-17 12:28:03,521] jepsen worker 44 - jepsen.util 314 :info :cas [14 [0 36]] :RequestCanceledException INFO [2019-12-17 12:28:03,521] jepsen worker 44 - jepsen.util 404 :invoke :cas [14 [34 10]] INFO [2019-12-17 12:28:03,522] jepsen worker 27 - jepsen.util 387 :invoke :write [9 32] INFO [2019-12-17 12:28:03,549] jepsen worker 6 - jepsen.util 546 :ok :read [2 15] INFO [2019-12-17 12:28:03,549] jepsen worker 6 - jepsen.util 546 :invoke :read [2 nil] INFO [2019-12-17 12:28:03,551] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,552] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,575] jepsen worker 24 - jepsen.util 474 :fail :cas [8 [42 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,575] jepsen worker 24 - jepsen.util 474 :invoke :write [8 9] INFO [2019-12-17 12:28:03,584] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [20 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,585] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [14 19]] INFO [2019-12-17 12:28:03,587] jepsen worker 2 - jepsen.util 452 :info :write [0 40] :RequestCanceledException INFO [2019-12-17 12:28:03,587] jepsen worker 79 - jepsen.util 439 :info :write [26 30] :RequestCanceledException INFO [2019-12-17 12:28:03,588] jepsen worker 2 - jepsen.util 542 :invoke :write [0 34] INFO [2019-12-17 12:28:03,588] jepsen worker 79 - jepsen.util 529 :invoke :write [26 1] INFO [2019-12-17 12:28:03,597] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,597] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,630] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,630] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,634] jepsen worker 23 - jepsen.util 383 :info :write [7 44] :RequestCanceledException INFO [2019-12-17 12:28:03,635] jepsen worker 23 - jepsen.util 473 :invoke :read [7 nil] INFO [2019-12-17 12:28:03,656] jepsen worker 42 - jepsen.util 852 :ok :read [14 27] INFO [2019-12-17 12:28:03,656] jepsen worker 42 - jepsen.util 852 :invoke :cas [14 [17 43]] INFO [2019-12-17 12:28:03,672] jepsen worker 19 - jepsen.util 649 :ok :write [6 49] INFO [2019-12-17 12:28:03,672] jepsen worker 19 - jepsen.util 649 :invoke :write [6 39] INFO [2019-12-17 12:28:03,676] jepsen worker 32 - jepsen.util 572 :ok :write [10 1] INFO [2019-12-17 12:28:03,676] jepsen worker 32 - jepsen.util 572 :invoke :read [10 nil] INFO [2019-12-17 12:28:03,678] jepsen worker 22 - jepsen.util 562 :fail :cas [7 [4 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,678] jepsen worker 22 - jepsen.util 562 :invoke :write [7 19] INFO [2019-12-17 12:28:03,684] jepsen worker 44 - jepsen.util 404 :fail :cas [14 [34 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,684] jepsen worker 44 - jepsen.util 404 :invoke :write [14 43] INFO [2019-12-17 12:28:03,686] jepsen worker 62 - jepsen.util 602 :info :write [20 25] :RequestCanceledException INFO [2019-12-17 12:28:03,686] jepsen worker 76 - jepsen.util 526 :info :write [25 49] :RequestCanceledException INFO [2019-12-17 12:28:03,688] jepsen worker 76 - jepsen.util 616 :invoke :write [25 17] INFO [2019-12-17 12:28:03,688] jepsen worker 62 - jepsen.util 692 :invoke :write [20 34] INFO [2019-12-17 12:28:03,688] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,689] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,694] jepsen worker 48 - jepsen.util 498 :ok :write [16 27] INFO [2019-12-17 12:28:03,694] jepsen worker 48 - jepsen.util 498 :invoke :write [16 12] INFO [2019-12-17 12:28:03,703] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,703] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [18 10]] INFO [2019-12-17 12:28:03,710] jepsen worker 1 - jepsen.util 631 :fail :cas [0 [44 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,711] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:03,721] jepsen worker 34 - jepsen.util 574 :info :write [11 47] :RequestCanceledException INFO [2019-12-17 12:28:03,722] jepsen worker 34 - jepsen.util 664 :invoke :cas [11 [40 49]] INFO [2019-12-17 12:28:03,763] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,763] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,791] jepsen worker 10 - jepsen.util 550 :fail :cas [3 [42 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,792] jepsen worker 10 - jepsen.util 550 :invoke :read [3 nil] INFO [2019-12-17 12:28:03,792] jepsen worker 69 - jepsen.util 699 :ok :read [23 29] INFO [2019-12-17 12:28:03,792] jepsen worker 69 - jepsen.util 699 :invoke :cas [23 [12 11]] INFO [2019-12-17 12:28:03,803] jepsen worker 10 - jepsen.util 550 :ok :read [3 14] INFO [2019-12-17 12:28:03,803] jepsen worker 10 - jepsen.util 550 :invoke :write [3 44] INFO [2019-12-17 12:28:03,852] jepsen worker 84 - jepsen.util 354 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:03,853] jepsen worker 84 - jepsen.util 354 :invoke :write [28 22] INFO [2019-12-17 12:28:03,884] jepsen worker 21 - jepsen.util 561 :info :write [7 27] :RequestCanceledException INFO [2019-12-17 12:28:03,884] jepsen worker 7 - jepsen.util 727 :fail :cas [2 [16 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,885] jepsen worker 7 - jepsen.util 727 :invoke :write [2 30] INFO [2019-12-17 12:28:03,885] jepsen worker 21 - jepsen.util 651 :invoke :write [7 4] INFO [2019-12-17 12:28:03,905] jepsen worker 15 - jepsen.util 555 :info :write [5 1] :RequestCanceledException INFO [2019-12-17 12:28:03,906] jepsen worker 15 - jepsen.util 645 :invoke :read [5 nil] INFO [2019-12-17 12:28:03,909] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:03,909] jepsen worker 78 - jepsen.util 618 :invoke :cas [26 [14 42]] INFO [2019-12-17 12:28:03,910] jepsen worker 8 - jepsen.util 638 :fail :cas [2 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,910] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [2 43]] INFO [2019-12-17 12:28:03,918] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2791,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2792,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:03,919] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2793,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:03,919] jepsen worker 82 - jepsen.util 622 :info :cas [27 [11 39]] :RequestCanceledException INFO [2019-12-17 12:28:03,919] jepsen worker 36 - jepsen.util 576 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:28:03,920] jepsen worker 82 - jepsen.util 712 :invoke :cas [27 [20 13]] INFO [2019-12-17 12:28:03,920] jepsen worker 36 - jepsen.util 666 :invoke :write [12 39] INFO [2019-12-17 12:28:03,926] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0703125}, :ns_1@172.28.128.126 {:progress 0.06474820143884896}, :ns_1@172.28.128.127 {:progress 0.132295719844358}, :ns_1@172.28.128.128 {:progress 0.309433962264151}} INFO [2019-12-17 12:28:03,926] jepsen nemesis - couchbase.util Rebalance check count: 15 INFO [2019-12-17 12:28:03,926] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:03,952] jepsen worker 16 - jepsen.util 556 :info :cas [5 [32 26]] :RequestCanceledException INFO [2019-12-17 12:28:03,953] jepsen worker 16 - jepsen.util 646 :invoke :write [5 17] INFO [2019-12-17 12:28:03,972] jepsen worker 78 - jepsen.util 618 :fail :cas [26 [14 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:03,972] jepsen worker 78 - jepsen.util 618 :invoke :read [26 nil] INFO [2019-12-17 12:28:03,987] jepsen worker 73 - jepsen.util 613 :info :write [24 1] :RequestCanceledException INFO [2019-12-17 12:28:03,987] jepsen worker 18 - jepsen.util 558 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:28:03,988] jepsen worker 73 - jepsen.util 703 :invoke :read [24 nil] INFO [2019-12-17 12:28:03,988] jepsen worker 18 - jepsen.util 648 :invoke :cas [6 [8 34]] INFO [2019-12-17 12:28:04,004] jepsen worker 51 - jepsen.util 411 :info :write [17 15] :RequestCanceledException INFO [2019-12-17 12:28:04,005] jepsen worker 51 - jepsen.util 501 :invoke :cas [17 [11 43]] INFO [2019-12-17 12:28:04,021] jepsen worker 88 - jepsen.util 358 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:04,022] jepsen worker 88 - jepsen.util 448 :invoke :write [29 32] INFO [2019-12-17 12:28:04,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:04,027] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2805,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:04,027] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:04,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2809,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:04,051] jepsen worker 68 - jepsen.util 338 :info :cas [22 [4 42]] :RequestCanceledException INFO [2019-12-17 12:28:04,051] jepsen worker 38 - jepsen.util 398 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:28:04,051] jepsen worker 66 - jepsen.util 426 :info :cas [22 [19 33]] :RequestCanceledException INFO [2019-12-17 12:28:04,053] jepsen worker 68 - jepsen.util 428 :invoke :cas [22 [26 8]] INFO [2019-12-17 12:28:04,053] jepsen worker 38 - jepsen.util 488 :invoke :cas [12 [2 24]] INFO [2019-12-17 12:28:04,053] jepsen worker 66 - jepsen.util 516 :invoke :read [22 nil] INFO [2019-12-17 12:28:04,060] jepsen worker 61 - jepsen.util 691 :ok :read [20 7] INFO [2019-12-17 12:28:04,060] jepsen worker 61 - jepsen.util 691 :invoke :write [20 4] INFO [2019-12-17 12:28:04,075] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [18 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,075] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [5 27]] INFO [2019-12-17 12:28:04,075] jepsen worker 6 - jepsen.util 546 :ok :read [2 15] INFO [2019-12-17 12:28:04,075] jepsen worker 6 - jepsen.util 546 :invoke :write [2 17] INFO [2019-12-17 12:28:04,116] jepsen worker 69 - jepsen.util 699 :fail :cas [23 [12 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,116] jepsen worker 69 - jepsen.util 699 :invoke :read [23 nil] INFO [2019-12-17 12:28:04,121] jepsen worker 40 - jepsen.util 670 :info :cas [13 [36 40]] :RequestCanceledException INFO [2019-12-17 12:28:04,121] jepsen worker 12 - jepsen.util 462 :info :write [4 41] :RequestCanceledException INFO [2019-12-17 12:28:04,122] jepsen worker 40 - jepsen.util 760 :invoke :write [13 32] INFO [2019-12-17 12:28:04,122] jepsen worker 12 - jepsen.util 552 :invoke :cas [4 [23 17]] INFO [2019-12-17 12:28:04,187] jepsen worker 54 - jepsen.util 594 :info :write [18 44] :RequestCanceledException INFO [2019-12-17 12:28:04,187] jepsen worker 13 - jepsen.util 553 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:28:04,188] jepsen worker 13 - jepsen.util 643 :invoke :write [4 46] INFO [2019-12-17 12:28:04,188] jepsen worker 54 - jepsen.util 684 :invoke :write [18 1] INFO [2019-12-17 12:28:04,224] jepsen worker 78 - jepsen.util 618 :ok :read [26 47] INFO [2019-12-17 12:28:04,225] jepsen worker 78 - jepsen.util 618 :invoke :write [26 37] INFO [2019-12-17 12:28:04,233] jepsen worker 42 - jepsen.util 852 :fail :cas [14 [17 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,233] jepsen worker 42 - jepsen.util 852 :invoke :cas [14 [27 1]] INFO [2019-12-17 12:28:04,235] jepsen worker 35 - jepsen.util 395 :info :write [11 5] :RequestCanceledException INFO [2019-12-17 12:28:04,236] jepsen worker 35 - jepsen.util 485 :invoke :read [11 nil] INFO [2019-12-17 12:28:04,245] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:04,245] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2832,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:04,246] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:04,246] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2835,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:04,250] jepsen worker 37 - jepsen.util 487 :info :write [12 11] :RequestCanceledException INFO [2019-12-17 12:28:04,250] jepsen worker 71 - jepsen.util 431 :info :write [23 17] :RequestCanceledException INFO [2019-12-17 12:28:04,250] jepsen worker 28 - jepsen.util 298 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,250] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,251] jepsen worker 71 - jepsen.util 521 :invoke :cas [23 [30 6]] INFO [2019-12-17 12:28:04,251] jepsen worker 37 - jepsen.util 577 :invoke :read [12 nil] INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2823,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:04,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2825,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:04,267] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,267] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,281] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,281] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [34 40]] INFO [2019-12-17 12:28:04,294] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [34 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,295] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,305] jepsen worker 81 - jepsen.util 531 :info :cas [27 [47 8]] :RequestCanceledException INFO [2019-12-17 12:28:04,306] jepsen worker 81 - jepsen.util 621 :invoke :cas [27 [11 7]] INFO [2019-12-17 12:28:04,307] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,307] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [49 20]] INFO [2019-12-17 12:28:04,312] jepsen worker 26 - jepsen.util 746 :fail :cas [8 [46 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,312] jepsen worker 26 - jepsen.util 746 :invoke :write [8 47] INFO [2019-12-17 12:28:04,321] jepsen worker 70 - jepsen.util 700 :info :write [23 39] :RequestCanceledException INFO [2019-12-17 12:28:04,321] jepsen worker 77 - jepsen.util 437 :info :cas [25 [2 14]] :RequestCanceledException INFO [2019-12-17 12:28:04,322] jepsen worker 77 - jepsen.util 527 :invoke :cas [25 [7 22]] INFO [2019-12-17 12:28:04,322] jepsen worker 70 - jepsen.util 790 :invoke :write [23 24] INFO [2019-12-17 12:28:04,325] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [49 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,325] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,342] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:04,343] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:28:04,396] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:28:04,396] jepsen worker 28 - jepsen.util 298 :invoke :write [9 20] INFO [2019-12-17 12:28:04,413] jepsen worker 9 - jepsen.util 549 :ok :write [3 23] INFO [2019-12-17 12:28:04,413] jepsen worker 9 - jepsen.util 549 :invoke :read [3 nil] INFO [2019-12-17 12:28:04,420] jepsen worker 86 - jepsen.util 356 :info :cas [28 [28 32]] :RequestCanceledException INFO [2019-12-17 12:28:04,421] jepsen worker 86 - jepsen.util 446 :invoke :write [28 37] INFO [2019-12-17 12:28:04,421] jepsen worker 83 - jepsen.util 623 :info :write [27 23] :RequestCanceledException INFO [2019-12-17 12:28:04,422] jepsen worker 83 - jepsen.util 713 :invoke :write [27 28] INFO [2019-12-17 12:28:04,435] jepsen worker 34 - jepsen.util 664 :fail :cas [11 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,435] jepsen worker 34 - jepsen.util 664 :invoke :read [11 nil] INFO [2019-12-17 12:28:04,439] jepsen worker 37 - jepsen.util 577 :ok :read [12 12] INFO [2019-12-17 12:28:04,439] jepsen worker 37 - jepsen.util 577 :invoke :write [12 6] INFO [2019-12-17 12:28:04,446] jepsen worker 28 - jepsen.util 298 :ok :write [9 20] INFO [2019-12-17 12:28:04,446] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,448] jepsen worker 64 - jepsen.util 424 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,448] jepsen worker 45 - jepsen.util 765 :info :write [15 10] :RequestCanceledException INFO [2019-12-17 12:28:04,448] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:04,449] jepsen worker 45 - jepsen.util 855 :invoke :write [15 11] INFO [2019-12-17 12:28:04,464] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:04,465] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2860,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:04,467] jepsen worker 28 - jepsen.util 298 :ok :read [9 20] INFO [2019-12-17 12:28:04,467] jepsen worker 28 - jepsen.util 298 :invoke :write [9 22] INFO [2019-12-17 12:28:04,482] jepsen worker 27 - jepsen.util 387 :ok :write [9 32] INFO [2019-12-17 12:28:04,482] jepsen worker 27 - jepsen.util 387 :invoke :cas [9 [32 5]] INFO [2019-12-17 12:28:04,487] jepsen worker 30 - jepsen.util 480 :info :write [10 38] :RequestCanceledException INFO [2019-12-17 12:28:04,488] jepsen worker 30 - jepsen.util 570 :invoke :write [10 30] INFO [2019-12-17 12:28:04,494] jepsen worker 64 - jepsen.util 424 :ok :read [21 35] INFO [2019-12-17 12:28:04,494] jepsen worker 64 - jepsen.util 424 :invoke :read [21 nil] INFO [2019-12-17 12:28:04,504] jepsen worker 51 - jepsen.util 501 :fail :cas [17 [11 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,504] jepsen worker 51 - jepsen.util 501 :invoke :write [17 14] INFO [2019-12-17 12:28:04,505] jepsen worker 89 - jepsen.util 629 :info :write [29 39] :RequestCanceledException INFO [2019-12-17 12:28:04,507] jepsen worker 89 - jepsen.util 719 :invoke :write [29 18] INFO [2019-12-17 12:28:04,516] jepsen worker 73 - jepsen.util 703 :ok :read [24 35] INFO [2019-12-17 12:28:04,516] jepsen worker 73 - jepsen.util 703 :invoke :write [24 8] INFO [2019-12-17 12:28:04,526] jepsen worker 9 - jepsen.util 549 :ok :read [3 23] INFO [2019-12-17 12:28:04,527] jepsen worker 9 - jepsen.util 549 :invoke :cas [3 [12 10]] INFO [2019-12-17 12:28:04,535] jepsen worker 41 - jepsen.util 311 :info :write [13 36] :RequestCanceledException INFO [2019-12-17 12:28:04,535] jepsen worker 33 - jepsen.util 573 :info :write [11 30] :RequestCanceledException INFO [2019-12-17 12:28:04,536] jepsen worker 41 - jepsen.util 401 :invoke :read [13 nil] INFO [2019-12-17 12:28:04,536] jepsen worker 33 - jepsen.util 663 :invoke :cas [11 [0 43]] INFO [2019-12-17 12:28:04,551] jepsen worker 39 - jepsen.util 579 :info :write [13 40] :RequestCanceledException INFO [2019-12-17 12:28:04,553] jepsen worker 23 - jepsen.util 473 :ok :read [7 20] INFO [2019-12-17 12:28:04,553] jepsen worker 39 - jepsen.util 669 :invoke :write [13 5] INFO [2019-12-17 12:28:04,553] jepsen worker 23 - jepsen.util 473 :invoke :write [7 21] INFO [2019-12-17 12:28:04,587] jepsen worker 80 - jepsen.util 710 :info :write [26 18] :RequestCanceledException INFO [2019-12-17 12:28:04,588] jepsen worker 80 - jepsen.util 800 :invoke :cas [26 [18 47]] INFO [2019-12-17 12:28:04,600] jepsen worker 28 - jepsen.util 298 :ok :write [9 22] INFO [2019-12-17 12:28:04,601] jepsen worker 28 - jepsen.util 298 :invoke :write [9 25] INFO [2019-12-17 12:28:04,605] jepsen worker 12 - jepsen.util 552 :fail :cas [4 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,605] jepsen worker 77 - jepsen.util 527 :fail :cas [25 [7 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,605] jepsen worker 77 - jepsen.util 527 :invoke :write [25 23] INFO [2019-12-17 12:28:04,605] jepsen worker 12 - jepsen.util 552 :invoke :write [4 28] INFO [2019-12-17 12:28:04,621] jepsen worker 58 - jepsen.util 418 :info :cas [19 [2 2]] :RequestCanceledException INFO [2019-12-17 12:28:04,621] jepsen worker 75 - jepsen.util 435 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:28:04,622] jepsen worker 75 - jepsen.util 525 :invoke :read [25 nil] INFO [2019-12-17 12:28:04,622] jepsen worker 58 - jepsen.util 508 :invoke :read [19 nil] INFO [2019-12-17 12:28:04,635] jepsen worker 65 - jepsen.util 425 :info :write [21 6] :RequestCanceledException INFO [2019-12-17 12:28:04,636] jepsen worker 65 - jepsen.util 515 :invoke :cas [21 [33 17]] INFO [2019-12-17 12:28:04,679] jepsen worker 28 - jepsen.util 298 :ok :write [9 25] INFO [2019-12-17 12:28:04,679] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [39 0]] INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:04,684] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2893,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:04,692] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [39 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,692] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [10 7]] INFO [2019-12-17 12:28:04,695] jepsen worker 35 - jepsen.util 485 :ok :read [11 7] INFO [2019-12-17 12:28:04,695] jepsen worker 35 - jepsen.util 485 :invoke :write [11 10] INFO [2019-12-17 12:28:04,705] jepsen worker 19 - jepsen.util 649 :ok :write [6 39] INFO [2019-12-17 12:28:04,705] jepsen worker 19 - jepsen.util 649 :invoke :write [6 28] INFO [2019-12-17 12:28:04,706] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [10 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,706] jepsen worker 57 - jepsen.util 597 :info :cas [19 [26 10]] :RequestCanceledException INFO [2019-12-17 12:28:04,706] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,707] jepsen worker 57 - jepsen.util 687 :invoke :read [19 nil] INFO [2019-12-17 12:28:04,721] jepsen worker 0 - jepsen.util 720 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,721] jepsen worker 0 - jepsen.util 720 :invoke :read [0 nil] INFO [2019-12-17 12:28:04,743] jepsen worker 65 - jepsen.util 515 :fail :cas [21 [33 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,743] jepsen worker 65 - jepsen.util 515 :invoke :read [21 nil] INFO [2019-12-17 12:28:04,754] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,754] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [38 19]] INFO [2019-12-17 12:28:04,760] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [38 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,760] jepsen worker 17 - jepsen.util 557 :invoke :read [5 nil] INFO [2019-12-17 12:28:04,761] jepsen worker 32 - jepsen.util 572 :ok :read [10 38] INFO [2019-12-17 12:28:04,762] jepsen worker 32 - jepsen.util 572 :invoke :write [10 41] INFO [2019-12-17 12:28:04,781] jepsen worker 59 - jepsen.util 509 :info :cas [19 [26 27]] :RequestCanceledException INFO [2019-12-17 12:28:04,783] jepsen worker 59 - jepsen.util 599 :invoke :cas [19 [35 17]] INFO [2019-12-17 12:28:04,798] jepsen worker 31 - jepsen.util 751 :fail :cas [10 [1 5]] :CASMismatchException INFO [2019-12-17 12:28:04,799] jepsen worker 31 - jepsen.util 751 :invoke :cas [10 [15 41]] INFO [2019-12-17 12:28:04,819] jepsen worker 71 - jepsen.util 521 :fail :cas [23 [30 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,819] jepsen worker 71 - jepsen.util 521 :invoke :read [23 nil] INFO [2019-12-17 12:28:04,819] jepsen worker 52 - jepsen.util 502 :info :write [17 34] :RequestCanceledException INFO [2019-12-17 12:28:04,821] jepsen worker 52 - jepsen.util 592 :invoke :write [17 20] INFO [2019-12-17 12:28:04,831] jepsen worker 8 - jepsen.util 638 :fail :cas [2 [2 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,831] jepsen worker 8 - jepsen.util 638 :invoke :cas [2 [15 43]] INFO [2019-12-17 12:28:04,862] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,862] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,881] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,881] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,883] jepsen worker 85 - jepsen.util 625 :ok :cas [28 [33 19]] INFO [2019-12-17 12:28:04,883] jepsen worker 85 - jepsen.util 625 :invoke :cas [28 [12 31]] INFO [2019-12-17 12:28:04,900] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,900] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:04,903] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:04,903] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:04,920] jepsen worker 49 - jepsen.util 409 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:04,921] jepsen worker 49 - jepsen.util 409 :invoke :write [16 12] INFO [2019-12-17 12:28:04,924] jepsen worker 28 - jepsen.util 298 :ok :read [9 25] INFO [2019-12-17 12:28:04,924] jepsen worker 28 - jepsen.util 298 :invoke :write [9 42] INFO [2019-12-17 12:28:04,925] jepsen worker 75 - jepsen.util 525 :ok :read [25 2] INFO [2019-12-17 12:28:04,925] jepsen worker 75 - jepsen.util 525 :invoke :read [25 nil] INFO [2019-12-17 12:28:04,934] jepsen worker 20 - jepsen.util 560 :info :write [6 25] :RequestCanceledException INFO [2019-12-17 12:28:04,934] jepsen worker 72 - jepsen.util 522 :info :cas [24 [17 17]] :RequestCanceledException INFO [2019-12-17 12:28:04,936] jepsen worker 20 - jepsen.util 650 :invoke :read [6 nil] INFO [2019-12-17 12:28:04,936] jepsen worker 72 - jepsen.util 612 :invoke :read [24 nil] INFO [2019-12-17 12:28:04,952] jepsen worker 25 - jepsen.util 655 :info :write [8 18] :RequestCanceledException INFO [2019-12-17 12:28:04,953] jepsen worker 25 - jepsen.util 745 :invoke :cas [8 [15 32]] INFO [2019-12-17 12:28:04,964] jepsen worker 38 - jepsen.util 488 :fail :cas [12 [2 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,964] jepsen worker 38 - jepsen.util 488 :invoke :write [12 13] INFO [2019-12-17 12:28:04,970] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0703125}, :ns_1@172.28.128.126 {:progress 0.06474820143884896}, :ns_1@172.28.128.127 {:progress 0.1478599221789884}, :ns_1@172.28.128.128 {:progress 0.339622641509434}} INFO [2019-12-17 12:28:04,970] jepsen nemesis - couchbase.util Rebalance check count: 16 INFO [2019-12-17 12:28:04,970] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:04,977] jepsen worker 72 - jepsen.util 612 :ok :read [24 35] INFO [2019-12-17 12:28:04,977] jepsen worker 72 - jepsen.util 612 :invoke :write [24 35] INFO [2019-12-17 12:28:04,992] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:04,992] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2930,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2933,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,013] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2934,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,021] jepsen worker 63 - jepsen.util 243 :info :write [21 38] :RequestCanceledException INFO [2019-12-17 12:28:05,022] jepsen worker 63 - jepsen.util 333 :invoke :cas [21 [1 3]] INFO [2019-12-17 12:28:05,024] jepsen worker 28 - jepsen.util 298 :ok :write [9 42] INFO [2019-12-17 12:28:05,024] jepsen worker 28 - jepsen.util 298 :invoke :write [9 30] INFO [2019-12-17 12:28:05,035] jepsen worker 60 - jepsen.util 600 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:28:05,036] jepsen worker 60 - jepsen.util 690 :invoke :read [20 nil] INFO [2019-12-17 12:28:05,044] jepsen worker 63 - jepsen.util 333 :fail :cas [21 [1 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,044] jepsen worker 63 - jepsen.util 333 :invoke :cas [21 [24 4]] INFO [2019-12-17 12:28:05,069] jepsen worker 74 - jepsen.util 524 :ok :write [24 39] INFO [2019-12-17 12:28:05,069] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [24 4]] INFO [2019-12-17 12:28:05,070] jepsen worker 69 - jepsen.util 699 :ok :read [23 29] INFO [2019-12-17 12:28:05,070] jepsen worker 69 - jepsen.util 699 :invoke :cas [23 [21 45]] INFO [2019-12-17 12:28:05,083] jepsen worker 28 - jepsen.util 298 :ok :write [9 30] INFO [2019-12-17 12:28:05,083] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [25 23]] INFO [2019-12-17 12:28:05,088] jepsen worker 75 - jepsen.util 525 :ok :read [25 2] INFO [2019-12-17 12:28:05,088] jepsen worker 75 - jepsen.util 525 :invoke :write [25 8] INFO [2019-12-17 12:28:05,091] jepsen worker 34 - jepsen.util 664 :ok :read [11 7] INFO [2019-12-17 12:28:05,091] jepsen worker 34 - jepsen.util 664 :invoke :read [11 nil] INFO [2019-12-17 12:28:05,106] jepsen worker 53 - jepsen.util 593 :info :write [17 43] :RequestCanceledException INFO [2019-12-17 12:28:05,106] jepsen worker 53 - jepsen.util 683 :invoke :write [17 37] INFO [2019-12-17 12:28:05,118] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [25 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,119] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,121] jepsen worker 67 - jepsen.util 787 :info :cas [22 [12 1]] :RequestCanceledException INFO [2019-12-17 12:28:05,122] jepsen worker 67 - jepsen.util 877 :invoke :write [22 33] INFO [2019-12-17 12:28:05,125] jepsen worker 71 - jepsen.util 521 :ok :read [23 29] INFO [2019-12-17 12:28:05,125] jepsen worker 71 - jepsen.util 521 :invoke :read [23 nil] INFO [2019-12-17 12:28:05,138] jepsen worker 28 - jepsen.util 298 :ok :read [9 30] INFO [2019-12-17 12:28:05,138] jepsen worker 28 - jepsen.util 298 :invoke :write [9 13] INFO [2019-12-17 12:28:05,199] jepsen worker 28 - jepsen.util 298 :ok :write [9 13] INFO [2019-12-17 12:28:05,199] jepsen worker 28 - jepsen.util 298 :invoke :write [9 24] INFO [2019-12-17 12:28:05,205] jepsen worker 47 - jepsen.util 767 :info :write [15 20] :RequestCanceledException INFO [2019-12-17 12:28:05,205] jepsen worker 11 - jepsen.util 641 :info :write [3 27] :RequestCanceledException INFO [2019-12-17 12:28:05,207] jepsen worker 47 - jepsen.util 857 :invoke :read [15 nil] INFO [2019-12-17 12:28:05,207] jepsen worker 11 - jepsen.util 731 :invoke :write [3 1] INFO [2019-12-17 12:28:05,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:05,232] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2961,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:05,235] jepsen worker 43 - jepsen.util 763 :info :cas [14 [4 0]] :RequestCanceledException INFO [2019-12-17 12:28:05,236] jepsen worker 43 - jepsen.util 853 :invoke :write [14 18] INFO [2019-12-17 12:28:05,242] jepsen worker 47 - jepsen.util 857 :ok :read [15 35] INFO [2019-12-17 12:28:05,242] jepsen worker 47 - jepsen.util 857 :invoke :read [15 nil] INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2952,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:05,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2953,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:05,251] jepsen worker 25 - jepsen.util 745 :fail :cas [8 [15 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,251] jepsen worker 25 - jepsen.util 745 :invoke :write [8 36] INFO [2019-12-17 12:28:05,273] jepsen worker 47 - jepsen.util 857 :ok :read [15 35] INFO [2019-12-17 12:28:05,273] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [44 16]] INFO [2019-12-17 12:28:05,275] jepsen worker 3 - jepsen.util 453 :ok :read [1 37] INFO [2019-12-17 12:28:05,275] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,283] jepsen worker 28 - jepsen.util 298 :ok :write [9 24] INFO [2019-12-17 12:28:05,283] jepsen worker 28 - jepsen.util 298 :invoke :write [9 6] INFO [2019-12-17 12:28:05,288] jepsen worker 55 - jepsen.util 775 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:28:05,290] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,290] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [42 4]] INFO [2019-12-17 12:28:05,290] jepsen worker 55 - jepsen.util 865 :invoke :read [18 nil] INFO [2019-12-17 12:28:05,324] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,324] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [18 13]] INFO [2019-12-17 12:28:05,349] jepsen worker 69 - jepsen.util 699 :fail :cas [23 [21 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,349] jepsen worker 69 - jepsen.util 699 :invoke :read [23 nil] INFO [2019-12-17 12:28:05,362] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [18 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,362] jepsen worker 47 - jepsen.util 857 :invoke :cas [15 [5 27]] INFO [2019-12-17 12:28:05,368] jepsen worker 28 - jepsen.util 298 :ok :write [9 6] INFO [2019-12-17 12:28:05,369] jepsen worker 28 - jepsen.util 298 :invoke :write [9 42] INFO [2019-12-17 12:28:05,379] jepsen worker 47 - jepsen.util 857 :fail :cas [15 [5 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,379] jepsen worker 47 - jepsen.util 857 :invoke :write [15 6] INFO [2019-12-17 12:28:05,391] jepsen worker 71 - jepsen.util 521 :ok :read [23 29] INFO [2019-12-17 12:28:05,392] jepsen worker 71 - jepsen.util 521 :invoke :write [23 16] INFO [2019-12-17 12:28:05,406] jepsen worker 29 - jepsen.util 479 :info :write [9 2] :RequestCanceledException INFO [2019-12-17 12:28:05,408] jepsen worker 29 - jepsen.util 569 :invoke :write [9 48] INFO [2019-12-17 12:28:05,421] jepsen worker 87 - jepsen.util 447 :info :write [29 17] :RequestCanceledException INFO [2019-12-17 12:28:05,422] jepsen worker 87 - jepsen.util 537 :invoke :write [29 30] INFO [2019-12-17 12:28:05,445] jepsen worker 28 - jepsen.util 298 :ok :write [9 42] INFO [2019-12-17 12:28:05,445] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [19 10]] INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2985,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:05,451] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":2986,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:05,459] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,459] jepsen worker 28 - jepsen.util 298 :invoke :write [9 1] INFO [2019-12-17 12:28:05,484] jepsen worker 50 - jepsen.util 590 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:05,484] jepsen worker 50 - jepsen.util 590 :invoke :cas [16 [3 30]] INFO [2019-12-17 12:28:05,500] jepsen worker 33 - jepsen.util 663 :fail :cas [11 [0 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,500] jepsen worker 33 - jepsen.util 663 :invoke :read [11 nil] INFO [2019-12-17 12:28:05,504] jepsen worker 69 - jepsen.util 699 :ok :read [23 29] INFO [2019-12-17 12:28:05,505] jepsen worker 69 - jepsen.util 699 :invoke :write [23 43] INFO [2019-12-17 12:28:05,506] jepsen worker 46 - jepsen.util 496 :info :cas [15 [43 40]] :RequestCanceledException INFO [2019-12-17 12:28:05,507] jepsen worker 46 - jepsen.util 586 :invoke :cas [15 [49 21]] INFO [2019-12-17 12:28:05,517] jepsen worker 28 - jepsen.util 298 :ok :write [9 1] INFO [2019-12-17 12:28:05,517] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [33 3]] INFO [2019-12-17 12:28:05,521] jepsen worker 14 - jepsen.util 644 :info :write [4 34] :RequestCanceledException INFO [2019-12-17 12:28:05,521] jepsen worker 56 - jepsen.util 506 :info :write [18 31] :RequestCanceledException INFO [2019-12-17 12:28:05,523] jepsen worker 56 - jepsen.util 596 :invoke :write [18 12] INFO [2019-12-17 12:28:05,523] jepsen worker 14 - jepsen.util 734 :invoke :cas [4 [21 47]] INFO [2019-12-17 12:28:05,540] jepsen worker 55 - jepsen.util 865 :ok :read [18 47] INFO [2019-12-17 12:28:05,540] jepsen worker 46 - jepsen.util 586 :fail :cas [15 [49 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,540] jepsen worker 55 - jepsen.util 865 :invoke :cas [18 [18 20]] INFO [2019-12-17 12:28:05,540] jepsen worker 46 - jepsen.util 586 :invoke :write [15 23] INFO [2019-12-17 12:28:05,552] jepsen worker 5 - jepsen.util 275 :info :write [1 16] :RequestCanceledException INFO [2019-12-17 12:28:05,553] jepsen worker 5 - jepsen.util 365 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,555] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [33 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,555] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [37 48]] INFO [2019-12-17 12:28:05,576] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [37 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,576] jepsen worker 28 - jepsen.util 298 :invoke :write [9 36] INFO [2019-12-17 12:28:05,605] jepsen worker 79 - jepsen.util 529 :info :write [26 1] :RequestCanceledException INFO [2019-12-17 12:28:05,606] jepsen worker 79 - jepsen.util 619 :invoke :cas [26 [35 19]] INFO [2019-12-17 12:28:05,611] jepsen worker 28 - jepsen.util 298 :ok :write [9 36] INFO [2019-12-17 12:28:05,611] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,621] jepsen worker 4 - jepsen.util 544 :info :write [1 18] :RequestCanceledException INFO [2019-12-17 12:28:05,622] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,624] jepsen worker 28 - jepsen.util 298 :ok :read [9 36] INFO [2019-12-17 12:28:05,624] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [7 43]] INFO [2019-12-17 12:28:05,637] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [7 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,637] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [18 42]] INFO [2019-12-17 12:28:05,649] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [18 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,649] jepsen worker 28 - jepsen.util 298 :invoke :write [9 10] INFO [2019-12-17 12:28:05,652] jepsen worker 24 - jepsen.util 474 :info :write [8 9] :RequestCanceledException INFO [2019-12-17 12:28:05,653] jepsen worker 24 - jepsen.util 564 :invoke :cas [8 [38 7]] INFO [2019-12-17 12:28:05,676] jepsen worker 30 - jepsen.util 570 :ok :write [10 30] INFO [2019-12-17 12:28:05,676] jepsen worker 30 - jepsen.util 570 :invoke :write [10 38] INFO [2019-12-17 12:28:05,677] jepsen worker 19 - jepsen.util 649 :ok :write [6 28] INFO [2019-12-17 12:28:05,677] jepsen worker 19 - jepsen.util 649 :invoke :write [6 25] INFO [2019-12-17 12:28:05,687] jepsen worker 2 - jepsen.util 542 :info :write [0 34] :RequestCanceledException INFO [2019-12-17 12:28:05,688] jepsen worker 2 - jepsen.util 632 :invoke :write [0 9] INFO [2019-12-17 12:28:05,718] jepsen worker 22 - jepsen.util 562 :info :write [7 19] :RequestCanceledException INFO [2019-12-17 12:28:05,720] jepsen worker 22 - jepsen.util 652 :invoke :cas [7 [44 47]] INFO [2019-12-17 12:28:05,722] jepsen worker 48 - jepsen.util 498 :ok :write [16 12] INFO [2019-12-17 12:28:05,722] jepsen worker 48 - jepsen.util 498 :invoke :write [16 17] INFO [2019-12-17 12:28:05,729] jepsen worker 76 - jepsen.util 616 :info :write [25 17] :RequestCanceledException INFO [2019-12-17 12:28:05,729] jepsen worker 1 - jepsen.util 631 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:05,729] jepsen worker 1 - jepsen.util 631 :invoke :cas [0 [49 45]] INFO [2019-12-17 12:28:05,730] jepsen worker 76 - jepsen.util 706 :invoke :read [25 nil] INFO [2019-12-17 12:28:05,731] jepsen worker 28 - jepsen.util 298 :ok :write [9 10] INFO [2019-12-17 12:28:05,731] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,732] jepsen worker 80 - jepsen.util 800 :fail :cas [26 [18 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,732] jepsen worker 80 - jepsen.util 800 :invoke :write [26 37] INFO [2019-12-17 12:28:05,733] jepsen worker 20 - jepsen.util 650 :ok :read [6 28] INFO [2019-12-17 12:28:05,733] jepsen worker 20 - jepsen.util 650 :invoke :cas [6 [4 0]] INFO [2019-12-17 12:28:05,749] jepsen worker 3 - jepsen.util 453 :ok :read [1 37] INFO [2019-12-17 12:28:05,749] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:05,750] jepsen worker 44 - jepsen.util 404 :info :write [14 43] :RequestCanceledException INFO [2019-12-17 12:28:05,750] jepsen worker 62 - jepsen.util 692 :info :write [20 34] :RequestCanceledException INFO [2019-12-17 12:28:05,752] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [45 32]] INFO [2019-12-17 12:28:05,752] jepsen worker 62 - jepsen.util 782 :invoke :write [20 39] INFO [2019-12-17 12:28:05,774] jepsen worker 68 - jepsen.util 428 :fail :cas [22 [26 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,775] jepsen worker 68 - jepsen.util 428 :invoke :write [22 10] INFO [2019-12-17 12:28:05,781] jepsen worker 74 - jepsen.util 524 :fail :cas [24 [24 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,782] jepsen worker 74 - jepsen.util 524 :invoke :read [24 nil] INFO [2019-12-17 12:28:05,787] jepsen worker 41 - jepsen.util 401 :ok :read [13 36] INFO [2019-12-17 12:28:05,787] jepsen worker 41 - jepsen.util 401 :invoke :cas [13 [29 15]] INFO [2019-12-17 12:28:05,800] jepsen worker 28 - jepsen.util 298 :ok :read [9 10] INFO [2019-12-17 12:28:05,800] jepsen worker 17 - jepsen.util 557 :ok :read [5 13] INFO [2019-12-17 12:28:05,800] jepsen worker 28 - jepsen.util 298 :invoke :write [9 9] INFO [2019-12-17 12:28:05,800] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [27 27]] INFO [2019-12-17 12:28:05,816] jepsen worker 85 - jepsen.util 625 :fail :cas [28 [12 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,816] jepsen worker 85 - jepsen.util 625 :invoke :write [28 38] INFO [2019-12-17 12:28:05,819] jepsen worker 10 - jepsen.util 550 :info :write [3 44] :RequestCanceledException INFO [2019-12-17 12:28:05,819] jepsen worker 55 - jepsen.util 865 :fail :cas [18 [18 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,820] jepsen worker 55 - jepsen.util 865 :invoke :read [18 nil] INFO [2019-12-17 12:28:05,821] jepsen worker 10 - jepsen.util 640 :invoke :read [3 nil] INFO [2019-12-17 12:28:05,847] jepsen worker 5 - jepsen.util 365 :ok :read [1 37] INFO [2019-12-17 12:28:05,847] jepsen worker 5 - jepsen.util 365 :invoke :write [1 46] INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3038,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:05,881] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3042,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:05,908] jepsen worker 10 - jepsen.util 640 :ok :read [3 23] INFO [2019-12-17 12:28:05,908] jepsen worker 10 - jepsen.util 640 :invoke :write [3 35] INFO [2019-12-17 12:28:05,910] jepsen worker 28 - jepsen.util 298 :ok :write [9 9] INFO [2019-12-17 12:28:05,910] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [30 10]] INFO [2019-12-17 12:28:05,936] jepsen worker 33 - jepsen.util 663 :ok :read [11 7] INFO [2019-12-17 12:28:05,936] jepsen worker 33 - jepsen.util 663 :invoke :write [11 33] INFO [2019-12-17 12:28:05,950] jepsen worker 65 - jepsen.util 515 :ok :read [21 35] INFO [2019-12-17 12:28:05,951] jepsen worker 65 - jepsen.util 515 :invoke :read [21 nil] INFO [2019-12-17 12:28:05,952] jepsen worker 84 - jepsen.util 354 :info :write [28 22] :RequestCanceledException INFO [2019-12-17 12:28:05,953] jepsen worker 84 - jepsen.util 444 :invoke :read [28 nil] INFO [2019-12-17 12:28:05,959] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [30 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:05,960] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,984] jepsen worker 28 - jepsen.util 298 :ok :read [9 9] INFO [2019-12-17 12:28:05,984] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:05,985] jepsen worker 21 - jepsen.util 651 :info :write [7 4] :RequestCanceledException INFO [2019-12-17 12:28:05,985] jepsen worker 7 - jepsen.util 727 :info :write [2 30] :RequestCanceledException INFO [2019-12-17 12:28:05,986] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.0703125}, :ns_1@172.28.128.126 {:progress 0.06474820143884896}, :ns_1@172.28.128.127 {:progress 0.1556420233463035}, :ns_1@172.28.128.128 {:progress 0.3924528301886793}} INFO [2019-12-17 12:28:05,986] jepsen worker 7 - jepsen.util 817 :invoke :read [2 nil] INFO [2019-12-17 12:28:05,986] jepsen worker 21 - jepsen.util 741 :invoke :write [7 28] INFO [2019-12-17 12:28:05,986] jepsen nemesis - couchbase.util Rebalance check count: 17 INFO [2019-12-17 12:28:05,986] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3055,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:05,992] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3056,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:06,003] jepsen worker 60 - jepsen.util 690 :ok :read [20 7] INFO [2019-12-17 12:28:06,003] jepsen worker 60 - jepsen.util 690 :invoke :write [20 15] INFO [2019-12-17 12:28:06,005] jepsen worker 15 - jepsen.util 645 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,006] jepsen worker 15 - jepsen.util 645 :invoke :read [5 nil] INFO [2019-12-17 12:28:06,010] jepsen worker 28 - jepsen.util 298 :ok :read [9 9] INFO [2019-12-17 12:28:06,010] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,014] jepsen worker 2 - jepsen.util 632 :ok :write [0 9] INFO [2019-12-17 12:28:06,014] jepsen worker 2 - jepsen.util 632 :invoke :write [0 37] INFO [2019-12-17 12:28:06,019] jepsen worker 82 - jepsen.util 712 :info :cas [27 [20 13]] :RequestCanceledException INFO [2019-12-17 12:28:06,020] jepsen worker 82 - jepsen.util 802 :invoke :cas [27 [3 15]] INFO [2019-12-17 12:28:06,026] jepsen worker 34 - jepsen.util 664 :ok :read [11 7] INFO [2019-12-17 12:28:06,026] jepsen worker 34 - jepsen.util 664 :invoke :write [11 29] INFO [2019-12-17 12:28:06,028] jepsen worker 11 - jepsen.util 731 :ok :write [3 1] INFO [2019-12-17 12:28:06,028] jepsen worker 11 - jepsen.util 731 :invoke :cas [3 [38 16]] INFO [2019-12-17 12:28:06,033] jepsen worker 36 - jepsen.util 666 :info :write [12 39] :RequestCanceledException INFO [2019-12-17 12:28:06,034] jepsen worker 36 - jepsen.util 756 :invoke :read [12 nil] INFO [2019-12-17 12:28:06,046] jepsen worker 28 - jepsen.util 298 :ok :read [9 9] INFO [2019-12-17 12:28:06,046] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [14 39]] INFO [2019-12-17 12:28:06,052] jepsen worker 1 - jepsen.util 631 :fail :cas [0 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,052] jepsen worker 16 - jepsen.util 646 :info :write [5 17] :RequestCanceledException INFO [2019-12-17 12:28:06,053] jepsen worker 1 - jepsen.util 631 :invoke :cas [0 [33 47]] INFO [2019-12-17 12:28:06,054] jepsen worker 16 - jepsen.util 736 :invoke :read [5 nil] INFO [2019-12-17 12:28:06,057] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [45 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,058] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [16 32]] INFO [2019-12-17 12:28:06,062] jepsen worker 30 - jepsen.util 570 :ok :write [10 38] INFO [2019-12-17 12:28:06,063] jepsen worker 30 - jepsen.util 570 :invoke :read [10 nil] INFO [2019-12-17 12:28:06,071] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [14 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,072] jepsen worker 28 - jepsen.util 298 :invoke :write [9 31] INFO [2019-12-17 12:28:06,087] jepsen worker 18 - jepsen.util 648 :info :cas [6 [8 34]] :RequestCanceledException INFO [2019-12-17 12:28:06,087] jepsen worker 61 - jepsen.util 691 :info :write [20 4] :RequestCanceledException INFO [2019-12-17 12:28:06,088] jepsen worker 18 - jepsen.util 738 :invoke :cas [6 [46 24]] INFO [2019-12-17 12:28:06,089] jepsen worker 76 - jepsen.util 706 :ok :read [25 2] INFO [2019-12-17 12:28:06,089] jepsen worker 76 - jepsen.util 706 :invoke :read [25 nil] INFO [2019-12-17 12:28:06,089] jepsen worker 14 - jepsen.util 734 :fail :cas [4 [21 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,089] jepsen worker 61 - jepsen.util 781 :invoke :read [20 nil] INFO [2019-12-17 12:28:06,089] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,117] jepsen worker 66 - jepsen.util 516 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,117] jepsen worker 6 - jepsen.util 546 :info :write [2 17] :RequestCanceledException INFO [2019-12-17 12:28:06,117] jepsen worker 88 - jepsen.util 448 :info :write [29 32] :RequestCanceledException INFO [2019-12-17 12:28:06,117] jepsen worker 66 - jepsen.util 516 :invoke :read [22 nil] INFO [2019-12-17 12:28:06,118] jepsen worker 68 - jepsen.util 428 :ok :write [22 10] INFO [2019-12-17 12:28:06,118] jepsen worker 6 - jepsen.util 636 :invoke :read [2 nil] INFO [2019-12-17 12:28:06,118] jepsen worker 88 - jepsen.util 538 :invoke :write [29 14] INFO [2019-12-17 12:28:06,118] jepsen worker 68 - jepsen.util 428 :invoke :read [22 nil] INFO [2019-12-17 12:28:06,119] jepsen worker 30 - jepsen.util 570 :ok :read [10 38] INFO [2019-12-17 12:28:06,119] jepsen worker 30 - jepsen.util 570 :invoke :write [10 35] INFO [2019-12-17 12:28:06,149] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [16 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,149] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,159] jepsen worker 41 - jepsen.util 401 :fail :cas [13 [29 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,159] jepsen worker 41 - jepsen.util 401 :invoke :read [13 nil] INFO [2019-12-17 12:28:06,169] jepsen worker 28 - jepsen.util 298 :ok :write [9 31] INFO [2019-12-17 12:28:06,169] jepsen worker 28 - jepsen.util 298 :invoke :write [9 43] INFO [2019-12-17 12:28:06,188] jepsen worker 40 - jepsen.util 760 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:28:06,189] jepsen worker 40 - jepsen.util 850 :invoke :cas [13 [32 3]] INFO [2019-12-17 12:28:06,201] jepsen worker 76 - jepsen.util 706 :ok :read [25 2] INFO [2019-12-17 12:28:06,201] jepsen worker 76 - jepsen.util 706 :invoke :cas [25 [10 33]] INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3093,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:06,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3097,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:06,215] jepsen worker 28 - jepsen.util 298 :ok :write [9 43] INFO [2019-12-17 12:28:06,215] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,221] jepsen worker 13 - jepsen.util 643 :info :write [4 46] :RequestCanceledException INFO [2019-12-17 12:28:06,222] jepsen worker 13 - jepsen.util 733 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:06,222] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3085,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:06,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:06,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3086,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:06,234] jepsen worker 54 - jepsen.util 684 :info :write [18 1] :RequestCanceledException INFO [2019-12-17 12:28:06,235] jepsen worker 78 - jepsen.util 618 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:06,236] jepsen worker 78 - jepsen.util 708 :invoke :write [26 39] INFO [2019-12-17 12:28:06,237] jepsen worker 54 - jepsen.util 774 :invoke :cas [18 [23 19]] INFO [2019-12-17 12:28:06,243] jepsen worker 28 - jepsen.util 298 :ok :read [9 43] INFO [2019-12-17 12:28:06,243] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [25 17]] INFO [2019-12-17 12:28:06,255] jepsen worker 3 - jepsen.util 453 :ok :read [1 37] INFO [2019-12-17 12:28:06,255] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [34 42]] INFO [2019-12-17 12:28:06,260] jepsen worker 22 - jepsen.util 652 :fail :cas [7 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,260] jepsen worker 22 - jepsen.util 652 :invoke :cas [7 [41 9]] INFO [2019-12-17 12:28:06,264] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,264] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,268] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [25 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,268] jepsen worker 28 - jepsen.util 298 :invoke :write [9 27] INFO [2019-12-17 12:28:06,276] jepsen worker 55 - jepsen.util 865 :ok :read [18 47] INFO [2019-12-17 12:28:06,276] jepsen worker 55 - jepsen.util 865 :invoke :cas [18 [48 22]] INFO [2019-12-17 12:28:06,296] jepsen worker 4 - jepsen.util 634 :ok :read [1 37] INFO [2019-12-17 12:28:06,296] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [1 24]] INFO [2019-12-17 12:28:06,327] jepsen worker 28 - jepsen.util 298 :ok :write [9 27] INFO [2019-12-17 12:28:06,328] jepsen worker 28 - jepsen.util 298 :invoke :write [9 7] INFO [2019-12-17 12:28:06,334] jepsen worker 26 - jepsen.util 746 :info :write [8 47] :RequestCanceledException INFO [2019-12-17 12:28:06,336] jepsen worker 26 - jepsen.util 836 :invoke :cas [8 [46 32]] INFO [2019-12-17 12:28:06,359] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [1 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,359] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [12 44]] INFO [2019-12-17 12:28:06,369] jepsen worker 7 - jepsen.util 817 :ok :read [2 15] INFO [2019-12-17 12:28:06,369] jepsen worker 7 - jepsen.util 817 :invoke :cas [2 [31 32]] INFO [2019-12-17 12:28:06,385] jepsen worker 28 - jepsen.util 298 :ok :write [9 7] INFO [2019-12-17 12:28:06,385] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [49 28]] INFO [2019-12-17 12:28:06,391] jepsen worker 36 - jepsen.util 756 :ok :read [12 12] INFO [2019-12-17 12:28:06,391] jepsen worker 36 - jepsen.util 756 :invoke :write [12 37] INFO [2019-12-17 12:28:06,405] jepsen worker 81 - jepsen.util 621 :info :cas [27 [11 7]] :RequestCanceledException INFO [2019-12-17 12:28:06,407] jepsen worker 81 - jepsen.util 711 :invoke :write [27 11] INFO [2019-12-17 12:28:06,418] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [49 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,419] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,419] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,419] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3117,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3120,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:06,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3121,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:06,435] jepsen worker 70 - jepsen.util 790 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:28:06,438] jepsen worker 70 - jepsen.util 880 :invoke :cas [23 [33 23]] INFO [2019-12-17 12:28:06,449] jepsen worker 86 - jepsen.util 446 :info :write [28 37] :RequestCanceledException INFO [2019-12-17 12:28:06,451] jepsen worker 86 - jepsen.util 536 :invoke :read [28 nil] INFO [2019-12-17 12:28:06,453] jepsen worker 28 - jepsen.util 298 :ok :read [9 7] INFO [2019-12-17 12:28:06,454] jepsen worker 28 - jepsen.util 298 :invoke :write [9 6] INFO [2019-12-17 12:28:06,454] jepsen worker 61 - jepsen.util 781 :ok :read [20 7] INFO [2019-12-17 12:28:06,454] jepsen worker 61 - jepsen.util 781 :invoke :cas [20 [35 46]] INFO [2019-12-17 12:28:06,455] jepsen worker 79 - jepsen.util 619 :fail :cas [26 [35 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,455] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:06,484] jepsen worker 14 - jepsen.util 734 :ok :read [4 4] INFO [2019-12-17 12:28:06,485] jepsen worker 14 - jepsen.util 734 :invoke :cas [4 [40 39]] INFO [2019-12-17 12:28:06,487] jepsen worker 37 - jepsen.util 577 :info :write [12 6] :RequestCanceledException INFO [2019-12-17 12:28:06,487] jepsen worker 45 - jepsen.util 855 :info :write [15 11] :RequestCanceledException INFO [2019-12-17 12:28:06,489] jepsen worker 45 - jepsen.util 945 :invoke :write [15 14] INFO [2019-12-17 12:28:06,489] jepsen worker 37 - jepsen.util 667 :invoke :write [12 12] INFO [2019-12-17 12:28:06,507] jepsen worker 28 - jepsen.util 298 :ok :write [9 6] INFO [2019-12-17 12:28:06,507] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,521] jepsen worker 89 - jepsen.util 719 :info :write [29 18] :RequestCanceledException INFO [2019-12-17 12:28:06,521] jepsen worker 83 - jepsen.util 713 :info :write [27 28] :RequestCanceledException INFO [2019-12-17 12:28:06,522] jepsen worker 89 - jepsen.util 809 :invoke :write [29 10] INFO [2019-12-17 12:28:06,522] jepsen worker 83 - jepsen.util 803 :invoke :write [27 1] INFO [2019-12-17 12:28:06,531] jepsen worker 28 - jepsen.util 298 :ok :read [9 6] INFO [2019-12-17 12:28:06,532] jepsen worker 28 - jepsen.util 298 :invoke :write [9 36] INFO [2019-12-17 12:28:06,533] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,533] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,548] jepsen worker 22 - jepsen.util 652 :fail :cas [7 [41 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,549] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:06,552] jepsen worker 64 - jepsen.util 424 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,552] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [49 39]] INFO [2019-12-17 12:28:06,578] jepsen worker 54 - jepsen.util 774 :fail :cas [18 [23 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,578] jepsen worker 54 - jepsen.util 774 :invoke :cas [18 [35 9]] INFO [2019-12-17 12:28:06,584] jepsen worker 28 - jepsen.util 298 :ok :write [9 36] INFO [2019-12-17 12:28:06,584] jepsen worker 28 - jepsen.util 298 :invoke :write [9 32] INFO [2019-12-17 12:28:06,600] jepsen worker 51 - jepsen.util 501 :info :write [17 14] :RequestCanceledException INFO [2019-12-17 12:28:06,608] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [49 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,614] jepsen worker 9 - jepsen.util 549 :info :cas [3 [12 10]] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 77 - jepsen.util 527 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 73 - jepsen.util 703 :info :write [24 8] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 23 - jepsen.util 473 :info :write [7 21] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 27 - jepsen.util 387 :info :cas [9 [32 5]] :RequestCanceledException INFO [2019-12-17 12:28:06,614] jepsen worker 39 - jepsen.util 669 :info :write [13 5] :RequestCanceledException INFO [2019-12-17 12:28:06,617] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,618] jepsen worker 55 - jepsen.util 865 :fail :cas [18 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,619] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:06,620] jepsen worker 7 - jepsen.util 817 :fail :cas [2 [31 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,636] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [34 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,643] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:06,653] jepsen worker 28 - jepsen.util 298 :ok :write [9 32] INFO [2019-12-17 12:28:06,657] jepsen worker 64 - jepsen.util 424 :invoke :write [21 31] INFO [2019-12-17 12:28:06,658] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,658] jepsen worker 55 - jepsen.util 865 :invoke :cas [18 [2 26]] INFO [2019-12-17 12:28:06,658] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:06,658] jepsen worker 7 - jepsen.util 817 :invoke :write [2 15] INFO [2019-12-17 12:28:06,658] jepsen worker 3 - jepsen.util 453 :invoke :read [1 nil] INFO [2019-12-17 12:28:06,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3142,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:06,658] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [43 36]] INFO [2019-12-17 12:28:06,658] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:06,658] jepsen worker 9 - jepsen.util 639 :invoke :cas [3 [26 10]] INFO [2019-12-17 12:28:06,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3145,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:06,659] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:06,659] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3146,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:06,659] jepsen worker 39 - jepsen.util 759 :invoke :read [13 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [12 32]] INFO [2019-12-17 12:28:06,659] jepsen worker 77 - jepsen.util 617 :invoke :read [25 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:06,659] jepsen worker 23 - jepsen.util 563 :invoke :read [7 nil] INFO [2019-12-17 12:28:06,684] jepsen worker 27 - jepsen.util 477 :ok :read [9 32] INFO [2019-12-17 12:28:06,684] jepsen worker 27 - jepsen.util 477 :invoke :write [9 27] INFO [2019-12-17 12:28:06,701] jepsen worker 14 - jepsen.util 734 :fail :cas [4 [40 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,701] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,703] jepsen worker 12 - jepsen.util 552 :info :write [4 28] :RequestCanceledException INFO [2019-12-17 12:28:06,704] jepsen worker 12 - jepsen.util 642 :invoke :read [4 nil] INFO [2019-12-17 12:28:06,706] jepsen worker 73 - jepsen.util 793 :ok :read [24 39] INFO [2019-12-17 12:28:06,706] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:06,715] jepsen worker 24 - jepsen.util 564 :fail :cas [8 [38 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,716] jepsen worker 24 - jepsen.util 564 :invoke :write [8 19] INFO [2019-12-17 12:28:06,717] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,718] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [37 0]] INFO [2019-12-17 12:28:06,718] jepsen worker 58 - jepsen.util 508 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,718] jepsen worker 42 - jepsen.util 852 :info :cas [14 [27 1]] :RequestCanceledException INFO [2019-12-17 12:28:06,718] jepsen worker 58 - jepsen.util 508 :invoke :cas [19 [10 9]] INFO [2019-12-17 12:28:06,721] jepsen worker 42 - jepsen.util 942 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,733] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [43 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,733] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [47 30]] INFO [2019-12-17 12:28:06,741] jepsen worker 73 - jepsen.util 793 :ok :read [24 39] INFO [2019-12-17 12:28:06,741] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [13 24]] INFO [2019-12-17 12:28:06,745] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [37 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,746] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [5 46]] INFO [2019-12-17 12:28:06,752] jepsen worker 35 - jepsen.util 485 :info :write [11 10] :RequestCanceledException INFO [2019-12-17 12:28:06,752] jepsen worker 57 - jepsen.util 687 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,753] jepsen worker 57 - jepsen.util 687 :invoke :cas [19 [23 32]] INFO [2019-12-17 12:28:06,753] jepsen worker 13 - jepsen.util 733 :ok :read [4 4] INFO [2019-12-17 12:28:06,754] jepsen worker 13 - jepsen.util 733 :invoke :cas [4 [3 7]] INFO [2019-12-17 12:28:06,754] jepsen worker 35 - jepsen.util 575 :invoke :write [11 49] INFO [2019-12-17 12:28:06,780] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [13 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,780] jepsen worker 73 - jepsen.util 793 :invoke :write [24 24] INFO [2019-12-17 12:28:06,784] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [5 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,784] jepsen worker 32 - jepsen.util 572 :info :write [10 41] :RequestCanceledException INFO [2019-12-17 12:28:06,785] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [25 33]] INFO [2019-12-17 12:28:06,786] jepsen worker 32 - jepsen.util 662 :invoke :cas [10 [36 28]] INFO [2019-12-17 12:28:06,801] jepsen worker 27 - jepsen.util 477 :ok :write [9 27] INFO [2019-12-17 12:28:06,802] jepsen worker 27 - jepsen.util 477 :invoke :write [9 37] INFO [2019-12-17 12:28:06,821] jepsen worker 0 - jepsen.util 720 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:06,821] jepsen worker 0 - jepsen.util 720 :invoke :cas [0 [28 23]] INFO [2019-12-17 12:28:06,821] jepsen worker 59 - jepsen.util 599 :info :cas [19 [35 17]] :RequestCanceledException INFO [2019-12-17 12:28:06,823] jepsen worker 59 - jepsen.util 689 :invoke :write [19 1] INFO [2019-12-17 12:28:06,832] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [12 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,832] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [21 20]] INFO [2019-12-17 12:28:06,845] jepsen worker 32 - jepsen.util 662 :fail :cas [10 [36 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,845] jepsen worker 32 - jepsen.util 662 :invoke :write [10 17] INFO [2019-12-17 12:28:06,850] jepsen worker 64 - jepsen.util 424 :ok :write [21 31] INFO [2019-12-17 12:28:06,851] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [0 37]] INFO [2019-12-17 12:28:06,861] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [25 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,861] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3185,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:06,862] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3186,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:06,883] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [47 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,884] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [26 47]] INFO [2019-12-17 12:28:06,887] jepsen worker 52 - jepsen.util 592 :info :write [17 20] :RequestCanceledException INFO [2019-12-17 12:28:06,887] jepsen worker 31 - jepsen.util 751 :info :cas [10 [15 41]] :RequestCanceledException INFO [2019-12-17 12:28:06,888] jepsen worker 52 - jepsen.util 682 :invoke :write [17 10] INFO [2019-12-17 12:28:06,889] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [13 38]] INFO [2019-12-17 12:28:06,896] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [0 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,896] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [2 14]] INFO [2019-12-17 12:28:06,908] jepsen worker 13 - jepsen.util 733 :fail :cas [4 [3 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,908] jepsen worker 13 - jepsen.util 733 :invoke :cas [4 [34 48]] INFO [2019-12-17 12:28:06,912] jepsen worker 54 - jepsen.util 774 :fail :cas [18 [35 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,912] jepsen worker 54 - jepsen.util 774 :invoke :read [18 nil] INFO [2019-12-17 12:28:06,922] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:06,922] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [1 45]] INFO [2019-12-17 12:28:06,923] jepsen worker 48 - jepsen.util 498 :ok :write [16 17] INFO [2019-12-17 12:28:06,923] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [46 8]] INFO [2019-12-17 12:28:06,935] jepsen worker 12 - jepsen.util 642 :ok :read [4 4] INFO [2019-12-17 12:28:06,936] jepsen worker 12 - jepsen.util 642 :invoke :write [4 12] INFO [2019-12-17 12:28:06,936] jepsen worker 19 - jepsen.util 649 :ok :write [6 25] INFO [2019-12-17 12:28:06,937] jepsen worker 19 - jepsen.util 649 :invoke :write [6 23] INFO [2019-12-17 12:28:06,941] jepsen worker 30 - jepsen.util 570 :ok :write [10 35] INFO [2019-12-17 12:28:06,941] jepsen worker 30 - jepsen.util 570 :invoke :cas [10 [22 31]] INFO [2019-12-17 12:28:06,964] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [27 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,964] jepsen worker 17 - jepsen.util 557 :invoke :cas [5 [12 23]] INFO [2019-12-17 12:28:06,967] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [26 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,967] jepsen worker 28 - jepsen.util 298 :invoke :read [9 nil] INFO [2019-12-17 12:28:06,979] jepsen worker 42 - jepsen.util 942 :ok :read [14 27] INFO [2019-12-17 12:28:06,979] jepsen worker 42 - jepsen.util 942 :invoke :cas [14 [8 32]] INFO [2019-12-17 12:28:06,988] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [21 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:06,989] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [35 5]] INFO [2019-12-17 12:28:07,013] jepsen worker 6 - jepsen.util 636 :ok :read [2 15] INFO [2019-12-17 12:28:07,013] jepsen worker 6 - jepsen.util 636 :invoke :cas [2 [9 9]] INFO [2019-12-17 12:28:07,019] jepsen worker 18 - jepsen.util 738 :fail :cas [6 [46 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,019] jepsen worker 18 - jepsen.util 738 :invoke :read [6 nil] INFO [2019-12-17 12:28:07,021] jepsen worker 49 - jepsen.util 409 :info :write [16 12] :RequestCanceledException INFO [2019-12-17 12:28:07,021] jepsen worker 72 - jepsen.util 612 :info :write [24 35] :RequestCanceledException INFO [2019-12-17 12:28:07,022] jepsen worker 72 - jepsen.util 702 :invoke :write [24 39] INFO [2019-12-17 12:28:07,023] jepsen worker 49 - jepsen.util 499 :invoke :read [16 nil] INFO [2019-12-17 12:28:07,035] jepsen worker 8 - jepsen.util 638 :info :cas [2 [15 43]] :RequestCanceledException INFO [2019-12-17 12:28:07,040] jepsen worker 8 - jepsen.util 728 :invoke :cas [2 [21 43]] INFO [2019-12-17 12:28:07,050] jepsen worker 27 - jepsen.util 477 :ok :write [9 37] INFO [2019-12-17 12:28:07,050] jepsen worker 31 - jepsen.util 841 :fail :cas [10 [13 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,051] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:07,051] jepsen worker 31 - jepsen.util 841 :invoke :write [10 45] INFO [2019-12-17 12:28:07,051] jepsen worker 38 - jepsen.util 488 :info :write [12 13] :RequestCanceledException INFO [2019-12-17 12:28:07,051] jepsen worker 63 - jepsen.util 333 :info :cas [21 [24 4]] :RequestCanceledException INFO [2019-12-17 12:28:07,053] jepsen worker 63 - jepsen.util 423 :invoke :cas [21 [47 33]] INFO [2019-12-17 12:28:07,053] jepsen worker 38 - jepsen.util 578 :invoke :cas [12 [44 19]] INFO [2019-12-17 12:28:07,056] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [2 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,056] jepsen worker 64 - jepsen.util 424 :invoke :cas [21 [28 30]] INFO [2019-12-17 12:28:07,063] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [1 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,063] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:07,068] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.08203125}, :ns_1@172.28.128.126 {:progress 0.07553956834532372}, :ns_1@172.28.128.127 {:progress 0.1556420233463035}, :ns_1@172.28.128.128 {:progress 0.6264150943396227}} INFO [2019-12-17 12:28:07,069] jepsen nemesis - couchbase.util Rebalance check count: 18 INFO [2019-12-17 12:28:07,069] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:07,075] jepsen worker 28 - jepsen.util 298 :ok :read [9 37] INFO [2019-12-17 12:28:07,075] jepsen worker 28 - jepsen.util 298 :invoke :write [9 36] INFO [2019-12-17 12:28:07,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:07,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3206,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:07,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:07,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3209,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:07,092] jepsen worker 9 - jepsen.util 639 :fail :cas [3 [26 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,092] jepsen worker 9 - jepsen.util 639 :invoke :read [3 nil] INFO [2019-12-17 12:28:07,103] jepsen worker 75 - jepsen.util 525 :info :write [25 8] :RequestCanceledException INFO [2019-12-17 12:28:07,104] jepsen worker 75 - jepsen.util 615 :invoke :read [25 nil] INFO [2019-12-17 12:28:07,105] jepsen worker 64 - jepsen.util 424 :fail :cas [21 [28 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,105] jepsen worker 64 - jepsen.util 424 :invoke :write [21 26] INFO [2019-12-17 12:28:07,107] jepsen worker 13 - jepsen.util 733 :fail :cas [4 [34 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,107] jepsen worker 13 - jepsen.util 733 :invoke :write [4 17] INFO [2019-12-17 12:28:07,115] jepsen worker 27 - jepsen.util 477 :ok :read [9 37] INFO [2019-12-17 12:28:07,116] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [17 16]] INFO [2019-12-17 12:28:07,134] jepsen worker 30 - jepsen.util 570 :fail :cas [10 [22 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,134] jepsen worker 30 - jepsen.util 570 :invoke :write [10 39] INFO [2019-12-17 12:28:07,168] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:07,168] jepsen worker 44 - jepsen.util 494 :invoke :read [14 nil] INFO [2019-12-17 12:28:07,171] jepsen worker 75 - jepsen.util 615 :ok :read [25 2] INFO [2019-12-17 12:28:07,171] jepsen worker 75 - jepsen.util 615 :invoke :read [25 nil] INFO [2019-12-17 12:28:07,174] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [35 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,174] jepsen worker 51 - jepsen.util 591 :invoke :read [17 nil] INFO [2019-12-17 12:28:07,188] jepsen worker 53 - jepsen.util 683 :info :write [17 37] :RequestCanceledException INFO [2019-12-17 12:28:07,189] jepsen worker 53 - jepsen.util 773 :invoke :write [17 7] INFO [2019-12-17 12:28:07,197] jepsen worker 9 - jepsen.util 639 :ok :read [3 1] INFO [2019-12-17 12:28:07,197] jepsen worker 9 - jepsen.util 639 :invoke :cas [3 [49 17]] INFO [2019-12-17 12:28:07,198] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:07,198] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3233,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:07,199] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:07,199] jepsen worker 67 - jepsen.util 877 :info :write [22 33] :RequestCanceledException INFO [2019-12-17 12:28:07,199] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3234,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:07,199] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [17 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,199] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [42 5]] INFO [2019-12-17 12:28:07,201] jepsen worker 67 - jepsen.util 967 :invoke :write [22 19] INFO [2019-12-17 12:28:07,205] jepsen worker 85 - jepsen.util 625 :ok :write [28 38] INFO [2019-12-17 12:28:07,206] jepsen worker 85 - jepsen.util 625 :invoke :read [28 nil] INFO [2019-12-17 12:28:07,236] jepsen worker 68 - jepsen.util 428 :ok :read [22 10] INFO [2019-12-17 12:28:07,236] jepsen worker 68 - jepsen.util 428 :invoke :write [22 0] INFO [2019-12-17 12:28:07,237] jepsen worker 44 - jepsen.util 494 :ok :read [14 27] INFO [2019-12-17 12:28:07,238] jepsen worker 44 - jepsen.util 494 :invoke :cas [14 [29 10]] INFO [2019-12-17 12:28:07,252] jepsen worker 43 - jepsen.util 853 :info :write [14 18] :RequestCanceledException INFO [2019-12-17 12:28:07,254] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:07,256] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [42 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,256] jepsen worker 27 - jepsen.util 477 :invoke :write [9 45] INFO [2019-12-17 12:28:07,264] jepsen worker 28 - jepsen.util 298 :ok :write [9 36] INFO [2019-12-17 12:28:07,264] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [30 33]] INFO [2019-12-17 12:28:07,265] jepsen worker 84 - jepsen.util 444 :ok :read [28 38] INFO [2019-12-17 12:28:07,265] jepsen worker 84 - jepsen.util 444 :invoke :read [28 nil] INFO [2019-12-17 12:28:07,294] jepsen worker 76 - jepsen.util 706 :fail :cas [25 [10 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,294] jepsen worker 76 - jepsen.util 706 :invoke :cas [25 [9 25]] INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3245,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:07,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3246,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:07,308] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [30 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,309] jepsen worker 28 - jepsen.util 298 :invoke :cas [9 [19 10]] INFO [2019-12-17 12:28:07,315] jepsen worker 44 - jepsen.util 494 :fail :cas [14 [29 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,315] jepsen worker 44 - jepsen.util 494 :invoke :write [14 17] INFO [2019-12-17 12:28:07,333] jepsen worker 55 - jepsen.util 865 :fail :cas [18 [2 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,333] jepsen worker 55 - jepsen.util 865 :invoke :write [18 28] INFO [2019-12-17 12:28:07,335] jepsen worker 25 - jepsen.util 745 :info :write [8 36] :RequestCanceledException INFO [2019-12-17 12:28:07,335] jepsen worker 2 - jepsen.util 632 :ok :write [0 37] INFO [2019-12-17 12:28:07,335] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:07,336] jepsen worker 25 - jepsen.util 835 :invoke :cas [8 [29 31]] INFO [2019-12-17 12:28:07,353] jepsen worker 28 - jepsen.util 298 :fail :cas [9 [19 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,353] jepsen worker 28 - jepsen.util 298 :invoke :write [9 32] INFO [2019-12-17 12:28:07,356] jepsen worker 73 - jepsen.util 793 :ok :write [24 24] INFO [2019-12-17 12:28:07,356] jepsen worker 73 - jepsen.util 793 :invoke :write [24 2] INFO [2019-12-17 12:28:07,383] jepsen worker 63 - jepsen.util 423 :fail :cas [21 [47 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,383] jepsen worker 1 - jepsen.util 631 :fail :cas [0 [33 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,383] jepsen worker 63 - jepsen.util 423 :invoke :write [21 25] INFO [2019-12-17 12:28:07,383] jepsen worker 1 - jepsen.util 631 :invoke :write [0 4] INFO [2019-12-17 12:28:07,399] jepsen worker 41 - jepsen.util 401 :ok :read [13 36] INFO [2019-12-17 12:28:07,399] jepsen worker 75 - jepsen.util 615 :ok :read [25 2] INFO [2019-12-17 12:28:07,399] jepsen worker 41 - jepsen.util 401 :invoke :cas [13 [26 3]] INFO [2019-12-17 12:28:07,400] jepsen worker 75 - jepsen.util 615 :invoke :write [25 42] INFO [2019-12-17 12:28:07,409] jepsen worker 48 - jepsen.util 498 :fail :cas [16 [46 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,409] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:07,418] jepsen worker 47 - jepsen.util 857 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:28:07,418] jepsen worker 71 - jepsen.util 521 :info :write [23 16] :RequestCanceledException INFO [2019-12-17 12:28:07,421] jepsen worker 47 - jepsen.util 947 :invoke :cas [15 [0 25]] INFO [2019-12-17 12:28:07,421] jepsen worker 71 - jepsen.util 611 :invoke :write [23 22] INFO [2019-12-17 12:28:07,421] jepsen worker 23 - jepsen.util 563 :ok :read [7 20] INFO [2019-12-17 12:28:07,421] jepsen worker 23 - jepsen.util 563 :invoke :cas [7 [41 33]] INFO [2019-12-17 12:28:07,425] jepsen worker 17 - jepsen.util 557 :fail :cas [5 [12 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,425] jepsen worker 17 - jepsen.util 557 :invoke :write [5 49] INFO [2019-12-17 12:28:07,443] jepsen worker 27 - jepsen.util 477 :ok :write [9 45] INFO [2019-12-17 12:28:07,444] jepsen worker 27 - jepsen.util 477 :invoke :write [9 6] INFO [2019-12-17 12:28:07,453] jepsen worker 85 - jepsen.util 625 :ok :read [28 38] INFO [2019-12-17 12:28:07,454] jepsen worker 85 - jepsen.util 625 :invoke :write [28 26] INFO [2019-12-17 12:28:07,454] jepsen worker 47 - jepsen.util 947 :fail :cas [15 [0 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,454] jepsen worker 47 - jepsen.util 947 :invoke :write [15 47] INFO [2019-12-17 12:28:07,456] jepsen worker 51 - jepsen.util 591 :ok :read [17 0] INFO [2019-12-17 12:28:07,457] jepsen worker 51 - jepsen.util 591 :invoke :read [17 nil] INFO [2019-12-17 12:28:07,476] jepsen worker 26 - jepsen.util 836 :fail :cas [8 [46 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,476] jepsen worker 26 - jepsen.util 836 :invoke :read [8 nil] INFO [2019-12-17 12:28:07,478] jepsen worker 29 - jepsen.util 569 :info :write [9 48] :RequestCanceledException INFO [2019-12-17 12:28:07,479] jepsen worker 29 - jepsen.util 659 :invoke :read [9 nil] INFO [2019-12-17 12:28:07,496] jepsen worker 87 - jepsen.util 537 :info :write [29 30] :RequestCanceledException INFO [2019-12-17 12:28:07,498] jepsen worker 87 - jepsen.util 627 :invoke :read [29 nil] INFO [2019-12-17 12:28:07,508] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:07,508] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3273,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:07,523] jepsen worker 68 - jepsen.util 428 :ok :write [22 0] INFO [2019-12-17 12:28:07,523] jepsen worker 68 - jepsen.util 428 :invoke :cas [22 [29 30]] INFO [2019-12-17 12:28:07,527] jepsen worker 31 - jepsen.util 841 :ok :write [10 45] INFO [2019-12-17 12:28:07,528] jepsen worker 31 - jepsen.util 841 :invoke :read [10 nil] INFO [2019-12-17 12:28:07,545] jepsen worker 69 - jepsen.util 699 :info :write [23 43] :RequestCanceledException INFO [2019-12-17 12:28:07,548] jepsen worker 69 - jepsen.util 789 :invoke :cas [23 [3 10]] INFO [2019-12-17 12:28:07,555] jepsen worker 31 - jepsen.util 841 :ok :read [10 45] INFO [2019-12-17 12:28:07,555] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [16 46]] INFO [2019-12-17 12:28:07,566] jepsen worker 27 - jepsen.util 477 :ok :write [9 6] INFO [2019-12-17 12:28:07,566] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [29 37]] INFO [2019-12-17 12:28:07,584] jepsen worker 61 - jepsen.util 781 :fail :cas [20 [35 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,584] jepsen worker 61 - jepsen.util 781 :invoke :cas [20 [19 13]] INFO [2019-12-17 12:28:07,584] jepsen worker 31 - jepsen.util 841 :fail :cas [10 [16 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,584] jepsen worker 31 - jepsen.util 841 :invoke :write [10 38] INFO [2019-12-17 12:28:07,587] jepsen worker 50 - jepsen.util 590 :info :cas [16 [3 30]] :RequestCanceledException INFO [2019-12-17 12:28:07,589] jepsen worker 50 - jepsen.util 680 :invoke :cas [16 [2 17]] INFO [2019-12-17 12:28:07,598] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,599] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [3 0]] INFO [2019-12-17 12:28:07,617] jepsen worker 77 - jepsen.util 617 :ok :read [25 2] INFO [2019-12-17 12:28:07,618] jepsen worker 77 - jepsen.util 617 :invoke :read [25 nil] INFO [2019-12-17 12:28:07,621] jepsen worker 46 - jepsen.util 586 :info :write [15 23] :RequestCanceledException INFO [2019-12-17 12:28:07,623] jepsen worker 46 - jepsen.util 676 :invoke :cas [15 [21 28]] INFO [2019-12-17 12:28:07,635] jepsen worker 56 - jepsen.util 596 :info :write [18 12] :RequestCanceledException INFO [2019-12-17 12:28:07,637] jepsen worker 56 - jepsen.util 686 :invoke :read [18 nil] INFO [2019-12-17 12:28:07,643] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [3 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,643] jepsen worker 27 - jepsen.util 477 :invoke :write [9 0] INFO [2019-12-17 12:28:07,644] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:07,644] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:07,660] jepsen worker 46 - jepsen.util 676 :fail :cas [15 [21 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,661] jepsen worker 46 - jepsen.util 676 :invoke :read [15 nil] INFO [2019-12-17 12:28:07,686] jepsen worker 46 - jepsen.util 676 :ok :read [15 35] INFO [2019-12-17 12:28:07,686] jepsen worker 46 - jepsen.util 676 :invoke :cas [15 [16 4]] INFO [2019-12-17 12:28:07,712] jepsen worker 27 - jepsen.util 477 :ok :write [9 0] INFO [2019-12-17 12:28:07,712] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:07,714] jepsen worker 46 - jepsen.util 676 :fail :cas [15 [16 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,715] jepsen worker 46 - jepsen.util 676 :invoke :read [15 nil] INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3298,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:07,739] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3299,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:07,743] jepsen worker 27 - jepsen.util 477 :ok :read [9 0] INFO [2019-12-17 12:28:07,744] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [25 13]] INFO [2019-12-17 12:28:07,745] jepsen worker 46 - jepsen.util 676 :ok :read [15 35] INFO [2019-12-17 12:28:07,745] jepsen worker 46 - jepsen.util 676 :invoke :cas [15 [3 19]] INFO [2019-12-17 12:28:07,774] jepsen worker 23 - jepsen.util 563 :fail :cas [7 [41 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,774] jepsen worker 23 - jepsen.util 563 :invoke :read [7 nil] INFO [2019-12-17 12:28:07,780] jepsen worker 46 - jepsen.util 676 :fail :cas [15 [3 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,780] jepsen worker 46 - jepsen.util 676 :invoke :write [15 1] INFO [2019-12-17 12:28:07,787] jepsen worker 80 - jepsen.util 800 :info :write [26 37] :RequestCanceledException INFO [2019-12-17 12:28:07,787] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [25 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,787] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [29 12]] INFO [2019-12-17 12:28:07,789] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:07,800] jepsen worker 51 - jepsen.util 591 :ok :read [17 0] INFO [2019-12-17 12:28:07,800] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [35 3]] INFO [2019-12-17 12:28:07,806] jepsen worker 20 - jepsen.util 650 :info :cas [6 [4 0]] :RequestCanceledException INFO [2019-12-17 12:28:07,806] jepsen worker 74 - jepsen.util 524 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:07,806] jepsen worker 62 - jepsen.util 782 :info :write [20 39] :RequestCanceledException INFO [2019-12-17 12:28:07,806] jepsen worker 74 - jepsen.util 524 :invoke :cas [24 [36 47]] INFO [2019-12-17 12:28:07,807] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [17 40]] INFO [2019-12-17 12:28:07,808] jepsen worker 62 - jepsen.util 872 :invoke :read [20 nil] INFO [2019-12-17 12:28:07,810] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [29 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,810] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [6 18]] INFO [2019-12-17 12:28:07,825] jepsen worker 6 - jepsen.util 636 :fail :cas [2 [9 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,825] jepsen worker 6 - jepsen.util 636 :invoke :write [2 37] INFO [2019-12-17 12:28:07,856] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [6 18]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,856] jepsen worker 27 - jepsen.util 477 :invoke :write [9 33] INFO [2019-12-17 12:28:07,862] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [17 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,862] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [47 10]] INFO [2019-12-17 12:28:07,879] jepsen worker 25 - jepsen.util 835 :fail :cas [8 [29 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,879] jepsen worker 25 - jepsen.util 835 :invoke :write [8 6] INFO [2019-12-17 12:28:07,904] jepsen worker 87 - jepsen.util 627 :ok :read [29 5] INFO [2019-12-17 12:28:07,904] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,904] jepsen worker 23 - jepsen.util 563 :ok :read [7 20] INFO [2019-12-17 12:28:07,904] jepsen worker 87 - jepsen.util 627 :invoke :write [29 10] INFO [2019-12-17 12:28:07,904] jepsen worker 20 - jepsen.util 740 :invoke :read [6 nil] INFO [2019-12-17 12:28:07,904] jepsen worker 23 - jepsen.util 563 :invoke :cas [7 [21 8]] INFO [2019-12-17 12:28:07,907] jepsen worker 42 - jepsen.util 942 :fail :cas [14 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,907] jepsen worker 42 - jepsen.util 942 :invoke :write [14 11] INFO [2019-12-17 12:28:07,919] jepsen worker 5 - jepsen.util 365 :info :write [1 46] :RequestCanceledException INFO [2019-12-17 12:28:07,921] jepsen worker 5 - jepsen.util 455 :invoke :write [1 49] INFO [2019-12-17 12:28:07,931] jepsen worker 20 - jepsen.util 740 :ok :read [6 25] INFO [2019-12-17 12:28:07,931] jepsen worker 20 - jepsen.util 740 :invoke :write [6 26] INFO [2019-12-17 12:28:07,943] jepsen worker 27 - jepsen.util 477 :ok :write [9 33] INFO [2019-12-17 12:28:07,943] jepsen worker 27 - jepsen.util 477 :invoke :write [9 1] INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3323,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:07,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3327,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:07,949] jepsen worker 33 - jepsen.util 663 :info :write [11 33] :RequestCanceledException INFO [2019-12-17 12:28:07,951] jepsen worker 33 - jepsen.util 753 :invoke :write [11 19] INFO [2019-12-17 12:28:07,967] jepsen worker 80 - jepsen.util 890 :ok :read [26 47] INFO [2019-12-17 12:28:07,968] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:07,991] jepsen worker 8 - jepsen.util 728 :fail :cas [2 [21 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:07,991] jepsen worker 8 - jepsen.util 728 :invoke :cas [2 [12 25]] INFO [2019-12-17 12:28:08,021] jepsen worker 21 - jepsen.util 741 :info :write [7 28] :RequestCanceledException INFO [2019-12-17 12:28:08,021] jepsen worker 60 - jepsen.util 690 :info :write [20 15] :RequestCanceledException INFO [2019-12-17 12:28:08,021] jepsen worker 10 - jepsen.util 640 :info :write [3 35] :RequestCanceledException INFO [2019-12-17 12:28:08,022] jepsen worker 54 - jepsen.util 774 :ok :read [18 47] INFO [2019-12-17 12:28:08,023] jepsen worker 54 - jepsen.util 774 :invoke :read [18 nil] INFO [2019-12-17 12:28:08,023] jepsen worker 21 - jepsen.util 831 :invoke :cas [7 [49 46]] INFO [2019-12-17 12:28:08,023] jepsen worker 60 - jepsen.util 780 :invoke :write [20 40] INFO [2019-12-17 12:28:08,023] jepsen worker 10 - jepsen.util 730 :invoke :read [3 nil] INFO [2019-12-17 12:28:08,034] jepsen worker 27 - jepsen.util 477 :ok :write [9 1] INFO [2019-12-17 12:28:08,034] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [38 44]] INFO [2019-12-17 12:28:08,035] jepsen worker 65 - jepsen.util 515 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,035] jepsen worker 65 - jepsen.util 515 :invoke :cas [21 [42 3]] INFO [2019-12-17 12:28:08,052] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:08,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3340,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:08,053] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:08,053] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:08,061] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [38 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,061] jepsen worker 27 - jepsen.util 477 :invoke :write [9 48] INFO [2019-12-17 12:28:08,062] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:08,063] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:08,064] jepsen worker 23 - jepsen.util 563 :fail :cas [7 [21 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,064] jepsen worker 23 - jepsen.util 563 :invoke :read [7 nil] INFO [2019-12-17 12:28:08,081] jepsen worker 51 - jepsen.util 591 :fail :cas [17 [35 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,081] jepsen worker 51 - jepsen.util 591 :invoke :cas [17 [0 30]] INFO [2019-12-17 12:28:08,082] jepsen worker 76 - jepsen.util 706 :fail :cas [25 [9 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,082] jepsen worker 76 - jepsen.util 706 :invoke :cas [25 [6 46]] INFO [2019-12-17 12:28:08,087] jepsen worker 82 - jepsen.util 802 :info :cas [27 [3 15]] :RequestCanceledException INFO [2019-12-17 12:28:08,088] jepsen worker 82 - jepsen.util 892 :invoke :cas [27 [24 35]] INFO [2019-12-17 12:28:08,099] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.08984375}, :ns_1@172.28.128.126 {:progress 0.08273381294964033}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:08,099] jepsen nemesis - couchbase.util Rebalance check count: 19 INFO [2019-12-17 12:28:08,099] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:08,106] jepsen worker 15 - jepsen.util 645 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,106] jepsen worker 15 - jepsen.util 645 :invoke :write [5 39] INFO [2019-12-17 12:28:08,121] jepsen worker 34 - jepsen.util 664 :info :write [11 29] :RequestCanceledException INFO [2019-12-17 12:28:08,121] jepsen worker 16 - jepsen.util 736 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,121] jepsen worker 16 - jepsen.util 736 :invoke :cas [5 [8 16]] INFO [2019-12-17 12:28:08,122] jepsen worker 34 - jepsen.util 754 :invoke :read [11 nil] INFO [2019-12-17 12:28:08,135] jepsen worker 11 - jepsen.util 731 :info :cas [3 [38 16]] :RequestCanceledException INFO [2019-12-17 12:28:08,136] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [46 39]] INFO [2019-12-17 12:28:08,142] jepsen worker 61 - jepsen.util 781 :fail :cas [20 [19 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,142] jepsen worker 61 - jepsen.util 781 :invoke :cas [20 [35 12]] INFO [2019-12-17 12:28:08,147] jepsen worker 27 - jepsen.util 477 :ok :write [9 48] INFO [2019-12-17 12:28:08,147] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [9 46]] INFO [2019-12-17 12:28:08,155] jepsen worker 56 - jepsen.util 686 :ok :read [18 47] INFO [2019-12-17 12:28:08,156] jepsen worker 56 - jepsen.util 686 :invoke :cas [18 [20 32]] INFO [2019-12-17 12:28:08,164] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [46 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,165] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [10 6]] INFO [2019-12-17 12:28:08,170] jepsen worker 77 - jepsen.util 617 :ok :read [25 2] INFO [2019-12-17 12:28:08,171] jepsen worker 77 - jepsen.util 617 :invoke :read [25 nil] INFO [2019-12-17 12:28:08,177] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [9 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,177] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [36 44]] INFO [2019-12-17 12:28:08,180] jepsen worker 88 - jepsen.util 538 :info :write [29 14] :RequestCanceledException INFO [2019-12-17 12:28:08,182] jepsen worker 88 - jepsen.util 628 :invoke :read [29 nil] INFO [2019-12-17 12:28:08,202] jepsen worker 40 - jepsen.util 850 :info :cas [13 [32 3]] :RequestCanceledException INFO [2019-12-17 12:28:08,204] jepsen worker 40 - jepsen.util 940 :invoke :cas [13 [38 28]] INFO [2019-12-17 12:28:08,221] jepsen worker 66 - jepsen.util 516 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,221] jepsen worker 66 - jepsen.util 516 :invoke :write [22 36] INFO [2019-12-17 12:28:08,263] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [10 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,263] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [27 2]] INFO [2019-12-17 12:28:08,267] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [36 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,267] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3378,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:08,273] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3382,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:08,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:08,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3369,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:08,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:08,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3371,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:08,286] jepsen worker 61 - jepsen.util 781 :fail :cas [20 [35 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,286] jepsen worker 61 - jepsen.util 781 :invoke :write [20 18] INFO [2019-12-17 12:28:08,303] jepsen worker 26 - jepsen.util 836 :ok :read [8 14] INFO [2019-12-17 12:28:08,303] jepsen worker 26 - jepsen.util 836 :invoke :cas [8 [0 3]] INFO [2019-12-17 12:28:08,315] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [27 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,316] jepsen worker 11 - jepsen.util 821 :invoke :cas [3 [42 45]] INFO [2019-12-17 12:28:08,320] jepsen worker 27 - jepsen.util 477 :ok :read [9 48] INFO [2019-12-17 12:28:08,320] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:08,335] jepsen worker 78 - jepsen.util 708 :info :write [26 39] :RequestCanceledException INFO [2019-12-17 12:28:08,337] jepsen worker 78 - jepsen.util 798 :invoke :cas [26 [33 7]] INFO [2019-12-17 12:28:08,352] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [12 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,352] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [26 48]] INFO [2019-12-17 12:28:08,355] jepsen worker 11 - jepsen.util 821 :fail :cas [3 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,355] jepsen worker 11 - jepsen.util 821 :invoke :read [3 nil] INFO [2019-12-17 12:28:08,364] jepsen worker 21 - jepsen.util 831 :fail :cas [7 [49 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,365] jepsen worker 21 - jepsen.util 831 :invoke :read [7 nil] INFO [2019-12-17 12:28:08,368] jepsen worker 27 - jepsen.util 477 :ok :read [9 48] INFO [2019-12-17 12:28:08,369] jepsen worker 27 - jepsen.util 477 :invoke :write [9 43] INFO [2019-12-17 12:28:08,387] jepsen worker 11 - jepsen.util 821 :ok :read [3 1] INFO [2019-12-17 12:28:08,388] jepsen worker 11 - jepsen.util 821 :invoke :write [3 1] INFO [2019-12-17 12:28:08,405] jepsen worker 36 - jepsen.util 756 :info :write [12 37] :RequestCanceledException INFO [2019-12-17 12:28:08,428] jepsen worker 36 - jepsen.util 846 :invoke :cas [12 [6 4]] INFO [2019-12-17 12:28:08,449] jepsen worker 27 - jepsen.util 477 :ok :write [9 43] INFO [2019-12-17 12:28:08,449] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:08,501] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:08,501] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3399,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:08,502] jepsen worker 81 - jepsen.util 711 :info :write [27 11] :RequestCanceledException INFO [2019-12-17 12:28:08,510] jepsen worker 81 - jepsen.util 801 :invoke :write [27 12] INFO [2019-12-17 12:28:08,527] jepsen worker 86 - jepsen.util 536 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,531] jepsen worker 86 - jepsen.util 536 :invoke :write [28 47] INFO [2019-12-17 12:28:08,537] jepsen worker 45 - jepsen.util 945 :info :write [15 14] :RequestCanceledException INFO [2019-12-17 12:28:08,560] jepsen worker 37 - jepsen.util 667 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:28:08,594] jepsen worker 83 - jepsen.util 803 :info :write [27 1] :RequestCanceledException INFO [2019-12-17 12:28:08,596] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 31] INFO [2019-12-17 12:28:08,602] jepsen worker 26 - jepsen.util 836 :fail :cas [8 [0 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,635] jepsen worker 26 - jepsen.util 836 :invoke :read [8 nil] INFO [2019-12-17 12:28:08,637] jepsen worker 37 - jepsen.util 757 :invoke :write [12 45] INFO [2019-12-17 12:28:08,639] jepsen worker 89 - jepsen.util 809 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:28:08,667] jepsen worker 22 - jepsen.util 652 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,667] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:08,676] jepsen worker 89 - jepsen.util 899 :invoke :write [29 48] INFO [2019-12-17 12:28:08,678] jepsen worker 83 - jepsen.util 893 :invoke :read [27 nil] INFO [2019-12-17 12:28:08,734] jepsen worker 39 - jepsen.util 759 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,739] jepsen worker 39 - jepsen.util 759 :invoke :read [13 nil] INFO [2019-12-17 12:28:08,759] jepsen worker 3 - jepsen.util 453 :fail :read [1 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,761] jepsen worker 14 - jepsen.util 734 :fail :read [4 nil] :RequestTimeoutException INFO [2019-12-17 12:28:08,774] jepsen worker 3 - jepsen.util 453 :invoke :cas [1 [25 25]] INFO [2019-12-17 12:28:08,774] jepsen worker 14 - jepsen.util 734 :invoke :write [4 49] INFO [2019-12-17 12:28:08,776] jepsen worker 7 - jepsen.util 817 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:28:08,832] jepsen worker 24 - jepsen.util 564 :info :write [8 19] :RequestCanceledException INFO [2019-12-17 12:28:08,832] jepsen worker 58 - jepsen.util 508 :info :cas [19 [10 9]] :RequestCanceledException INFO [2019-12-17 12:28:08,958] jepsen worker 0 - jepsen.util 720 :info :cas [0 [28 23]] :RequestCanceledException INFO [2019-12-17 12:28:08,963] jepsen worker 62 - jepsen.util 872 :ok :read [20 7] INFO [2019-12-17 12:28:08,972] jepsen worker 32 - jepsen.util 662 :info :write [10 17] :RequestCanceledException INFO [2019-12-17 12:28:08,974] jepsen worker 78 - jepsen.util 798 :fail :cas [26 [33 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:08,975] jepsen worker 57 - jepsen.util 687 :info :cas [19 [23 32]] :RequestCanceledException INFO [2019-12-17 12:28:08,981] jepsen worker 35 - jepsen.util 575 :info :write [11 49] :RequestCanceledException INFO [2019-12-17 12:28:08,978] jepsen worker 59 - jepsen.util 689 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:28:08,993] jepsen worker 52 - jepsen.util 682 :info :write [17 10] :RequestCanceledException INFO [2019-12-17 12:28:08,999] jepsen worker 70 - jepsen.util 880 :fail :cas [23 [33 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,012] jepsen worker 78 - jepsen.util 798 :invoke :cas [26 [24 37]] INFO [2019-12-17 12:28:09,012] jepsen worker 70 - jepsen.util 880 :invoke :cas [23 [3 36]] INFO [2019-12-17 12:28:09,013] jepsen worker 62 - jepsen.util 872 :invoke :cas [20 [22 15]] INFO [2019-12-17 12:28:09,027] jepsen worker 24 - jepsen.util 654 :invoke :read [8 nil] INFO [2019-12-17 12:28:09,028] jepsen worker 7 - jepsen.util 907 :invoke :cas [2 [48 11]] INFO [2019-12-17 12:28:09,032] jepsen worker 52 - jepsen.util 772 :invoke :read [17 nil] INFO [2019-12-17 12:28:09,054] jepsen worker 59 - jepsen.util 779 :invoke :read [19 nil] INFO [2019-12-17 12:28:09,054] jepsen worker 58 - jepsen.util 598 :invoke :write [19 1] INFO [2019-12-17 12:28:09,054] jepsen worker 0 - jepsen.util 810 :invoke :write [0 43] INFO [2019-12-17 12:28:09,073] jepsen worker 12 - jepsen.util 642 :info :write [4 12] :RequestCanceledException INFO [2019-12-17 12:28:09,092] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,107] jepsen worker 19 - jepsen.util 649 :info :write [6 23] :RequestCanceledException INFO [2019-12-17 12:28:09,128] jepsen worker 49 - jepsen.util 499 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,128] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3426,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,131] jepsen worker 31 - jepsen.util 841 :ok :write [10 38] INFO [2019-12-17 12:28:09,131] jepsen worker 35 - jepsen.util 665 :invoke :read [11 nil] INFO [2019-12-17 12:28:09,137] jepsen worker 38 - jepsen.util 578 :info :cas [12 [44 19]] :RequestCanceledException INFO [2019-12-17 12:28:09,138] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,140] jepsen worker 57 - jepsen.util 777 :invoke :read [19 nil] INFO [2019-12-17 12:28:09,139] jepsen worker 73 - jepsen.util 793 :ok :write [24 2] INFO [2019-12-17 12:28:09,144] jepsen worker 49 - jepsen.util 499 :invoke :cas [16 [2 21]] INFO [2019-12-17 12:28:09,148] jepsen worker 20 - jepsen.util 740 :ok :write [6 26] INFO [2019-12-17 12:28:09,147] jepsen worker 18 - jepsen.util 738 :fail :read [6 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,148] jepsen worker 1 - jepsen.util 631 :ok :write [0 4] INFO [2019-12-17 12:28:09,156] jepsen worker 18 - jepsen.util 738 :invoke :read [6 nil] INFO [2019-12-17 12:28:09,158] jepsen worker 20 - jepsen.util 740 :invoke :write [6 32] INFO [2019-12-17 12:28:09,162] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3435,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,167] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [14 29]] INFO [2019-12-17 12:28:09,178] jepsen worker 72 - jepsen.util 702 :info :write [24 39] :RequestCanceledException INFO [2019-12-17 12:28:09,182] jepsen worker 1 - jepsen.util 631 :invoke :write [0 5] INFO [2019-12-17 12:28:09,183] jepsen worker 73 - jepsen.util 793 :invoke :write [24 42] INFO [2019-12-17 12:28:09,192] jepsen worker 64 - jepsen.util 424 :info :write [21 26] :RequestCanceledException INFO [2019-12-17 12:28:09,212] jepsen worker 32 - jepsen.util 752 :invoke :cas [10 [24 29]] INFO [2019-12-17 12:28:09,224] jepsen worker 12 - jepsen.util 732 :invoke :write [4 28] INFO [2019-12-17 12:28:09,232] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:09,234] jepsen worker 30 - jepsen.util 570 :info :write [10 39] :RequestCanceledException INFO [2019-12-17 12:28:09,237] jepsen worker 53 - jepsen.util 773 :info :write [17 7] :RequestCanceledException INFO [2019-12-17 12:28:09,250] jepsen worker 13 - jepsen.util 733 :info :write [4 17] :RequestCanceledException INFO [2019-12-17 12:28:09,301] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:09,318] jepsen worker 19 - jepsen.util 739 :invoke :write [6 21] INFO [2019-12-17 12:28:09,328] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3443,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:09,360] jepsen worker 38 - jepsen.util 668 :invoke :cas [12 [11 23]] INFO [2019-12-17 12:28:09,370] jepsen worker 67 - jepsen.util 967 :info :write [22 19] :RequestCanceledException INFO [2019-12-17 12:28:09,392] jepsen worker 9 - jepsen.util 639 :info :cas [3 [49 17]] :RequestCanceledException INFO [2019-12-17 12:28:09,432] jepsen worker 84 - jepsen.util 444 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,433] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [19 45]] INFO [2019-12-17 12:28:09,432] jepsen worker 75 - jepsen.util 615 :info :write [25 42] :RequestCanceledException INFO [2019-12-17 12:28:09,440] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [6 49]] INFO [2019-12-17 12:28:09,441] jepsen worker 53 - jepsen.util 863 :invoke :write [17 46] INFO [2019-12-17 12:28:09,441] jepsen worker 55 - jepsen.util 865 :info :write [18 28] :RequestCanceledException INFO [2019-12-17 12:28:09,443] jepsen worker 44 - jepsen.util 494 :info :write [14 17] :RequestCanceledException INFO [2019-12-17 12:28:09,450] jepsen worker 13 - jepsen.util 823 :invoke :write [4 31] INFO [2019-12-17 12:28:09,452] jepsen worker 84 - jepsen.util 444 :invoke :write [28 24] INFO [2019-12-17 12:28:09,457] jepsen worker 41 - jepsen.util 401 :info :cas [13 [26 3]] :RequestCanceledException INFO [2019-12-17 12:28:09,469] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,476] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:09,501] jepsen worker 28 - jepsen.util 298 :info :write [9 32] :RequestCanceledException INFO [2019-12-17 12:28:09,517] jepsen worker 27 - jepsen.util 477 :ok :read [9 43] INFO [2019-12-17 12:28:09,518] jepsen worker 27 - jepsen.util 477 :invoke :read [9 nil] INFO [2019-12-17 12:28:09,518] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3462,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:09,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3463,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:09,530] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [38 19]] INFO [2019-12-17 12:28:09,532] jepsen worker 63 - jepsen.util 423 :info :write [21 25] :RequestCanceledException INFO [2019-12-17 12:28:09,534] jepsen worker 80 - jepsen.util 890 :ok :read [26 47] INFO [2019-12-17 12:28:09,540] jepsen worker 17 - jepsen.util 557 :info :write [5 49] :RequestCanceledException INFO [2019-12-17 12:28:09,544] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [19 23]] INFO [2019-12-17 12:28:09,547] jepsen worker 47 - jepsen.util 947 :info :write [15 47] :RequestCanceledException INFO [2019-12-17 12:28:09,553] jepsen worker 71 - jepsen.util 611 :info :write [23 22] :RequestCanceledException INFO [2019-12-17 12:28:09,558] jepsen worker 30 - jepsen.util 660 :invoke :cas [10 [3 26]] INFO [2019-12-17 12:28:09,559] jepsen worker 80 - jepsen.util 890 :invoke :cas [26 [8 26]] INFO [2019-12-17 12:28:09,564] jepsen worker 75 - jepsen.util 705 :invoke :cas [25 [32 25]] INFO [2019-12-17 12:28:09,566] jepsen worker 85 - jepsen.util 625 :info :write [28 26] :RequestCanceledException INFO [2019-12-17 12:28:09,643] jepsen worker 55 - jepsen.util 955 :invoke :cas [18 [42 4]] INFO [2019-12-17 12:28:09,653] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:09,654] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3446,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:09,659] jepsen worker 79 - jepsen.util 619 :invoke :read [26 nil] INFO [2019-12-17 12:28:09,673] jepsen worker 28 - jepsen.util 388 :invoke :write [9 38] INFO [2019-12-17 12:28:09,682] jepsen worker 67 - jepsen.util 1057 :invoke :write [22 32] INFO [2019-12-17 12:28:09,705] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:09,723] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3455,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:09,730] jepsen worker 50 - jepsen.util 680 :info :cas [16 [2 17]] :RequestCanceledException INFO [2019-12-17 12:28:09,730] jepsen worker 44 - jepsen.util 584 :invoke :cas [14 [44 46]] INFO [2019-12-17 12:28:09,738] jepsen worker 71 - jepsen.util 701 :invoke :write [23 22] INFO [2019-12-17 12:28:09,739] jepsen worker 41 - jepsen.util 491 :invoke :write [13 10] INFO [2019-12-17 12:28:09,740] jepsen worker 63 - jepsen.util 513 :invoke :cas [21 [21 29]] INFO [2019-12-17 12:28:09,740] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:09,753] jepsen worker 17 - jepsen.util 647 :invoke :cas [5 [36 33]] INFO [2019-12-17 12:28:09,788] jepsen worker 85 - jepsen.util 715 :invoke :write [28 30] INFO [2019-12-17 12:28:09,810] jepsen worker 11 - jepsen.util 821 :ok :write [3 1] INFO [2019-12-17 12:28:09,817] jepsen worker 11 - jepsen.util 821 :invoke :write [3 8] INFO [2019-12-17 12:28:09,822] jepsen worker 46 - jepsen.util 676 :info :write [15 1] :RequestCanceledException INFO [2019-12-17 12:28:09,825] jepsen worker 50 - jepsen.util 770 :invoke :write [16 34] INFO [2019-12-17 12:28:09,867] jepsen worker 54 - jepsen.util 774 :ok :read [18 47] INFO [2019-12-17 12:28:09,868] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:09,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3494,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:09,869] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:09,869] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3498,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:09,870] jepsen worker 54 - jepsen.util 774 :invoke :cas [18 [24 37]] INFO [2019-12-17 12:28:09,875] jepsen worker 46 - jepsen.util 766 :invoke :read [15 nil] INFO [2019-12-17 12:28:09,905] jepsen worker 74 - jepsen.util 524 :info :cas [24 [36 47]] :RequestCanceledException INFO [2019-12-17 12:28:09,918] jepsen worker 6 - jepsen.util 636 :info :write [2 37] :RequestCanceledException INFO [2019-12-17 12:28:09,925] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [26 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,927] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:09,933] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [9 34]] INFO [2019-12-17 12:28:09,935] jepsen worker 6 - jepsen.util 726 :invoke :read [2 nil] INFO [2019-12-17 12:28:09,935] jepsen worker 42 - jepsen.util 942 :info :write [14 11] :RequestCanceledException INFO [2019-12-17 12:28:09,938] jepsen worker 69 - jepsen.util 789 :fail :cas [23 [3 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,939] jepsen worker 42 - jepsen.util 1032 :invoke :cas [14 [11 12]] INFO [2019-12-17 12:28:09,941] jepsen worker 8 - jepsen.util 728 :fail :cas [2 [12 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,943] jepsen worker 8 - jepsen.util 728 :invoke :read [2 nil] INFO [2019-12-17 12:28:09,944] jepsen worker 69 - jepsen.util 789 :invoke :read [23 nil] INFO [2019-12-17 12:28:09,985] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:09,986] jepsen nemesis - couchbase.util Rebalance check count: 20 INFO [2019-12-17 12:28:09,986] jepsen nemesis - couchbase.util Rebalance stuck count: 1 INFO [2019-12-17 12:28:09,989] jepsen worker 56 - jepsen.util 686 :fail :cas [18 [20 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:09,989] jepsen worker 56 - jepsen.util 686 :invoke :write [18 13] INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3518,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:09,993] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:09,996] jepsen worker 87 - jepsen.util 627 :info :write [29 10] :RequestCanceledException INFO [2019-12-17 12:28:09,996] jepsen worker 5 - jepsen.util 455 :info :write [1 49] :RequestCanceledException INFO [2019-12-17 12:28:09,996] jepsen worker 25 - jepsen.util 835 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:28:10,000] jepsen worker 21 - jepsen.util 831 :ok :read [7 20] INFO [2019-12-17 12:28:10,001] jepsen worker 25 - jepsen.util 925 :invoke :read [8 nil] INFO [2019-12-17 12:28:10,001] jepsen worker 21 - jepsen.util 831 :invoke :cas [7 [4 22]] INFO [2019-12-17 12:28:10,002] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [31 15]] INFO [2019-12-17 12:28:10,002] jepsen worker 87 - jepsen.util 717 :invoke :cas [29 [9 40]] INFO [2019-12-17 12:28:10,002] jepsen worker 76 - jepsen.util 706 :fail :cas [25 [6 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,003] jepsen worker 76 - jepsen.util 706 :invoke :write [25 37] INFO [2019-12-17 12:28:10,005] jepsen worker 33 - jepsen.util 753 :info :write [11 19] :RequestCanceledException INFO [2019-12-17 12:28:10,009] jepsen worker 33 - jepsen.util 843 :invoke :write [11 23] INFO [2019-12-17 12:28:10,048] jepsen worker 27 - jepsen.util 477 :ok :read [9 43] INFO [2019-12-17 12:28:10,050] jepsen worker 27 - jepsen.util 477 :invoke :cas [9 [20 21]] INFO [2019-12-17 12:28:10,053] jepsen worker 32 - jepsen.util 752 :fail :cas [10 [24 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,054] jepsen worker 32 - jepsen.util 752 :invoke :read [10 nil] INFO [2019-12-17 12:28:10,065] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:10,068] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:10,070] jepsen worker 88 - jepsen.util 628 :ok :read [29 5] INFO [2019-12-17 12:28:10,070] jepsen worker 88 - jepsen.util 628 :invoke :write [29 1] INFO [2019-12-17 12:28:10,081] jepsen worker 34 - jepsen.util 754 :ok :read [11 7] INFO [2019-12-17 12:28:10,081] jepsen worker 23 - jepsen.util 563 :ok :read [7 20] INFO [2019-12-17 12:28:10,082] jepsen worker 34 - jepsen.util 754 :invoke :write [11 25] INFO [2019-12-17 12:28:10,083] jepsen worker 68 - jepsen.util 428 :fail :cas [22 [29 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,084] jepsen worker 68 - jepsen.util 428 :invoke :read [22 nil] INFO [2019-12-17 12:28:10,086] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [19 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,086] jepsen worker 23 - jepsen.util 563 :invoke :cas [7 [33 40]] INFO [2019-12-17 12:28:10,086] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [0 48]] INFO [2019-12-17 12:28:10,087] jepsen worker 30 - jepsen.util 660 :fail :cas [10 [3 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,087] jepsen worker 30 - jepsen.util 660 :invoke :write [10 28] INFO [2019-12-17 12:28:10,087] jepsen worker 29 - jepsen.util 659 :ok :read [9 43] INFO [2019-12-17 12:28:10,088] jepsen worker 29 - jepsen.util 659 :invoke :read [9 nil] INFO [2019-12-17 12:28:10,094] jepsen worker 10 - jepsen.util 730 :ok :read [3 1] INFO [2019-12-17 12:28:10,095] jepsen worker 10 - jepsen.util 730 :invoke :write [3 37] INFO [2019-12-17 12:28:10,096] jepsen worker 36 - jepsen.util 846 :fail :cas [12 [6 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,098] jepsen worker 31 - jepsen.util 841 :fail :cas [10 [14 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,098] jepsen worker 36 - jepsen.util 846 :invoke :cas [12 [48 49]] INFO [2019-12-17 12:28:10,099] jepsen worker 31 - jepsen.util 841 :invoke :read [10 nil] INFO [2019-12-17 12:28:10,103] jepsen worker 65 - jepsen.util 515 :fail :cas [21 [42 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,104] jepsen worker 65 - jepsen.util 515 :invoke :cas [21 [44 48]] INFO [2019-12-17 12:28:10,105] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [6 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,105] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:10,107] jepsen worker 77 - jepsen.util 617 :ok :read [25 2] INFO [2019-12-17 12:28:10,107] jepsen worker 77 - jepsen.util 617 :invoke :cas [25 [3 29]] INFO [2019-12-17 12:28:10,117] jepsen worker 7 - jepsen.util 907 :fail :cas [2 [48 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,117] jepsen worker 7 - jepsen.util 907 :invoke :cas [2 [38 35]] INFO [2019-12-17 12:28:10,119] jepsen worker 82 - jepsen.util 892 :info :cas [27 [24 35]] :RequestCanceledException INFO [2019-12-17 12:28:10,123] jepsen worker 82 - jepsen.util 982 :invoke :cas [27 [7 19]] INFO [2019-12-17 12:28:10,128] jepsen worker 63 - jepsen.util 513 :fail :cas [21 [21 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,128] jepsen worker 63 - jepsen.util 513 :invoke :write [21 4] INFO [2019-12-17 12:28:10,133] jepsen worker 60 - jepsen.util 780 :info :write [20 40] :RequestCanceledException INFO [2019-12-17 12:28:10,136] jepsen worker 60 - jepsen.util 870 :invoke :write [20 15] INFO [2019-12-17 12:28:10,192] jepsen worker 46 - jepsen.util 766 :ok :read [15 35] INFO [2019-12-17 12:28:10,193] jepsen worker 46 - jepsen.util 766 :invoke :write [15 2] INFO [2019-12-17 12:28:10,196] jepsen worker 18 - jepsen.util 738 :ok :read [6 26] INFO [2019-12-17 12:28:10,196] jepsen worker 18 - jepsen.util 738 :invoke :write [6 48] INFO [2019-12-17 12:28:10,197] jepsen worker 15 - jepsen.util 645 :info :write [5 39] :RequestCanceledException INFO [2019-12-17 12:28:10,199] jepsen worker 15 - jepsen.util 735 :invoke :read [5 nil] INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3560,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:10,211] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3562,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:10,221] jepsen worker 16 - jepsen.util 736 :info :cas [5 [8 16]] :RequestCanceledException INFO [2019-12-17 12:28:10,223] jepsen worker 16 - jepsen.util 826 :invoke :read [5 nil] INFO [2019-12-17 12:28:10,234] jepsen worker 40 - jepsen.util 940 :info :cas [13 [38 28]] :RequestCanceledException INFO [2019-12-17 12:28:10,237] jepsen worker 40 - jepsen.util 1030 :invoke :write [13 23] INFO [2019-12-17 12:28:10,241] jepsen worker 26 - jepsen.util 836 :ok :read [8 14] INFO [2019-12-17 12:28:10,241] jepsen worker 26 - jepsen.util 836 :invoke :cas [8 [44 20]] INFO [2019-12-17 12:28:10,256] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:10,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3551,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:10,257] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:10,257] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3553,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:10,265] jepsen worker 32 - jepsen.util 752 :ok :read [10 38] INFO [2019-12-17 12:28:10,266] jepsen worker 32 - jepsen.util 752 :invoke :write [10 22] INFO [2019-12-17 12:28:10,275] jepsen worker 42 - jepsen.util 1032 :fail :cas [14 [11 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,275] jepsen worker 42 - jepsen.util 1032 :invoke :cas [14 [2 5]] INFO [2019-12-17 12:28:10,293] jepsen worker 55 - jepsen.util 955 :fail :cas [18 [42 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,293] jepsen worker 55 - jepsen.util 955 :invoke :cas [18 [15 4]] INFO [2019-12-17 12:28:10,319] jepsen worker 66 - jepsen.util 516 :info :write [22 36] :RequestCanceledException INFO [2019-12-17 12:28:10,321] jepsen worker 66 - jepsen.util 606 :invoke :cas [22 [18 7]] INFO [2019-12-17 12:28:10,329] jepsen worker 22 - jepsen.util 652 :ok :read [7 20] INFO [2019-12-17 12:28:10,330] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:10,338] jepsen worker 21 - jepsen.util 831 :fail :cas [7 [4 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,338] jepsen worker 21 - jepsen.util 831 :invoke :cas [7 [40 49]] INFO [2019-12-17 12:28:10,345] jepsen worker 2 - jepsen.util 632 :ok :read [0 4] INFO [2019-12-17 12:28:10,345] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:10,350] jepsen worker 61 - jepsen.util 781 :info :write [20 18] :RequestCanceledException INFO [2019-12-17 12:28:10,352] jepsen worker 61 - jepsen.util 871 :invoke :read [20 nil] INFO [2019-12-17 12:28:10,358] jepsen worker 31 - jepsen.util 841 :ok :read [10 38] INFO [2019-12-17 12:28:10,358] jepsen worker 31 - jepsen.util 841 :invoke :cas [10 [1 2]] INFO [2019-12-17 12:28:10,362] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [19 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,362] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:10,371] jepsen worker 62 - jepsen.util 872 :fail :cas [20 [22 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,371] jepsen worker 48 - jepsen.util 498 :fail :cas [16 [38 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,371] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:10,371] jepsen worker 62 - jepsen.util 872 :invoke :write [20 23] INFO [2019-12-17 12:28:10,377] jepsen worker 27 - jepsen.util 477 :fail :cas [9 [20 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,378] jepsen worker 27 - jepsen.util 477 :invoke :write [9 42] INFO [2019-12-17 12:28:10,391] jepsen worker 79 - jepsen.util 619 :ok :read [26 47] INFO [2019-12-17 12:28:10,391] jepsen worker 79 - jepsen.util 619 :invoke :write [26 43] INFO [2019-12-17 12:28:10,398] jepsen worker 6 - jepsen.util 726 :ok :read [2 15] INFO [2019-12-17 12:28:10,399] jepsen worker 6 - jepsen.util 726 :invoke :cas [2 [21 44]] INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:10,429] jepsen worker 3 - jepsen.util 453 :fail :cas [1 [25 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3585,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:10,429] jepsen worker 3 - jepsen.util 453 :invoke :write [1 19] INFO [2019-12-17 12:28:10,429] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3588,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:10,456] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [0 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,456] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:10,474] jepsen worker 77 - jepsen.util 617 :fail :cas [25 [3 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,474] jepsen worker 77 - jepsen.util 617 :invoke :cas [25 [2 3]] INFO [2019-12-17 12:28:10,491] jepsen worker 78 - jepsen.util 798 :fail :cas [26 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,491] jepsen worker 78 - jepsen.util 798 :invoke :write [26 4] INFO [2019-12-17 12:28:10,494] jepsen worker 9 - jepsen.util 729 :ok :read [3 1] INFO [2019-12-17 12:28:10,494] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:10,530] jepsen worker 35 - jepsen.util 665 :ok :read [11 7] INFO [2019-12-17 12:28:10,530] jepsen worker 35 - jepsen.util 665 :invoke :cas [11 [41 30]] INFO [2019-12-17 12:28:10,534] jepsen worker 12 - jepsen.util 732 :ok :write [4 28] INFO [2019-12-17 12:28:10,534] jepsen worker 12 - jepsen.util 732 :invoke :write [4 21] INFO [2019-12-17 12:28:10,540] jepsen worker 37 - jepsen.util 757 :ok :write [12 45] INFO [2019-12-17 12:28:10,541] jepsen worker 37 - jepsen.util 757 :invoke :cas [12 [11 21]] INFO [2019-12-17 12:28:10,551] jepsen worker 9 - jepsen.util 729 :ok :read [3 1] INFO [2019-12-17 12:28:10,557] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:10,563] jepsen worker 55 - jepsen.util 955 :fail :cas [18 [15 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,563] jepsen worker 55 - jepsen.util 955 :invoke :write [18 2] INFO [2019-12-17 12:28:10,587] jepsen worker 1 - jepsen.util 631 :ok :write [0 5] INFO [2019-12-17 12:28:10,587] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:10,589] jepsen worker 23 - jepsen.util 563 :fail :cas [7 [33 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,590] jepsen worker 23 - jepsen.util 563 :invoke :write [7 41] INFO [2019-12-17 12:28:10,614] jepsen worker 9 - jepsen.util 729 :ok :read [3 1] INFO [2019-12-17 12:28:10,614] jepsen worker 9 - jepsen.util 729 :invoke :write [3 14] INFO [2019-12-17 12:28:10,639] jepsen worker 75 - jepsen.util 705 :fail :cas [25 [32 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,639] jepsen worker 75 - jepsen.util 705 :invoke :write [25 41] INFO [2019-12-17 12:28:10,672] jepsen worker 89 - jepsen.util 899 :ok :write [29 48] INFO [2019-12-17 12:28:10,673] jepsen worker 89 - jepsen.util 899 :invoke :read [29 nil] INFO [2019-12-17 12:28:10,673] jepsen worker 14 - jepsen.util 734 :ok :write [4 49] INFO [2019-12-17 12:28:10,673] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:10,675] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:10,675] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3604,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:10,695] jepsen worker 42 - jepsen.util 1032 :fail :cas [14 [2 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,695] jepsen worker 42 - jepsen.util 1032 :invoke :read [14 nil] INFO [2019-12-17 12:28:10,698] jepsen worker 84 - jepsen.util 444 :ok :write [28 24] INFO [2019-12-17 12:28:10,698] jepsen worker 84 - jepsen.util 444 :invoke :write [28 28] INFO [2019-12-17 12:28:10,718] jepsen worker 80 - jepsen.util 890 :fail :cas [26 [8 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,718] jepsen worker 80 - jepsen.util 890 :invoke :cas [26 [48 22]] INFO [2019-12-17 12:28:10,720] jepsen worker 86 - jepsen.util 536 :info :write [28 47] :RequestCanceledException INFO [2019-12-17 12:28:10,721] jepsen worker 86 - jepsen.util 626 :invoke :cas [28 [25 46]] INFO [2019-12-17 12:28:10,730] jepsen worker 38 - jepsen.util 668 :fail :cas [12 [11 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,730] jepsen worker 38 - jepsen.util 668 :invoke :write [12 22] INFO [2019-12-17 12:28:10,746] jepsen worker 61 - jepsen.util 871 :ok :read [20 7] INFO [2019-12-17 12:28:10,747] jepsen worker 61 - jepsen.util 871 :invoke :cas [20 [12 39]] INFO [2019-12-17 12:28:10,750] jepsen worker 83 - jepsen.util 893 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:10,751] jepsen worker 83 - jepsen.util 893 :invoke :write [27 46] INFO [2019-12-17 12:28:10,778] jepsen worker 8 - jepsen.util 728 :ok :read [2 15] INFO [2019-12-17 12:28:10,778] jepsen worker 8 - jepsen.util 728 :invoke :write [2 41] INFO [2019-12-17 12:28:10,782] jepsen worker 81 - jepsen.util 801 :info :write [27 12] :RequestCanceledException INFO [2019-12-17 12:28:10,785] jepsen worker 81 - jepsen.util 891 :invoke :read [27 nil] INFO [2019-12-17 12:28:10,788] jepsen worker 80 - jepsen.util 890 :fail :cas [26 [48 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,788] jepsen worker 22 - jepsen.util 652 :ok :read [7 20] INFO [2019-12-17 12:28:10,788] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:10,789] jepsen worker 22 - jepsen.util 652 :invoke :read [7 nil] INFO [2019-12-17 12:28:10,792] jepsen worker 85 - jepsen.util 715 :ok :write [28 30] INFO [2019-12-17 12:28:10,792] jepsen worker 85 - jepsen.util 715 :invoke :cas [28 [1 34]] INFO [2019-12-17 12:28:10,794] jepsen worker 54 - jepsen.util 774 :fail :cas [18 [24 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,794] jepsen worker 54 - jepsen.util 774 :invoke :write [18 20] INFO [2019-12-17 12:28:10,819] jepsen worker 39 - jepsen.util 759 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:10,819] jepsen worker 39 - jepsen.util 759 :invoke :cas [13 [16 10]] INFO [2019-12-17 12:28:10,829] jepsen worker 53 - jepsen.util 863 :ok :write [17 46] INFO [2019-12-17 12:28:10,829] jepsen worker 53 - jepsen.util 863 :invoke :cas [17 [42 44]] INFO [2019-12-17 12:28:10,848] jepsen worker 89 - jepsen.util 899 :ok :read [29 48] INFO [2019-12-17 12:28:10,848] jepsen worker 89 - jepsen.util 899 :invoke :read [29 nil] INFO [2019-12-17 12:28:10,852] jepsen worker 68 - jepsen.util 428 :ok :read [22 0] INFO [2019-12-17 12:28:10,852] jepsen worker 68 - jepsen.util 428 :invoke :read [22 nil] INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3633,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:10,859] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3634,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:10,900] jepsen worker 24 - jepsen.util 654 :ok :read [8 14] INFO [2019-12-17 12:28:10,900] jepsen worker 24 - jepsen.util 654 :invoke :read [8 nil] INFO [2019-12-17 12:28:10,911] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:10,911] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:10,925] jepsen worker 21 - jepsen.util 831 :fail :cas [7 [40 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,925] jepsen worker 21 - jepsen.util 831 :invoke :write [7 39] INFO [2019-12-17 12:28:10,969] jepsen worker 45 - jepsen.util 1035 :ok :write [15 31] INFO [2019-12-17 12:28:10,969] jepsen worker 45 - jepsen.util 1035 :invoke :cas [15 [13 26]] INFO [2019-12-17 12:28:10,971] jepsen worker 20 - jepsen.util 740 :ok :write [6 32] INFO [2019-12-17 12:28:10,971] jepsen worker 20 - jepsen.util 740 :invoke :read [6 nil] INFO [2019-12-17 12:28:10,972] jepsen worker 73 - jepsen.util 793 :ok :write [24 42] INFO [2019-12-17 12:28:10,972] jepsen worker 6 - jepsen.util 726 :fail :cas [2 [21 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,972] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:10,972] jepsen worker 6 - jepsen.util 726 :invoke :cas [2 [17 41]] INFO [2019-12-17 12:28:10,979] jepsen worker 36 - jepsen.util 846 :fail :cas [12 [48 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,979] jepsen worker 36 - jepsen.util 846 :invoke :write [12 3] INFO [2019-12-17 12:28:10,986] jepsen worker 79 - jepsen.util 619 :ok :write [26 43] INFO [2019-12-17 12:28:10,986] jepsen worker 79 - jepsen.util 619 :invoke :write [26 12] INFO [2019-12-17 12:28:10,986] jepsen worker 80 - jepsen.util 890 :ok :read [26 43] INFO [2019-12-17 12:28:10,987] jepsen worker 80 - jepsen.util 890 :invoke :read [26 nil] INFO [2019-12-17 12:28:10,993] jepsen worker 7 - jepsen.util 907 :fail :cas [2 [38 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:10,994] jepsen worker 7 - jepsen.util 907 :invoke :write [2 7] INFO [2019-12-17 12:28:11,008] jepsen worker 74 - jepsen.util 614 :fail :cas [24 [9 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,008] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [13 35]] INFO [2019-12-17 12:28:11,011] jepsen worker 20 - jepsen.util 740 :ok :read [6 32] INFO [2019-12-17 12:28:11,012] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [45 17]] INFO [2019-12-17 12:28:11,014] jepsen worker 73 - jepsen.util 793 :ok :read [24 42] INFO [2019-12-17 12:28:11,014] jepsen worker 73 - jepsen.util 793 :invoke :write [24 11] INFO [2019-12-17 12:28:11,052] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:11,053] jepsen nemesis - couchbase.util Rebalance check count: 21 INFO [2019-12-17 12:28:11,053] jepsen nemesis - couchbase.util Rebalance stuck count: 2 INFO [2019-12-17 12:28:11,054] jepsen worker 4 - jepsen.util 634 :ok :read [1 49] INFO [2019-12-17 12:28:11,054] jepsen worker 4 - jepsen.util 634 :invoke :cas [1 [1 22]] INFO [2019-12-17 12:28:11,076] jepsen worker 47 - jepsen.util 1037 :ok :read [15 31] INFO [2019-12-17 12:28:11,076] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:11,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3643,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:11,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:11,081] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3647,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:11,087] jepsen worker 74 - jepsen.util 614 :fail :cas [24 [13 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,087] jepsen worker 74 - jepsen.util 614 :invoke :write [24 32] INFO [2019-12-17 12:28:11,096] jepsen worker 61 - jepsen.util 871 :fail :cas [20 [12 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,096] jepsen worker 61 - jepsen.util 871 :invoke :cas [20 [5 42]] INFO [2019-12-17 12:28:11,099] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [45 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,099] jepsen worker 20 - jepsen.util 740 :invoke :write [6 18] INFO [2019-12-17 12:28:11,100] jepsen worker 52 - jepsen.util 772 :fail :read [17 nil] :RequestTimeoutException INFO [2019-12-17 12:28:11,100] jepsen worker 70 - jepsen.util 880 :info :cas [23 [3 36]] :RequestCanceledException INFO [2019-12-17 12:28:11,101] jepsen worker 52 - jepsen.util 772 :invoke :read [17 nil] INFO [2019-12-17 12:28:11,102] jepsen worker 70 - jepsen.util 970 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,106] jepsen worker 44 - jepsen.util 584 :fail :cas [14 [44 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,106] jepsen worker 44 - jepsen.util 584 :invoke :cas [14 [32 46]] INFO [2019-12-17 12:28:11,114] jepsen worker 80 - jepsen.util 890 :ok :read [26 43] INFO [2019-12-17 12:28:11,114] jepsen worker 80 - jepsen.util 890 :invoke :write [26 43] INFO [2019-12-17 12:28:11,121] jepsen worker 59 - jepsen.util 779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:11,121] jepsen worker 59 - jepsen.util 779 :invoke :read [19 nil] INFO [2019-12-17 12:28:11,136] jepsen worker 45 - jepsen.util 1035 :fail :cas [15 [13 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,137] jepsen worker 45 - jepsen.util 1035 :invoke :cas [15 [13 11]] INFO [2019-12-17 12:28:11,157] jepsen worker 47 - jepsen.util 1037 :ok :read [15 31] INFO [2019-12-17 12:28:11,157] jepsen worker 47 - jepsen.util 1037 :invoke :cas [15 [28 40]] INFO [2019-12-17 12:28:11,175] jepsen worker 6 - jepsen.util 726 :fail :cas [2 [17 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,175] jepsen worker 6 - jepsen.util 726 :invoke :read [2 nil] INFO [2019-12-17 12:28:11,177] jepsen worker 45 - jepsen.util 1035 :fail :cas [15 [13 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,177] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,183] jepsen worker 57 - jepsen.util 777 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:11,184] jepsen worker 57 - jepsen.util 777 :invoke :cas [19 [4 49]] INFO [2019-12-17 12:28:11,185] jepsen worker 47 - jepsen.util 1037 :fail :cas [15 [28 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,185] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,197] jepsen worker 69 - jepsen.util 789 :ok :read [23 29] INFO [2019-12-17 12:28:11,197] jepsen worker 69 - jepsen.util 789 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,204] jepsen worker 45 - jepsen.util 1035 :ok :read [15 31] INFO [2019-12-17 12:28:11,205] jepsen worker 22 - jepsen.util 652 :ok :read [7 20] INFO [2019-12-17 12:28:11,208] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,208] jepsen worker 22 - jepsen.util 652 :invoke :write [7 16] INFO [2019-12-17 12:28:11,214] jepsen worker 71 - jepsen.util 701 :ok :write [23 22] INFO [2019-12-17 12:28:11,214] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,215] jepsen worker 47 - jepsen.util 1037 :ok :read [15 31] INFO [2019-12-17 12:28:11,215] jepsen worker 47 - jepsen.util 1037 :invoke :write [15 1] INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3672,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:11,220] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3673,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:11,249] jepsen worker 49 - jepsen.util 499 :info :cas [16 [2 21]] :RequestCanceledException INFO [2019-12-17 12:28:11,252] jepsen worker 49 - jepsen.util 589 :invoke :write [16 19] INFO [2019-12-17 12:28:11,254] jepsen worker 61 - jepsen.util 871 :fail :cas [20 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,254] jepsen worker 61 - jepsen.util 871 :invoke :write [20 27] INFO [2019-12-17 12:28:11,255] jepsen worker 87 - jepsen.util 717 :fail :cas [29 [9 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,255] jepsen worker 87 - jepsen.util 717 :invoke :cas [29 [17 27]] INFO [2019-12-17 12:28:11,265] jepsen worker 25 - jepsen.util 925 :ok :read [8 14] INFO [2019-12-17 12:28:11,265] jepsen worker 25 - jepsen.util 925 :invoke :cas [8 [45 48]] INFO [2019-12-17 12:28:11,280] jepsen worker 89 - jepsen.util 899 :ok :read [29 48] INFO [2019-12-17 12:28:11,280] jepsen worker 89 - jepsen.util 899 :invoke :write [29 49] INFO [2019-12-17 12:28:11,284] jepsen worker 45 - jepsen.util 1035 :ok :read [15 31] INFO [2019-12-17 12:28:11,284] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:11,286] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:11,286] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3682,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:11,295] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [31 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,295] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [17 13]] INFO [2019-12-17 12:28:11,318] jepsen worker 73 - jepsen.util 793 :ok :write [24 11] INFO [2019-12-17 12:28:11,318] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [9 7]] INFO [2019-12-17 12:28:11,318] jepsen worker 32 - jepsen.util 752 :ok :write [10 22] INFO [2019-12-17 12:28:11,318] jepsen worker 32 - jepsen.util 752 :invoke :write [10 6] INFO [2019-12-17 12:28:11,333] jepsen worker 9 - jepsen.util 729 :ok :write [3 14] INFO [2019-12-17 12:28:11,333] jepsen worker 9 - jepsen.util 729 :invoke :write [3 16] INFO [2019-12-17 12:28:11,367] jepsen worker 24 - jepsen.util 654 :ok :read [8 14] INFO [2019-12-17 12:28:11,367] jepsen worker 24 - jepsen.util 654 :invoke :cas [8 [1 34]] INFO [2019-12-17 12:28:11,369] jepsen worker 56 - jepsen.util 686 :ok :write [18 13] INFO [2019-12-17 12:28:11,369] jepsen worker 56 - jepsen.util 686 :invoke :write [18 38] INFO [2019-12-17 12:28:11,375] jepsen worker 20 - jepsen.util 740 :ok :write [6 18] INFO [2019-12-17 12:28:11,375] jepsen worker 20 - jepsen.util 740 :invoke :write [6 13] INFO [2019-12-17 12:28:11,392] jepsen worker 44 - jepsen.util 584 :fail :cas [14 [32 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,393] jepsen worker 44 - jepsen.util 584 :invoke :cas [14 [36 35]] INFO [2019-12-17 12:28:11,403] jepsen worker 42 - jepsen.util 1032 :ok :read [14 27] INFO [2019-12-17 12:28:11,403] jepsen worker 42 - jepsen.util 1032 :invoke :read [14 nil] INFO [2019-12-17 12:28:11,447] jepsen worker 53 - jepsen.util 863 :fail :cas [17 [42 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,447] jepsen worker 53 - jepsen.util 863 :invoke :read [17 nil] INFO [2019-12-17 12:28:11,472] jepsen worker 4 - jepsen.util 634 :fail :cas [1 [1 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,473] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:11,502] jepsen worker 24 - jepsen.util 654 :fail :cas [8 [1 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,502] jepsen worker 24 - jepsen.util 654 :invoke :write [8 37] INFO [2019-12-17 12:28:11,503] jepsen worker 52 - jepsen.util 772 :ok :read [17 46] INFO [2019-12-17 12:28:11,504] jepsen worker 52 - jepsen.util 772 :invoke :cas [17 [44 47]] INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3705,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:11,514] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3708,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:11,526] jepsen worker 44 - jepsen.util 584 :fail :cas [14 [36 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,526] jepsen worker 44 - jepsen.util 584 :invoke :write [14 4] INFO [2019-12-17 12:28:11,585] jepsen worker 25 - jepsen.util 925 :fail :cas [8 [45 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,585] jepsen worker 25 - jepsen.util 925 :invoke :write [8 14] INFO [2019-12-17 12:28:11,613] jepsen worker 6 - jepsen.util 726 :ok :read [2 15] INFO [2019-12-17 12:28:11,613] jepsen worker 6 - jepsen.util 726 :invoke :cas [2 [12 32]] INFO [2019-12-17 12:28:11,640] jepsen worker 37 - jepsen.util 757 :fail :cas [12 [11 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,640] jepsen worker 37 - jepsen.util 757 :invoke :read [12 nil] INFO [2019-12-17 12:28:11,648] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:11,648] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3721,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:11,677] jepsen worker 53 - jepsen.util 863 :ok :read [17 46] INFO [2019-12-17 12:28:11,677] jepsen worker 53 - jepsen.util 863 :invoke :cas [17 [16 28]] INFO [2019-12-17 12:28:11,695] jepsen worker 4 - jepsen.util 634 :ok :read [1 49] INFO [2019-12-17 12:28:11,695] jepsen worker 4 - jepsen.util 634 :invoke :read [1 nil] INFO [2019-12-17 12:28:11,696] jepsen worker 26 - jepsen.util 836 :fail :cas [8 [44 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,697] jepsen worker 26 - jepsen.util 836 :invoke :write [8 31] INFO [2019-12-17 12:28:11,735] jepsen worker 35 - jepsen.util 665 :fail :cas [11 [41 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,736] jepsen worker 35 - jepsen.util 665 :invoke :read [11 nil] INFO [2019-12-17 12:28:11,740] jepsen worker 69 - jepsen.util 789 :ok :read [23 22] INFO [2019-12-17 12:28:11,741] jepsen worker 69 - jepsen.util 789 :invoke :read [23 nil] INFO [2019-12-17 12:28:11,772] jepsen worker 6 - jepsen.util 726 :fail :cas [2 [12 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,772] jepsen worker 6 - jepsen.util 726 :invoke :write [2 14] INFO [2019-12-17 12:28:11,774] jepsen worker 52 - jepsen.util 772 :fail :cas [17 [44 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,774] jepsen worker 52 - jepsen.util 772 :invoke :write [17 7] INFO [2019-12-17 12:28:11,794] jepsen worker 87 - jepsen.util 717 :fail :cas [29 [17 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,794] jepsen worker 87 - jepsen.util 717 :invoke :cas [29 [37 36]] INFO [2019-12-17 12:28:11,821] jepsen worker 58 - jepsen.util 598 :info :write [19 1] :RequestCanceledException INFO [2019-12-17 12:28:11,821] jepsen worker 17 - jepsen.util 647 :info :cas [5 [36 33]] :RequestCanceledException INFO [2019-12-17 12:28:11,823] jepsen worker 58 - jepsen.util 688 :invoke :write [19 8] INFO [2019-12-17 12:28:11,823] jepsen worker 17 - jepsen.util 737 :invoke :write [5 25] INFO [2019-12-17 12:28:11,828] jepsen worker 14 - jepsen.util 734 :ok :read [4 49] INFO [2019-12-17 12:28:11,828] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:11,834] jepsen worker 19 - jepsen.util 739 :info :write [6 21] :RequestCanceledException INFO [2019-12-17 12:28:11,837] jepsen worker 19 - jepsen.util 829 :invoke :read [6 nil] INFO [2019-12-17 12:28:11,907] jepsen worker 28 - jepsen.util 388 :info :write [9 38] :RequestCanceledException INFO [2019-12-17 12:28:11,907] jepsen worker 70 - jepsen.util 970 :ok :read [23 22] INFO [2019-12-17 12:28:11,908] jepsen worker 70 - jepsen.util 970 :invoke :write [23 32] INFO [2019-12-17 12:28:11,909] jepsen worker 28 - jepsen.util 478 :invoke :cas [9 [43 46]] INFO [2019-12-17 12:28:11,921] jepsen worker 13 - jepsen.util 823 :info :write [4 31] :RequestCanceledException INFO [2019-12-17 12:28:11,921] jepsen worker 67 - jepsen.util 1057 :info :write [22 32] :RequestCanceledException INFO [2019-12-17 12:28:11,923] jepsen worker 13 - jepsen.util 913 :invoke :write [4 30] INFO [2019-12-17 12:28:11,923] jepsen worker 67 - jepsen.util 1147 :invoke :cas [22 [6 18]] INFO [2019-12-17 12:28:11,936] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:11,937] jepsen worker 71 - jepsen.util 701 :invoke :cas [23 [44 16]] INFO [2019-12-17 12:28:11,946] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:11,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3737,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3740,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:11,947] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3741,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:11,953] jepsen worker 0 - jepsen.util 810 :info :write [0 43] :RequestCanceledException INFO [2019-12-17 12:28:11,954] jepsen worker 0 - jepsen.util 900 :invoke :write [0 8] INFO [2019-12-17 12:28:11,964] jepsen worker 69 - jepsen.util 789 :ok :read [23 22] INFO [2019-12-17 12:28:11,964] jepsen worker 69 - jepsen.util 789 :invoke :write [23 21] INFO [2019-12-17 12:28:11,979] jepsen worker 87 - jepsen.util 717 :fail :cas [29 [37 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:11,979] jepsen worker 87 - jepsen.util 717 :invoke :write [29 37] INFO [2019-12-17 12:28:12,005] jepsen worker 11 - jepsen.util 821 :info :write [3 8] :RequestCanceledException INFO [2019-12-17 12:28:12,007] jepsen worker 11 - jepsen.util 911 :invoke :cas [3 [22 24]] INFO [2019-12-17 12:28:12,021] jepsen worker 41 - jepsen.util 491 :info :write [13 10] :RequestCanceledException INFO [2019-12-17 12:28:12,022] jepsen worker 41 - jepsen.util 581 :invoke :read [13 nil] INFO [2019-12-17 12:28:12,024] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [17 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,024] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [2 13]] INFO [2019-12-17 12:28:12,052] jepsen worker 50 - jepsen.util 770 :info :write [16 34] :RequestCanceledException INFO [2019-12-17 12:28:12,054] jepsen worker 50 - jepsen.util 860 :invoke :cas [16 [5 43]] INFO [2019-12-17 12:28:12,056] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:12,056] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3751,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:12,066] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:12,066] jepsen nemesis - couchbase.util Rebalance check count: 22 INFO [2019-12-17 12:28:12,066] jepsen nemesis - couchbase.util Rebalance stuck count: 3 INFO [2019-12-17 12:28:12,084] jepsen worker 37 - jepsen.util 757 :ok :read [12 45] INFO [2019-12-17 12:28:12,084] jepsen worker 37 - jepsen.util 757 :invoke :cas [12 [12 47]] INFO [2019-12-17 12:28:12,087] jepsen worker 33 - jepsen.util 843 :info :write [11 23] :RequestCanceledException INFO [2019-12-17 12:28:12,088] jepsen worker 33 - jepsen.util 933 :invoke :write [11 20] INFO [2019-12-17 12:28:12,121] jepsen worker 76 - jepsen.util 706 :info :write [25 37] :RequestCanceledException INFO [2019-12-17 12:28:12,122] jepsen worker 76 - jepsen.util 796 :invoke :write [25 46] INFO [2019-12-17 12:28:12,135] jepsen worker 65 - jepsen.util 515 :info :cas [21 [44 48]] :RequestCanceledException INFO [2019-12-17 12:28:12,135] jepsen worker 29 - jepsen.util 659 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,135] jepsen worker 51 - jepsen.util 591 :info :cas [17 [0 30]] :RequestCanceledException INFO [2019-12-17 12:28:12,135] jepsen worker 29 - jepsen.util 659 :invoke :cas [9 [33 12]] INFO [2019-12-17 12:28:12,137] jepsen worker 51 - jepsen.util 681 :invoke :write [17 47] INFO [2019-12-17 12:28:12,137] jepsen worker 65 - jepsen.util 605 :invoke :cas [21 [36 39]] INFO [2019-12-17 12:28:12,140] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [2 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,140] jepsen worker 5 - jepsen.util 545 :invoke :cas [1 [14 8]] INFO [2019-12-17 12:28:12,152] jepsen worker 30 - jepsen.util 660 :info :write [10 28] :RequestCanceledException INFO [2019-12-17 12:28:12,152] jepsen worker 10 - jepsen.util 730 :info :write [3 37] :RequestCanceledException INFO [2019-12-17 12:28:12,153] jepsen worker 10 - jepsen.util 820 :invoke :read [3 nil] INFO [2019-12-17 12:28:12,154] jepsen worker 30 - jepsen.util 750 :invoke :read [10 nil] INFO [2019-12-17 12:28:12,163] jepsen worker 42 - jepsen.util 1032 :ok :read [14 27] INFO [2019-12-17 12:28:12,163] jepsen worker 42 - jepsen.util 1032 :invoke :cas [14 [21 40]] INFO [2019-12-17 12:28:12,184] jepsen worker 72 - jepsen.util 792 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,184] jepsen worker 63 - jepsen.util 513 :info :write [21 4] :RequestCanceledException INFO [2019-12-17 12:28:12,184] jepsen worker 34 - jepsen.util 754 :info :write [11 25] :RequestCanceledException INFO [2019-12-17 12:28:12,184] jepsen worker 72 - jepsen.util 792 :invoke :write [24 42] INFO [2019-12-17 12:28:12,185] jepsen worker 34 - jepsen.util 844 :invoke :cas [11 [11 37]] INFO [2019-12-17 12:28:12,185] jepsen worker 63 - jepsen.util 603 :invoke :write [21 20] INFO [2019-12-17 12:28:12,199] jepsen worker 42 - jepsen.util 1032 :fail :cas [14 [21 40]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,199] jepsen worker 42 - jepsen.util 1032 :invoke :write [14 33] INFO [2019-12-17 12:28:12,206] jepsen worker 82 - jepsen.util 982 :info :cas [27 [7 19]] :RequestCanceledException INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3772,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:12,207] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3774,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:12,207] jepsen worker 82 - jepsen.util 1072 :invoke :write [27 6] INFO [2019-12-17 12:28:12,220] jepsen worker 46 - jepsen.util 766 :info :write [15 2] :RequestCanceledException INFO [2019-12-17 12:28:12,220] jepsen worker 60 - jepsen.util 870 :info :write [20 15] :RequestCanceledException INFO [2019-12-17 12:28:12,220] jepsen worker 88 - jepsen.util 628 :info :write [29 1] :RequestCanceledException INFO [2019-12-17 12:28:12,222] jepsen worker 60 - jepsen.util 960 :invoke :cas [20 [37 31]] INFO [2019-12-17 12:28:12,222] jepsen worker 46 - jepsen.util 856 :invoke :read [15 nil] INFO [2019-12-17 12:28:12,222] jepsen worker 88 - jepsen.util 718 :invoke :write [29 46] INFO [2019-12-17 12:28:12,226] jepsen worker 63 - jepsen.util 603 :ok :write [21 20] INFO [2019-12-17 12:28:12,227] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:12,234] jepsen worker 18 - jepsen.util 738 :info :write [6 48] :RequestCanceledException INFO [2019-12-17 12:28:12,234] jepsen worker 15 - jepsen.util 735 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,235] jepsen worker 15 - jepsen.util 735 :invoke :read [5 nil] INFO [2019-12-17 12:28:12,236] jepsen worker 18 - jepsen.util 828 :invoke :cas [6 [18 30]] INFO [2019-12-17 12:28:12,249] jepsen worker 63 - jepsen.util 603 :ok :read [21 20] INFO [2019-12-17 12:28:12,249] jepsen worker 63 - jepsen.util 603 :invoke :write [21 26] INFO [2019-12-17 12:28:12,252] jepsen worker 16 - jepsen.util 826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,252] jepsen worker 16 - jepsen.util 826 :invoke :read [5 nil] INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3781,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:12,265] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3785,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:12,273] jepsen worker 4 - jepsen.util 634 :ok :read [1 49] INFO [2019-12-17 12:28:12,273] jepsen worker 4 - jepsen.util 634 :invoke :write [1 32] INFO [2019-12-17 12:28:12,280] jepsen worker 37 - jepsen.util 757 :fail :cas [12 [12 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,280] jepsen worker 37 - jepsen.util 757 :invoke :cas [12 [21 15]] INFO [2019-12-17 12:28:12,292] jepsen worker 63 - jepsen.util 603 :ok :write [21 26] INFO [2019-12-17 12:28:12,292] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [13 2]] INFO [2019-12-17 12:28:12,310] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [13 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,310] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [22 35]] INFO [2019-12-17 12:28:12,321] jepsen worker 40 - jepsen.util 1030 :info :write [13 23] :RequestCanceledException INFO [2019-12-17 12:28:12,322] jepsen worker 40 - jepsen.util 1120 :invoke :write [13 20] INFO [2019-12-17 12:28:12,329] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [22 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,329] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:12,345] jepsen worker 63 - jepsen.util 603 :ok :read [21 26] INFO [2019-12-17 12:28:12,345] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [49 26]] INFO [2019-12-17 12:28:12,360] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [49 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,360] jepsen worker 63 - jepsen.util 603 :invoke :write [21 46] INFO [2019-12-17 12:28:12,373] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:12,373] jepsen worker 43 - jepsen.util 943 :invoke :cas [14 [34 12]] INFO [2019-12-17 12:28:12,380] jepsen worker 37 - jepsen.util 757 :fail :cas [12 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,380] jepsen worker 37 - jepsen.util 757 :invoke :read [12 nil] INFO [2019-12-17 12:28:12,384] jepsen worker 66 - jepsen.util 606 :info :cas [22 [18 7]] :RequestCanceledException INFO [2019-12-17 12:28:12,386] jepsen worker 66 - jepsen.util 696 :invoke :read [22 nil] INFO [2019-12-17 12:28:12,396] jepsen worker 63 - jepsen.util 603 :ok :write [21 46] INFO [2019-12-17 12:28:12,396] jepsen worker 63 - jepsen.util 603 :invoke :write [21 16] INFO [2019-12-17 12:28:12,419] jepsen worker 62 - jepsen.util 872 :info :write [20 23] :RequestCanceledException INFO [2019-12-17 12:28:12,419] jepsen worker 27 - jepsen.util 477 :info :write [9 42] :RequestCanceledException INFO [2019-12-17 12:28:12,419] jepsen worker 31 - jepsen.util 841 :info :cas [10 [1 2]] :RequestCanceledException INFO [2019-12-17 12:28:12,421] jepsen worker 31 - jepsen.util 931 :invoke :write [10 23] INFO [2019-12-17 12:28:12,422] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [40 22]] INFO [2019-12-17 12:28:12,422] jepsen worker 27 - jepsen.util 567 :invoke :cas [9 [21 39]] INFO [2019-12-17 12:28:12,436] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,436] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,436] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:12,436] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:12,439] jepsen worker 63 - jepsen.util 603 :ok :write [21 16] INFO [2019-12-17 12:28:12,439] jepsen worker 63 - jepsen.util 603 :invoke :write [21 30] INFO [2019-12-17 12:28:12,452] jepsen worker 3 - jepsen.util 453 :info :write [1 19] :RequestCanceledException INFO [2019-12-17 12:28:12,453] jepsen worker 3 - jepsen.util 543 :invoke :write [1 22] INFO [2019-12-17 12:28:12,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:12,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3810,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:12,494] jepsen worker 53 - jepsen.util 863 :fail :cas [17 [16 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,494] jepsen worker 53 - jepsen.util 863 :invoke :write [17 3] INFO [2019-12-17 12:28:12,496] jepsen worker 63 - jepsen.util 603 :ok :write [21 30] INFO [2019-12-17 12:28:12,496] jepsen worker 63 - jepsen.util 603 :invoke :write [21 30] INFO [2019-12-17 12:28:12,504] jepsen worker 14 - jepsen.util 734 :ok :read [4 49] INFO [2019-12-17 12:28:12,504] jepsen worker 14 - jepsen.util 734 :invoke :cas [4 [30 41]] INFO [2019-12-17 12:28:12,521] jepsen worker 64 - jepsen.util 514 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,521] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [22 47]] INFO [2019-12-17 12:28:12,524] jepsen worker 60 - jepsen.util 960 :fail :cas [20 [37 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,525] jepsen worker 60 - jepsen.util 960 :invoke :read [20 nil] INFO [2019-12-17 12:28:12,549] jepsen worker 78 - jepsen.util 798 :info :write [26 4] :RequestCanceledException INFO [2019-12-17 12:28:12,551] jepsen worker 78 - jepsen.util 888 :invoke :write [26 33] INFO [2019-12-17 12:28:12,560] jepsen worker 63 - jepsen.util 603 :ok :write [21 30] INFO [2019-12-17 12:28:12,561] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [11 9]] INFO [2019-12-17 12:28:12,561] jepsen worker 71 - jepsen.util 701 :fail :cas [23 [44 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,561] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:12,564] jepsen worker 37 - jepsen.util 757 :ok :read [12 45] INFO [2019-12-17 12:28:12,564] jepsen worker 37 - jepsen.util 757 :invoke :read [12 nil] INFO [2019-12-17 12:28:12,572] jepsen worker 32 - jepsen.util 752 :ok :write [10 6] INFO [2019-12-17 12:28:12,573] jepsen worker 32 - jepsen.util 752 :invoke :read [10 nil] INFO [2019-12-17 12:28:12,578] jepsen worker 74 - jepsen.util 614 :ok :write [24 32] INFO [2019-12-17 12:28:12,579] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [45 5]] INFO [2019-12-17 12:28:12,585] jepsen worker 20 - jepsen.util 740 :ok :write [6 13] INFO [2019-12-17 12:28:12,585] jepsen worker 20 - jepsen.util 740 :invoke :cas [6 [44 15]] INFO [2019-12-17 12:28:12,587] jepsen worker 55 - jepsen.util 955 :info :write [18 2] :RequestCanceledException INFO [2019-12-17 12:28:12,588] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [22 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,588] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [23 17]] INFO [2019-12-17 12:28:12,589] jepsen worker 55 - jepsen.util 1045 :invoke :write [18 0] INFO [2019-12-17 12:28:12,592] jepsen worker 47 - jepsen.util 1037 :ok :write [15 1] INFO [2019-12-17 12:28:12,592] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:12,605] jepsen worker 12 - jepsen.util 732 :info :write [4 21] :RequestCanceledException INFO [2019-12-17 12:28:12,606] jepsen worker 23 - jepsen.util 563 :info :write [7 41] :RequestCanceledException INFO [2019-12-17 12:28:12,607] jepsen worker 12 - jepsen.util 822 :invoke :read [4 nil] INFO [2019-12-17 12:28:12,608] jepsen worker 23 - jepsen.util 653 :invoke :write [7 48] INFO [2019-12-17 12:28:12,624] jepsen worker 9 - jepsen.util 729 :ok :write [3 16] INFO [2019-12-17 12:28:12,624] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [27 45]] INFO [2019-12-17 12:28:12,626] jepsen worker 1 - jepsen.util 631 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,626] jepsen worker 1 - jepsen.util 631 :invoke :read [0 nil] INFO [2019-12-17 12:28:12,631] jepsen worker 34 - jepsen.util 844 :fail :cas [11 [11 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,631] jepsen worker 34 - jepsen.util 844 :invoke :cas [11 [21 13]] INFO [2019-12-17 12:28:12,649] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,650] jepsen worker 14 - jepsen.util 734 :fail :cas [4 [30 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,650] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:12,650] jepsen worker 14 - jepsen.util 734 :invoke :read [4 nil] INFO [2019-12-17 12:28:12,691] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [11 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,691] jepsen worker 63 - jepsen.util 603 :invoke :write [21 10] INFO [2019-12-17 12:28:12,706] jepsen worker 34 - jepsen.util 844 :fail :cas [11 [21 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,707] jepsen worker 34 - jepsen.util 844 :invoke :write [11 31] INFO [2019-12-17 12:28:12,709] jepsen worker 19 - jepsen.util 829 :ok :read [6 18] INFO [2019-12-17 12:28:12,710] jepsen worker 19 - jepsen.util 829 :invoke :write [6 1] INFO [2019-12-17 12:28:12,714] jepsen worker 75 - jepsen.util 705 :info :write [25 41] :RequestCanceledException INFO [2019-12-17 12:28:12,720] jepsen worker 18 - jepsen.util 828 :fail :cas [6 [18 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,721] jepsen worker 18 - jepsen.util 828 :invoke :write [6 23] INFO [2019-12-17 12:28:12,723] jepsen worker 75 - jepsen.util 795 :invoke :write [25 22] INFO [2019-12-17 12:28:12,728] jepsen worker 45 - jepsen.util 1035 :ok :read [15 1] INFO [2019-12-17 12:28:12,728] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 37] INFO [2019-12-17 12:28:12,729] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:12,730] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3841,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:12,730] jepsen worker 84 - jepsen.util 444 :info :write [28 28] :RequestCanceledException INFO [2019-12-17 12:28:12,731] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:12,731] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3844,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:12,736] jepsen worker 84 - jepsen.util 534 :invoke :write [28 35] INFO [2019-12-17 12:28:12,752] jepsen worker 38 - jepsen.util 668 :info :write [12 22] :RequestCanceledException INFO [2019-12-17 12:28:12,765] jepsen worker 38 - jepsen.util 758 :invoke :cas [12 [36 23]] INFO [2019-12-17 12:28:12,779] jepsen worker 11 - jepsen.util 911 :fail :cas [3 [22 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,780] jepsen worker 11 - jepsen.util 911 :invoke :write [3 45] INFO [2019-12-17 12:28:12,790] jepsen worker 20 - jepsen.util 740 :fail :cas [6 [44 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,791] jepsen worker 20 - jepsen.util 740 :invoke :write [6 45] INFO [2019-12-17 12:28:12,809] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [23 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,810] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:12,815] jepsen worker 8 - jepsen.util 728 :info :write [2 41] :RequestCanceledException INFO [2019-12-17 12:28:12,815] jepsen worker 86 - jepsen.util 626 :info :cas [28 [25 46]] :RequestCanceledException INFO [2019-12-17 12:28:12,818] jepsen worker 8 - jepsen.util 818 :invoke :cas [2 [40 32]] INFO [2019-12-17 12:28:12,821] jepsen worker 86 - jepsen.util 716 :invoke :cas [28 [24 38]] INFO [2019-12-17 12:28:12,834] jepsen worker 81 - jepsen.util 891 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,834] jepsen worker 85 - jepsen.util 715 :info :cas [28 [1 34]] :RequestCanceledException INFO [2019-12-17 12:28:12,834] jepsen worker 81 - jepsen.util 891 :invoke :cas [27 [21 32]] INFO [2019-12-17 12:28:12,838] jepsen worker 10 - jepsen.util 820 :ok :read [3 16] INFO [2019-12-17 12:28:12,838] jepsen worker 85 - jepsen.util 805 :invoke :cas [28 [48 39]] INFO [2019-12-17 12:28:12,838] jepsen worker 10 - jepsen.util 820 :invoke :write [3 17] INFO [2019-12-17 12:28:12,838] jepsen worker 29 - jepsen.util 659 :fail :cas [9 [33 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,838] jepsen worker 29 - jepsen.util 659 :invoke :write [9 33] INFO [2019-12-17 12:28:12,842] jepsen worker 65 - jepsen.util 605 :fail :cas [21 [36 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,842] jepsen worker 65 - jepsen.util 605 :invoke :write [21 12] INFO [2019-12-17 12:28:12,851] jepsen worker 83 - jepsen.util 893 :info :write [27 46] :RequestCanceledException INFO [2019-12-17 12:28:12,851] jepsen worker 54 - jepsen.util 774 :info :write [18 20] :RequestCanceledException INFO [2019-12-17 12:28:12,855] jepsen worker 54 - jepsen.util 864 :invoke :read [18 nil] INFO [2019-12-17 12:28:12,856] jepsen worker 83 - jepsen.util 983 :invoke :read [27 nil] INFO [2019-12-17 12:28:12,904] jepsen worker 46 - jepsen.util 856 :ok :read [15 1] INFO [2019-12-17 12:28:12,904] jepsen worker 46 - jepsen.util 856 :invoke :write [15 15] INFO [2019-12-17 12:28:12,919] jepsen worker 30 - jepsen.util 750 :ok :read [10 6] INFO [2019-12-17 12:28:12,919] jepsen worker 30 - jepsen.util 750 :invoke :write [10 31] INFO [2019-12-17 12:28:12,921] jepsen worker 39 - jepsen.util 759 :info :cas [13 [16 10]] :RequestCanceledException INFO [2019-12-17 12:28:12,923] jepsen worker 39 - jepsen.util 849 :invoke :write [13 42] INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3867,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3870,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:12,924] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:12,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3871,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:12,935] jepsen worker 68 - jepsen.util 428 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:12,935] jepsen worker 68 - jepsen.util 428 :invoke :write [22 15] INFO [2019-12-17 12:28:12,967] jepsen worker 27 - jepsen.util 567 :fail :cas [9 [21 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:12,967] jepsen worker 27 - jepsen.util 567 :invoke :write [9 33] INFO [2019-12-17 12:28:12,997] jepsen worker 32 - jepsen.util 752 :ok :read [10 6] INFO [2019-12-17 12:28:12,997] jepsen worker 32 - jepsen.util 752 :invoke :write [10 25] INFO [2019-12-17 12:28:13,007] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [40 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,008] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [3 32]] INFO [2019-12-17 12:28:13,017] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [27 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,017] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [33 38]] INFO [2019-12-17 12:28:13,021] jepsen worker 21 - jepsen.util 831 :info :write [7 39] :RequestCanceledException INFO [2019-12-17 12:28:13,021] jepsen worker 36 - jepsen.util 846 :info :write [12 3] :RequestCanceledException INFO [2019-12-17 12:28:13,022] jepsen worker 36 - jepsen.util 936 :invoke :cas [12 [38 0]] INFO [2019-12-17 12:28:13,022] jepsen worker 21 - jepsen.util 921 :invoke :cas [7 [27 20]] INFO [2019-12-17 12:28:13,033] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:13,033] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3880,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:13,034] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:13,034] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3882,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:13,038] jepsen worker 63 - jepsen.util 603 :ok :write [21 10] INFO [2019-12-17 12:28:13,038] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [2 3]] INFO [2019-12-17 12:28:13,049] jepsen worker 28 - jepsen.util 478 :ok :cas [9 [43 46]] INFO [2019-12-17 12:28:13,049] jepsen worker 28 - jepsen.util 478 :invoke :read [9 nil] INFO [2019-12-17 12:28:13,052] jepsen worker 77 - jepsen.util 617 :info :cas [25 [2 3]] :RequestCanceledException INFO [2019-12-17 12:28:13,054] jepsen worker 77 - jepsen.util 707 :invoke :write [25 22] INFO [2019-12-17 12:28:13,058] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [2 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,058] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [46 5]] INFO [2019-12-17 12:28:13,068] jepsen worker 72 - jepsen.util 792 :ok :write [24 42] INFO [2019-12-17 12:28:13,068] jepsen worker 72 - jepsen.util 792 :invoke :write [24 17] INFO [2019-12-17 12:28:13,080] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [33 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,080] jepsen worker 9 - jepsen.util 729 :invoke :write [3 45] INFO [2019-12-17 12:28:13,085] jepsen worker 7 - jepsen.util 907 :info :write [2 7] :RequestCanceledException INFO [2019-12-17 12:28:13,085] jepsen worker 79 - jepsen.util 619 :info :write [26 12] :RequestCanceledException INFO [2019-12-17 12:28:13,087] jepsen worker 79 - jepsen.util 709 :invoke :read [26 nil] INFO [2019-12-17 12:28:13,087] jepsen worker 7 - jepsen.util 997 :invoke :write [2 48] INFO [2019-12-17 12:28:13,102] jepsen worker 35 - jepsen.util 665 :ok :read [11 7] INFO [2019-12-17 12:28:13,103] jepsen worker 35 - jepsen.util 665 :invoke :write [11 21] INFO [2019-12-17 12:28:13,109] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:13,109] jepsen nemesis - couchbase.util Rebalance check count: 23 INFO [2019-12-17 12:28:13,109] jepsen nemesis - couchbase.util Rebalance stuck count: 4 INFO [2019-12-17 12:28:13,120] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [46 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,120] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [26 1]] INFO [2019-12-17 12:28:13,128] jepsen worker 37 - jepsen.util 757 :ok :read [12 45] INFO [2019-12-17 12:28:13,128] jepsen worker 37 - jepsen.util 757 :invoke :write [12 49] INFO [2019-12-17 12:28:13,134] jepsen worker 80 - jepsen.util 890 :info :write [26 43] :RequestCanceledException INFO [2019-12-17 12:28:13,136] jepsen worker 80 - jepsen.util 980 :invoke :read [26 nil] INFO [2019-12-17 12:28:13,174] jepsen worker 8 - jepsen.util 818 :fail :cas [2 [40 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,174] jepsen worker 8 - jepsen.util 818 :invoke :cas [2 [47 10]] INFO [2019-12-17 12:28:13,192] jepsen worker 79 - jepsen.util 709 :ok :read [26 43] INFO [2019-12-17 12:28:13,192] jepsen worker 79 - jepsen.util 709 :invoke :cas [26 [2 31]] INFO [2019-12-17 12:28:13,201] jepsen worker 74 - jepsen.util 614 :fail :cas [24 [45 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,201] jepsen worker 74 - jepsen.util 614 :invoke :cas [24 [1 10]] INFO [2019-12-17 12:28:13,204] jepsen worker 28 - jepsen.util 478 :ok :read [9 46] INFO [2019-12-17 12:28:13,205] jepsen worker 28 - jepsen.util 478 :invoke :cas [9 [35 37]] INFO [2019-12-17 12:28:13,211] jepsen worker 43 - jepsen.util 943 :fail :cas [14 [34 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,211] jepsen worker 43 - jepsen.util 943 :invoke :read [14 nil] INFO [2019-12-17 12:28:13,221] jepsen worker 59 - jepsen.util 779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:13,221] jepsen worker 59 - jepsen.util 779 :invoke :read [19 nil] INFO [2019-12-17 12:28:13,226] jepsen worker 5 - jepsen.util 545 :fail :cas [1 [14 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,226] jepsen worker 5 - jepsen.util 545 :invoke :read [1 nil] INFO [2019-12-17 12:28:13,237] jepsen worker 8 - jepsen.util 818 :fail :cas [2 [47 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,238] jepsen worker 8 - jepsen.util 818 :invoke :cas [2 [11 9]] INFO [2019-12-17 12:28:13,243] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [26 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,244] jepsen worker 63 - jepsen.util 603 :invoke :write [21 8] INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3915,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:13,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3918,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:13,249] jepsen worker 22 - jepsen.util 652 :info :write [7 16] :RequestCanceledException INFO [2019-12-17 12:28:13,251] jepsen worker 22 - jepsen.util 742 :invoke :cas [7 [27 35]] INFO [2019-12-17 12:28:13,254] jepsen worker 73 - jepsen.util 793 :ok :read [24 42] INFO [2019-12-17 12:28:13,254] jepsen worker 73 - jepsen.util 793 :invoke :write [24 26] INFO [2019-12-17 12:28:13,278] jepsen worker 8 - jepsen.util 818 :fail :cas [2 [11 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,278] jepsen worker 8 - jepsen.util 818 :invoke :write [2 45] INFO [2019-12-17 12:28:13,287] jepsen worker 57 - jepsen.util 777 :info :cas [19 [4 49]] :RequestCanceledException INFO [2019-12-17 12:28:13,288] jepsen worker 57 - jepsen.util 867 :invoke :read [19 nil] INFO [2019-12-17 12:28:13,297] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:13,297] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3905,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:13,305] jepsen worker 22 - jepsen.util 742 :fail :cas [7 [27 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,305] jepsen worker 22 - jepsen.util 742 :invoke :write [7 2] INFO [2019-12-17 12:28:13,305] jepsen worker 61 - jepsen.util 871 :info :write [20 27] :RequestCanceledException INFO [2019-12-17 12:28:13,308] jepsen worker 61 - jepsen.util 961 :invoke :cas [20 [19 7]] INFO [2019-12-17 12:28:13,321] jepsen worker 49 - jepsen.util 589 :info :write [16 19] :RequestCanceledException INFO [2019-12-17 12:28:13,323] jepsen worker 49 - jepsen.util 679 :invoke :cas [16 [22 30]] INFO [2019-12-17 12:28:13,324] jepsen worker 63 - jepsen.util 603 :ok :write [21 8] INFO [2019-12-17 12:28:13,324] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,334] jepsen worker 12 - jepsen.util 822 :ok :read [4 49] INFO [2019-12-17 12:28:13,335] jepsen worker 12 - jepsen.util 822 :invoke :read [4 nil] INFO [2019-12-17 12:28:13,335] jepsen worker 89 - jepsen.util 899 :info :write [29 49] :RequestCanceledException INFO [2019-12-17 12:28:13,337] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:13,354] jepsen worker 60 - jepsen.util 960 :ok :read [20 7] INFO [2019-12-17 12:28:13,354] jepsen worker 60 - jepsen.util 960 :invoke :write [20 30] INFO [2019-12-17 12:28:13,363] jepsen worker 63 - jepsen.util 603 :ok :read [21 8] INFO [2019-12-17 12:28:13,363] jepsen worker 63 - jepsen.util 603 :invoke :write [21 45] INFO [2019-12-17 12:28:13,397] jepsen worker 54 - jepsen.util 864 :ok :read [18 13] INFO [2019-12-17 12:28:13,397] jepsen worker 54 - jepsen.util 864 :invoke :cas [18 [13 39]] INFO [2019-12-17 12:28:13,422] jepsen worker 56 - jepsen.util 686 :info :write [18 38] :RequestCanceledException INFO [2019-12-17 12:28:13,423] jepsen worker 56 - jepsen.util 776 :invoke :read [18 nil] INFO [2019-12-17 12:28:13,431] jepsen worker 63 - jepsen.util 603 :ok :write [21 45] INFO [2019-12-17 12:28:13,431] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [28 47]] INFO [2019-12-17 12:28:13,457] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [28 47]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,458] jepsen worker 63 - jepsen.util 603 :invoke :write [21 42] INFO [2019-12-17 12:28:13,466] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:13,466] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3939,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:13,467] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:13,467] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3942,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:13,488] jepsen worker 43 - jepsen.util 943 :ok :read [14 27] INFO [2019-12-17 12:28:13,489] jepsen worker 43 - jepsen.util 943 :invoke :write [14 33] INFO [2019-12-17 12:28:13,514] jepsen worker 79 - jepsen.util 709 :fail :cas [26 [2 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,514] jepsen worker 79 - jepsen.util 709 :invoke :cas [26 [36 20]] INFO [2019-12-17 12:28:13,530] jepsen worker 63 - jepsen.util 603 :ok :write [21 42] INFO [2019-12-17 12:28:13,530] jepsen worker 63 - jepsen.util 603 :invoke :write [21 38] INFO [2019-12-17 12:28:13,535] jepsen worker 24 - jepsen.util 654 :info :write [8 37] :RequestCanceledException INFO [2019-12-17 12:28:13,536] jepsen worker 24 - jepsen.util 744 :invoke :cas [8 [15 7]] INFO [2019-12-17 12:28:13,588] jepsen worker 63 - jepsen.util 603 :ok :write [21 38] INFO [2019-12-17 12:28:13,589] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,617] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,618] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,638] jepsen worker 25 - jepsen.util 925 :info :write [8 14] :RequestCanceledException INFO [2019-12-17 12:28:13,638] jepsen worker 44 - jepsen.util 584 :info :write [14 4] :RequestCanceledException INFO [2019-12-17 12:28:13,659] jepsen worker 44 - jepsen.util 674 :invoke :read [14 nil] INFO [2019-12-17 12:28:13,661] jepsen worker 25 - jepsen.util 1015 :invoke :read [8 nil] INFO [2019-12-17 12:28:13,695] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [3 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,695] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [47 1]] INFO [2019-12-17 12:28:13,697] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,697] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [26 37]] INFO [2019-12-17 12:28:13,700] jepsen worker 24 - jepsen.util 744 :fail :cas [8 [15 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,700] jepsen worker 24 - jepsen.util 744 :invoke :cas [8 [8 32]] INFO [2019-12-17 12:28:13,712] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:13,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3955,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:13,718] jepsen worker 26 - jepsen.util 836 :info :write [8 31] :RequestCanceledException INFO [2019-12-17 12:28:13,724] jepsen worker 26 - jepsen.util 926 :invoke :cas [8 [44 5]] INFO [2019-12-17 12:28:13,735] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [26 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,735] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [27 14]] INFO [2019-12-17 12:28:13,759] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [27 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,760] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [7 44]] INFO [2019-12-17 12:28:13,764] jepsen worker 36 - jepsen.util 936 :fail :cas [12 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,764] jepsen worker 36 - jepsen.util 936 :invoke :write [12 47] INFO [2019-12-17 12:28:13,780] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [7 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,780] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,793] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,794] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [28 12]] INFO [2019-12-17 12:28:13,806] jepsen worker 6 - jepsen.util 726 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:28:13,806] jepsen worker 52 - jepsen.util 772 :info :write [17 7] :RequestCanceledException INFO [2019-12-17 12:28:13,807] jepsen worker 52 - jepsen.util 862 :invoke :read [17 nil] INFO [2019-12-17 12:28:13,807] jepsen worker 6 - jepsen.util 816 :invoke :read [2 nil] INFO [2019-12-17 12:28:13,812] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [28 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,813] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,833] jepsen worker 26 - jepsen.util 926 :fail :cas [8 [44 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,833] jepsen worker 26 - jepsen.util 926 :invoke :write [8 6] INFO [2019-12-17 12:28:13,836] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,836] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [21 15]] INFO [2019-12-17 12:28:13,844] jepsen worker 6 - jepsen.util 816 :ok :read [2 15] INFO [2019-12-17 12:28:13,845] jepsen worker 6 - jepsen.util 816 :invoke :cas [2 [3 11]] INFO [2019-12-17 12:28:13,876] jepsen worker 38 - jepsen.util 758 :fail :cas [12 [36 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,876] jepsen worker 38 - jepsen.util 758 :invoke :read [12 nil] INFO [2019-12-17 12:28:13,881] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,881] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,887] jepsen worker 58 - jepsen.util 688 :info :write [19 8] :RequestCanceledException INFO [2019-12-17 12:28:13,889] jepsen worker 58 - jepsen.util 778 :invoke :write [19 44] INFO [2019-12-17 12:28:13,895] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:13,895] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3977,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:13,896] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:13,896] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3981,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:13,905] jepsen worker 17 - jepsen.util 737 :info :write [5 25] :RequestCanceledException INFO [2019-12-17 12:28:13,907] jepsen worker 17 - jepsen.util 827 :invoke :write [5 28] INFO [2019-12-17 12:28:13,917] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,917] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,919] jepsen worker 80 - jepsen.util 980 :ok :read [26 43] INFO [2019-12-17 12:28:13,920] jepsen worker 80 - jepsen.util 980 :invoke :cas [26 [49 27]] INFO [2019-12-17 12:28:13,949] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,949] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:13,951] jepsen worker 14 - jepsen.util 734 :ok :read [4 49] INFO [2019-12-17 12:28:13,952] jepsen worker 14 - jepsen.util 734 :invoke :write [4 15] INFO [2019-12-17 12:28:13,952] jepsen worker 13 - jepsen.util 913 :info :write [4 30] :RequestCanceledException INFO [2019-12-17 12:28:13,952] jepsen worker 70 - jepsen.util 970 :info :write [23 32] :RequestCanceledException INFO [2019-12-17 12:28:13,955] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [9 7]] INFO [2019-12-17 12:28:13,955] jepsen worker 70 - jepsen.util 1060 :invoke :write [23 24] INFO [2019-12-17 12:28:13,963] jepsen worker 6 - jepsen.util 816 :fail :cas [2 [3 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:13,963] jepsen worker 6 - jepsen.util 816 :invoke :write [2 9] INFO [2019-12-17 12:28:13,984] jepsen worker 63 - jepsen.util 603 :ok :read [21 38] INFO [2019-12-17 12:28:13,984] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [24 11]] INFO [2019-12-17 12:28:13,988] jepsen worker 0 - jepsen.util 900 :info :write [0 8] :RequestCanceledException INFO [2019-12-17 12:28:13,991] jepsen worker 0 - jepsen.util 990 :invoke :write [0 11] INFO [2019-12-17 12:28:14,000] jepsen worker 5 - jepsen.util 545 :ok :read [1 49] INFO [2019-12-17 12:28:14,001] jepsen worker 5 - jepsen.util 545 :invoke :write [1 45] INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3997,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:14,004] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":3999,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:14,012] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [24 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,013] jepsen worker 63 - jepsen.util 603 :invoke :write [21 21] INFO [2019-12-17 12:28:14,021] jepsen worker 67 - jepsen.util 1147 :info :cas [22 [6 18]] :RequestCanceledException INFO [2019-12-17 12:28:14,021] jepsen worker 69 - jepsen.util 789 :info :write [23 21] :RequestCanceledException INFO [2019-12-17 12:28:14,021] jepsen worker 87 - jepsen.util 717 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:14,025] jepsen worker 67 - jepsen.util 1237 :invoke :read [22 nil] INFO [2019-12-17 12:28:14,025] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:14,025] jepsen worker 87 - jepsen.util 807 :invoke :write [29 21] INFO [2019-12-17 12:28:14,043] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:14,043] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:14,065] jepsen worker 63 - jepsen.util 603 :ok :write [21 21] INFO [2019-12-17 12:28:14,065] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,078] jepsen worker 12 - jepsen.util 822 :ok :read [4 49] INFO [2019-12-17 12:28:14,078] jepsen worker 12 - jepsen.util 822 :invoke :cas [4 [40 42]] INFO [2019-12-17 12:28:14,088] jepsen worker 63 - jepsen.util 603 :ok :read [21 21] INFO [2019-12-17 12:28:14,088] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [4 34]] INFO [2019-12-17 12:28:14,094] jepsen worker 52 - jepsen.util 862 :ok :read [17 46] INFO [2019-12-17 12:28:14,094] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [20 41]] INFO [2019-12-17 12:28:14,107] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [4 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,107] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [16 34]] INFO [2019-12-17 12:28:14,121] jepsen worker 41 - jepsen.util 581 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,121] jepsen worker 41 - jepsen.util 581 :invoke :cas [13 [49 32]] INFO [2019-12-17 12:28:14,123] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:14,123] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:14,123] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:14,123] jepsen nemesis - couchbase.util Rebalance check count: 24 INFO [2019-12-17 12:28:14,123] jepsen nemesis - couchbase.util Rebalance stuck count: 5 INFO [2019-12-17 12:28:14,130] jepsen worker 72 - jepsen.util 792 :ok :write [24 17] INFO [2019-12-17 12:28:14,130] jepsen worker 31 - jepsen.util 931 :ok :write [10 23] INFO [2019-12-17 12:28:14,130] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [37 27]] INFO [2019-12-17 12:28:14,130] jepsen worker 31 - jepsen.util 931 :invoke :read [10 nil] INFO [2019-12-17 12:28:14,131] jepsen worker 18 - jepsen.util 828 :ok :write [6 23] INFO [2019-12-17 12:28:14,131] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:14,132] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [16 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,132] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [33 29]] INFO [2019-12-17 12:28:14,133] jepsen worker 50 - jepsen.util 860 :info :cas [16 [5 43]] :RequestCanceledException INFO [2019-12-17 12:28:14,134] jepsen worker 50 - jepsen.util 950 :invoke :write [16 15] INFO [2019-12-17 12:28:14,141] jepsen worker 61 - jepsen.util 961 :fail :cas [20 [19 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,141] jepsen worker 61 - jepsen.util 961 :invoke :cas [20 [33 45]] INFO [2019-12-17 12:28:14,148] jepsen worker 9 - jepsen.util 729 :ok :write [3 45] INFO [2019-12-17 12:28:14,149] jepsen worker 9 - jepsen.util 729 :invoke :write [3 3] INFO [2019-12-17 12:28:14,153] jepsen worker 33 - jepsen.util 933 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:28:14,154] jepsen worker 79 - jepsen.util 709 :fail :cas [26 [36 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,154] jepsen worker 79 - jepsen.util 709 :invoke :write [26 32] INFO [2019-12-17 12:28:14,155] jepsen worker 33 - jepsen.util 1023 :invoke :cas [11 [12 37]] INFO [2019-12-17 12:28:14,171] jepsen worker 25 - jepsen.util 1015 :ok :read [8 14] INFO [2019-12-17 12:28:14,171] jepsen worker 25 - jepsen.util 1015 :invoke :write [8 49] INFO [2019-12-17 12:28:14,183] jepsen worker 51 - jepsen.util 681 :info :write [17 47] :RequestCanceledException INFO [2019-12-17 12:28:14,185] jepsen worker 51 - jepsen.util 771 :invoke :cas [17 [21 24]] INFO [2019-12-17 12:28:14,190] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [47 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,191] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [36 34]] INFO [2019-12-17 12:28:14,198] jepsen worker 18 - jepsen.util 828 :ok :read [6 23] INFO [2019-12-17 12:28:14,198] jepsen worker 18 - jepsen.util 828 :invoke :write [6 32] INFO [2019-12-17 12:28:14,199] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,199] jepsen worker 63 - jepsen.util 603 :invoke :write [21 9] INFO [2019-12-17 12:28:14,221] jepsen worker 76 - jepsen.util 796 :info :write [25 46] :RequestCanceledException INFO [2019-12-17 12:28:14,221] jepsen worker 42 - jepsen.util 1032 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:14,223] jepsen worker 42 - jepsen.util 1122 :invoke :read [14 nil] INFO [2019-12-17 12:28:14,223] jepsen worker 76 - jepsen.util 886 :invoke :write [25 22] INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4035,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:14,223] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4039,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:14,235] jepsen worker 88 - jepsen.util 718 :info :write [29 46] :RequestCanceledException INFO [2019-12-17 12:28:14,236] jepsen worker 88 - jepsen.util 808 :invoke :write [29 38] INFO [2019-12-17 12:28:14,258] jepsen worker 56 - jepsen.util 776 :ok :read [18 13] INFO [2019-12-17 12:28:14,258] jepsen worker 56 - jepsen.util 776 :invoke :read [18 nil] INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4025,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:14,275] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4026,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:14,278] jepsen worker 63 - jepsen.util 603 :ok :write [21 9] INFO [2019-12-17 12:28:14,278] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [7 10]] INFO [2019-12-17 12:28:14,288] jepsen worker 82 - jepsen.util 1072 :info :write [27 6] :RequestCanceledException INFO [2019-12-17 12:28:14,290] jepsen worker 82 - jepsen.util 1162 :invoke :cas [27 [27 0]] INFO [2019-12-17 12:28:14,295] jepsen worker 61 - jepsen.util 961 :fail :cas [20 [33 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,295] jepsen worker 61 - jepsen.util 961 :invoke :cas [20 [1 39]] INFO [2019-12-17 12:28:14,301] jepsen worker 4 - jepsen.util 634 :info :write [1 32] :RequestCanceledException INFO [2019-12-17 12:28:14,303] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [45 30]] INFO [2019-12-17 12:28:14,304] jepsen worker 45 - jepsen.util 1035 :ok :write [15 37] INFO [2019-12-17 12:28:14,305] jepsen worker 73 - jepsen.util 793 :ok :write [24 26] INFO [2019-12-17 12:28:14,305] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 17] INFO [2019-12-17 12:28:14,305] jepsen worker 73 - jepsen.util 793 :invoke :write [24 37] INFO [2019-12-17 12:28:14,305] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,306] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,335] jepsen worker 15 - jepsen.util 735 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,336] jepsen worker 15 - jepsen.util 735 :invoke :write [5 14] INFO [2019-12-17 12:28:14,353] jepsen worker 16 - jepsen.util 826 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,353] jepsen worker 16 - jepsen.util 826 :invoke :cas [5 [26 1]] INFO [2019-12-17 12:28:14,363] jepsen worker 31 - jepsen.util 931 :ok :read [10 23] INFO [2019-12-17 12:28:14,363] jepsen worker 31 - jepsen.util 931 :invoke :read [10 nil] INFO [2019-12-17 12:28:14,363] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [37 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,363] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [45 36]] INFO [2019-12-17 12:28:14,366] jepsen worker 63 - jepsen.util 603 :ok :read [21 9] INFO [2019-12-17 12:28:14,366] jepsen worker 63 - jepsen.util 603 :invoke :write [21 28] INFO [2019-12-17 12:28:14,395] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:14,395] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:14,398] jepsen worker 61 - jepsen.util 961 :fail :cas [20 [1 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,399] jepsen worker 61 - jepsen.util 961 :invoke :write [20 26] INFO [2019-12-17 12:28:14,415] jepsen worker 24 - jepsen.util 744 :fail :cas [8 [8 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,416] jepsen worker 24 - jepsen.util 744 :invoke :write [8 44] INFO [2019-12-17 12:28:14,421] jepsen worker 40 - jepsen.util 1120 :info :write [13 20] :RequestCanceledException INFO [2019-12-17 12:28:14,422] jepsen worker 63 - jepsen.util 603 :ok :write [21 28] INFO [2019-12-17 12:28:14,422] jepsen worker 63 - jepsen.util 603 :invoke :write [21 39] INFO [2019-12-17 12:28:14,423] jepsen worker 40 - jepsen.util 1210 :invoke :cas [13 [38 42]] INFO [2019-12-17 12:28:14,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:14,442] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4060,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:14,442] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:14,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4063,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:14,443] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:14,443] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4064,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:14,452] jepsen worker 33 - jepsen.util 1023 :fail :cas [11 [12 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,452] jepsen worker 33 - jepsen.util 1023 :invoke :cas [11 [42 25]] INFO [2019-12-17 12:28:14,489] jepsen worker 66 - jepsen.util 696 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,491] jepsen worker 66 - jepsen.util 696 :invoke :cas [22 [19 15]] INFO [2019-12-17 12:28:14,493] jepsen worker 63 - jepsen.util 603 :ok :write [21 39] INFO [2019-12-17 12:28:14,496] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,502] jepsen worker 80 - jepsen.util 980 :fail :cas [26 [49 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,502] jepsen worker 80 - jepsen.util 980 :invoke :cas [26 [22 46]] INFO [2019-12-17 12:28:14,529] jepsen worker 38 - jepsen.util 758 :ok :read [12 45] INFO [2019-12-17 12:28:14,529] jepsen worker 38 - jepsen.util 758 :invoke :cas [12 [38 33]] INFO [2019-12-17 12:28:14,535] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,536] jepsen worker 33 - jepsen.util 1023 :fail :cas [11 [42 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,536] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,536] jepsen worker 2 - jepsen.util 632 :invoke :read [0 nil] INFO [2019-12-17 12:28:14,536] jepsen worker 33 - jepsen.util 1023 :invoke :write [11 15] INFO [2019-12-17 12:28:14,536] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:14,537] jepsen worker 42 - jepsen.util 1122 :ok :read [14 27] INFO [2019-12-17 12:28:14,538] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [32 6]] INFO [2019-12-17 12:28:14,545] jepsen worker 3 - jepsen.util 543 :info :write [1 22] :RequestCanceledException INFO [2019-12-17 12:28:14,548] jepsen worker 3 - jepsen.util 633 :invoke :write [1 43] INFO [2019-12-17 12:28:14,573] jepsen worker 63 - jepsen.util 603 :ok :read [21 39] INFO [2019-12-17 12:28:14,573] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [29 7]] INFO [2019-12-17 12:28:14,603] jepsen worker 53 - jepsen.util 863 :info :write [17 3] :RequestCanceledException INFO [2019-12-17 12:28:14,603] jepsen worker 47 - jepsen.util 1037 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,604] jepsen worker 47 - jepsen.util 1037 :invoke :read [15 nil] INFO [2019-12-17 12:28:14,604] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [29 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,605] jepsen worker 63 - jepsen.util 603 :invoke :write [21 22] INFO [2019-12-17 12:28:14,605] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:14,611] jepsen worker 78 - jepsen.util 888 :info :write [26 33] :RequestCanceledException INFO [2019-12-17 12:28:14,614] jepsen worker 78 - jepsen.util 978 :invoke :read [26 nil] INFO [2019-12-17 12:28:14,621] jepsen worker 56 - jepsen.util 776 :ok :read [18 13] INFO [2019-12-17 12:28:14,622] jepsen worker 56 - jepsen.util 776 :invoke :write [18 7] INFO [2019-12-17 12:28:14,668] jepsen worker 80 - jepsen.util 980 :fail :cas [26 [22 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,669] jepsen worker 63 - jepsen.util 603 :ok :write [21 22] INFO [2019-12-17 12:28:14,669] jepsen worker 80 - jepsen.util 980 :invoke :read [26 nil] INFO [2019-12-17 12:28:14,669] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [24 43]] INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4084,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:14,685] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4088,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:14,687] jepsen worker 55 - jepsen.util 1045 :info :write [18 0] :RequestCanceledException INFO [2019-12-17 12:28:14,690] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [39 15]] INFO [2019-12-17 12:28:14,693] jepsen worker 38 - jepsen.util 758 :fail :cas [12 [38 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,693] jepsen worker 38 - jepsen.util 758 :invoke :write [12 12] INFO [2019-12-17 12:28:14,702] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [24 43]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,702] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [14 19]] INFO [2019-12-17 12:28:14,721] jepsen worker 23 - jepsen.util 653 :info :write [7 48] :RequestCanceledException INFO [2019-12-17 12:28:14,723] jepsen worker 23 - jepsen.util 743 :invoke :cas [7 [26 26]] INFO [2019-12-17 12:28:14,728] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [14 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,729] jepsen worker 63 - jepsen.util 603 :invoke :write [21 16] INFO [2019-12-17 12:28:14,734] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [45 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,734] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:14,735] jepsen worker 1 - jepsen.util 631 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,735] jepsen worker 1 - jepsen.util 631 :invoke :write [0 19] INFO [2019-12-17 12:28:14,752] jepsen worker 34 - jepsen.util 844 :info :write [11 31] :RequestCanceledException INFO [2019-12-17 12:28:14,754] jepsen worker 34 - jepsen.util 934 :invoke :read [11 nil] INFO [2019-12-17 12:28:14,776] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [20 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,776] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [36 42]] INFO [2019-12-17 12:28:14,786] jepsen worker 63 - jepsen.util 603 :ok :write [21 16] INFO [2019-12-17 12:28:14,786] jepsen worker 63 - jepsen.util 603 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,805] jepsen worker 63 - jepsen.util 603 :ok :read [21 16] INFO [2019-12-17 12:28:14,806] jepsen worker 63 - jepsen.util 603 :invoke :cas [21 [24 7]] INFO [2019-12-17 12:28:14,821] jepsen worker 19 - jepsen.util 829 :info :write [6 1] :RequestCanceledException INFO [2019-12-17 12:28:14,821] jepsen worker 64 - jepsen.util 514 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,821] jepsen worker 64 - jepsen.util 514 :invoke :write [21 4] INFO [2019-12-17 12:28:14,823] jepsen worker 19 - jepsen.util 919 :invoke :write [6 9] INFO [2019-12-17 12:28:14,831] jepsen worker 63 - jepsen.util 603 :fail :cas [21 [24 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,831] jepsen worker 63 - jepsen.util 603 :invoke :write [21 8] INFO [2019-12-17 12:28:14,835] jepsen worker 75 - jepsen.util 795 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:14,837] jepsen worker 75 - jepsen.util 885 :invoke :read [25 nil] INFO [2019-12-17 12:28:14,840] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [36 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,840] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [25 19]] INFO [2019-12-17 12:28:14,842] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:14,842] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:14,850] jepsen worker 84 - jepsen.util 534 :info :write [28 35] :RequestCanceledException INFO [2019-12-17 12:28:14,850] jepsen worker 20 - jepsen.util 740 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:28:14,852] jepsen worker 20 - jepsen.util 830 :invoke :write [6 1] INFO [2019-12-17 12:28:14,852] jepsen worker 84 - jepsen.util 624 :invoke :write [28 12] INFO [2019-12-17 12:28:14,852] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [39 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:14,852] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [1 4]] INFO [2019-12-17 12:28:14,872] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:14,872] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4112,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:14,873] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:14,873] jepsen worker 71 - jepsen.util 701 :invoke :cas [23 [17 9]] INFO [2019-12-17 12:28:14,891] jepsen worker 53 - jepsen.util 953 :ok :read [17 46] INFO [2019-12-17 12:28:14,891] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:14,906] jepsen worker 11 - jepsen.util 911 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:28:14,906] jepsen worker 86 - jepsen.util 716 :info :cas [28 [24 38]] :RequestCanceledException INFO [2019-12-17 12:28:14,906] jepsen worker 65 - jepsen.util 605 :info :write [21 12] :RequestCanceledException INFO [2019-12-17 12:28:14,907] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,907] jepsen worker 86 - jepsen.util 806 :invoke :cas [28 [14 34]] INFO [2019-12-17 12:28:14,907] jepsen worker 11 - jepsen.util 1001 :invoke :cas [3 [5 16]] INFO [2019-12-17 12:28:14,908] jepsen worker 64 - jepsen.util 514 :ok :write [21 4] INFO [2019-12-17 12:28:14,909] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:14,918] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:14,918] jepsen worker 69 - jepsen.util 879 :invoke :cas [23 [25 1]] INFO [2019-12-17 12:28:14,921] jepsen worker 10 - jepsen.util 820 :info :write [3 17] :RequestCanceledException INFO [2019-12-17 12:28:14,921] jepsen worker 85 - jepsen.util 805 :info :cas [28 [48 39]] :RequestCanceledException INFO [2019-12-17 12:28:14,921] jepsen worker 46 - jepsen.util 856 :info :write [15 15] :RequestCanceledException INFO [2019-12-17 12:28:14,923] jepsen worker 85 - jepsen.util 895 :invoke :read [28 nil] INFO [2019-12-17 12:28:14,923] jepsen worker 10 - jepsen.util 910 :invoke :cas [3 [5 12]] INFO [2019-12-17 12:28:14,923] jepsen worker 46 - jepsen.util 946 :invoke :write [15 38] INFO [2019-12-17 12:28:14,935] jepsen worker 29 - jepsen.util 659 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:28:14,935] jepsen worker 81 - jepsen.util 891 :info :cas [27 [21 32]] :RequestCanceledException INFO [2019-12-17 12:28:14,938] jepsen worker 81 - jepsen.util 981 :invoke :write [27 35] INFO [2019-12-17 12:28:14,939] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:14,958] jepsen worker 83 - jepsen.util 983 :fail :read [27 nil] :RequestTimeoutException INFO [2019-12-17 12:28:14,959] jepsen worker 83 - jepsen.util 983 :invoke :write [27 22] INFO [2019-12-17 12:28:14,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:14,995] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4128,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:14,995] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:15,001] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4132,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:15,002] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:15,007] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4133,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:15,023] jepsen worker 32 - jepsen.util 752 :info :write [10 25] :RequestCanceledException INFO [2019-12-17 12:28:15,023] jepsen worker 23 - jepsen.util 743 :fail :cas [7 [26 26]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,026] jepsen worker 39 - jepsen.util 849 :info :write [13 42] :RequestCanceledException INFO [2019-12-17 12:28:15,026] jepsen worker 23 - jepsen.util 743 :invoke :read [7 nil] INFO [2019-12-17 12:28:15,040] jepsen worker 39 - jepsen.util 939 :invoke :write [13 38] INFO [2019-12-17 12:28:15,041] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [25 11]] INFO [2019-12-17 12:28:15,042] jepsen worker 68 - jepsen.util 428 :info :write [22 15] :RequestCanceledException INFO [2019-12-17 12:28:15,042] jepsen worker 27 - jepsen.util 567 :info :write [9 33] :RequestCanceledException INFO [2019-12-17 12:28:15,042] jepsen worker 30 - jepsen.util 750 :info :write [10 31] :RequestCanceledException INFO [2019-12-17 12:28:15,049] jepsen worker 30 - jepsen.util 840 :invoke :cas [10 [28 40]] INFO [2019-12-17 12:28:15,050] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,050] jepsen worker 68 - jepsen.util 518 :invoke :write [22 12] INFO [2019-12-17 12:28:15,098] jepsen worker 51 - jepsen.util 771 :fail :cas [17 [21 24]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,098] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [9 7]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,099] jepsen worker 51 - jepsen.util 771 :invoke :write [17 9] INFO [2019-12-17 12:28:15,099] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:15,105] jepsen worker 21 - jepsen.util 921 :info :cas [7 [27 20]] :RequestCanceledException INFO [2019-12-17 12:28:15,110] jepsen worker 21 - jepsen.util 1011 :invoke :cas [7 [29 11]] INFO [2019-12-17 12:28:15,121] jepsen worker 35 - jepsen.util 665 :info :write [11 21] :RequestCanceledException INFO [2019-12-17 12:28:15,121] jepsen worker 77 - jepsen.util 707 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:15,123] jepsen worker 12 - jepsen.util 822 :fail :cas [4 [40 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,123] jepsen worker 12 - jepsen.util 822 :invoke :write [4 11] INFO [2019-12-17 12:28:15,124] jepsen worker 77 - jepsen.util 797 :invoke :write [25 30] INFO [2019-12-17 12:28:15,124] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:15,124] jepsen worker 89 - jepsen.util 989 :invoke :cas [29 [17 1]] INFO [2019-12-17 12:28:15,125] jepsen worker 35 - jepsen.util 755 :invoke :read [11 nil] INFO [2019-12-17 12:28:15,146] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:15,148] jepsen nemesis - couchbase.util Rebalance check count: 25 INFO [2019-12-17 12:28:15,148] jepsen nemesis - couchbase.util Rebalance stuck count: 6 INFO [2019-12-17 12:28:15,154] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,154] jepsen worker 7 - jepsen.util 997 :info :write [2 48] :RequestCanceledException INFO [2019-12-17 12:28:15,154] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,156] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [32 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,158] jepsen worker 7 - jepsen.util 1087 :invoke :read [2 nil] INFO [2019-12-17 12:28:15,159] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [19 36]] INFO [2019-12-17 12:28:15,167] jepsen worker 4 - jepsen.util 724 :ok :read [1 49] INFO [2019-12-17 12:28:15,168] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [18 31]] INFO [2019-12-17 12:28:15,183] jepsen worker 80 - jepsen.util 980 :ok :read [26 43] INFO [2019-12-17 12:28:15,184] jepsen worker 80 - jepsen.util 980 :invoke :cas [26 [38 0]] INFO [2019-12-17 12:28:15,199] jepsen worker 23 - jepsen.util 743 :ok :read [7 20] INFO [2019-12-17 12:28:15,199] jepsen worker 23 - jepsen.util 743 :invoke :write [7 40] INFO [2019-12-17 12:28:15,202] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,202] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,215] jepsen worker 37 - jepsen.util 757 :info :write [12 49] :RequestCanceledException INFO [2019-12-17 12:28:15,220] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:15,234] jepsen worker 28 - jepsen.util 478 :info :cas [9 [35 37]] :RequestCanceledException INFO [2019-12-17 12:28:15,235] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,235] jepsen worker 27 - jepsen.util 657 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,235] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":3840,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,236] jepsen worker 28 - jepsen.util 568 :invoke :cas [9 [35 25]] INFO [2019-12-17 12:28:15,259] jepsen worker 27 - jepsen.util 657 :ok :read [9 46] INFO [2019-12-17 12:28:15,259] jepsen worker 27 - jepsen.util 657 :invoke :write [9 14] INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4153,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:15,263] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4154,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:15,273] jepsen worker 53 - jepsen.util 953 :ok :read [17 46] INFO [2019-12-17 12:28:15,273] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:15,282] jepsen worker 74 - jepsen.util 614 :info :cas [24 [1 10]] :RequestCanceledException INFO [2019-12-17 12:28:15,284] jepsen worker 74 - jepsen.util 704 :invoke :read [24 nil] INFO [2019-12-17 12:28:15,285] jepsen worker 13 - jepsen.util 1003 :ok :read [4 49] INFO [2019-12-17 12:28:15,285] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 46] INFO [2019-12-17 12:28:15,293] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [36 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,294] jepsen worker 75 - jepsen.util 885 :ok :read [25 2] INFO [2019-12-17 12:28:15,294] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [20 23]] INFO [2019-12-17 12:28:15,294] jepsen worker 75 - jepsen.util 885 :invoke :write [25 23] INFO [2019-12-17 12:28:15,303] jepsen worker 57 - jepsen.util 867 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:15,303] jepsen worker 57 - jepsen.util 867 :invoke :cas [19 [44 43]] INFO [2019-12-17 12:28:15,304] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:15,305] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4164,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:15,312] jepsen worker 59 - jepsen.util 779 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:15,312] jepsen worker 59 - jepsen.util 779 :invoke :write [19 0] INFO [2019-12-17 12:28:15,322] jepsen worker 27 - jepsen.util 657 :ok :write [9 14] INFO [2019-12-17 12:28:15,322] jepsen worker 27 - jepsen.util 657 :invoke :write [9 25] INFO [2019-12-17 12:28:15,325] jepsen worker 49 - jepsen.util 679 :info :cas [16 [22 30]] :RequestCanceledException INFO [2019-12-17 12:28:15,327] jepsen worker 49 - jepsen.util 769 :invoke :write [16 42] INFO [2019-12-17 12:28:15,343] jepsen worker 69 - jepsen.util 879 :fail :cas [23 [25 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,344] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:15,349] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [1 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,350] jepsen worker 55 - jepsen.util 1135 :invoke :write [18 9] INFO [2019-12-17 12:28:15,350] jepsen worker 71 - jepsen.util 701 :fail :cas [23 [17 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,351] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:15,375] jepsen worker 27 - jepsen.util 657 :ok :write [9 25] INFO [2019-12-17 12:28:15,375] jepsen worker 27 - jepsen.util 657 :invoke :write [9 40] INFO [2019-12-17 12:28:15,387] jepsen worker 22 - jepsen.util 742 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:28:15,387] jepsen worker 8 - jepsen.util 818 :info :write [2 45] :RequestCanceledException INFO [2019-12-17 12:28:15,389] jepsen worker 22 - jepsen.util 832 :invoke :read [7 nil] INFO [2019-12-17 12:28:15,389] jepsen worker 8 - jepsen.util 908 :invoke :cas [2 [19 7]] INFO [2019-12-17 12:28:15,405] jepsen worker 60 - jepsen.util 960 :info :write [20 30] :RequestCanceledException INFO [2019-12-17 12:28:15,407] jepsen worker 60 - jepsen.util 1050 :invoke :cas [20 [38 13]] INFO [2019-12-17 12:28:15,429] jepsen worker 53 - jepsen.util 953 :ok :read [17 46] INFO [2019-12-17 12:28:15,429] jepsen worker 53 - jepsen.util 953 :invoke :write [17 23] INFO [2019-12-17 12:28:15,438] jepsen worker 89 - jepsen.util 989 :fail :cas [29 [17 1]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,438] jepsen worker 89 - jepsen.util 989 :invoke :read [29 nil] INFO [2019-12-17 12:28:15,441] jepsen worker 27 - jepsen.util 657 :ok :write [9 40] INFO [2019-12-17 12:28:15,442] jepsen worker 27 - jepsen.util 657 :invoke :write [9 9] INFO [2019-12-17 12:28:15,458] jepsen worker 34 - jepsen.util 934 :ok :read [11 7] INFO [2019-12-17 12:28:15,459] jepsen worker 34 - jepsen.util 934 :invoke :read [11 nil] INFO [2019-12-17 12:28:15,489] jepsen worker 27 - jepsen.util 657 :ok :write [9 9] INFO [2019-12-17 12:28:15,489] jepsen worker 27 - jepsen.util 657 :invoke :cas [9 [7 10]] INFO [2019-12-17 12:28:15,506] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [25 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,507] jepsen worker 62 - jepsen.util 962 :invoke :read [20 nil] INFO [2019-12-17 12:28:15,510] jepsen worker 73 - jepsen.util 793 :ok :write [24 37] INFO [2019-12-17 12:28:15,510] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [26 39]] INFO [2019-12-17 12:28:15,512] jepsen worker 27 - jepsen.util 657 :fail :cas [9 [7 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,513] jepsen worker 27 - jepsen.util 657 :invoke :write [9 24] INFO [2019-12-17 12:28:15,518] jepsen worker 21 - jepsen.util 1011 :fail :cas [7 [29 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,518] jepsen worker 21 - jepsen.util 1011 :invoke :write [7 49] INFO [2019-12-17 12:28:15,520] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:15,520] jepsen worker 43 - jepsen.util 943 :info :write [14 33] :RequestCanceledException INFO [2019-12-17 12:28:15,520] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4189,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:15,521] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:15,521] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4192,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:15,523] jepsen worker 43 - jepsen.util 1033 :invoke :read [14 nil] INFO [2019-12-17 12:28:15,543] jepsen worker 34 - jepsen.util 934 :ok :read [11 7] INFO [2019-12-17 12:28:15,544] jepsen worker 34 - jepsen.util 934 :invoke :write [11 20] INFO [2019-12-17 12:28:15,553] jepsen worker 9 - jepsen.util 729 :ok :write [3 3] INFO [2019-12-17 12:28:15,553] jepsen worker 9 - jepsen.util 729 :invoke :write [3 5] INFO [2019-12-17 12:28:15,554] jepsen worker 45 - jepsen.util 1035 :ok :write [15 17] INFO [2019-12-17 12:28:15,554] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 26] INFO [2019-12-17 12:28:15,563] jepsen worker 18 - jepsen.util 828 :ok :write [6 32] INFO [2019-12-17 12:28:15,564] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:15,564] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [45 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,564] jepsen worker 72 - jepsen.util 792 :invoke :write [24 42] INFO [2019-12-17 12:28:15,587] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [25 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,587] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:15,601] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [26 39]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,602] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [37 38]] INFO [2019-12-17 12:28:15,627] jepsen worker 37 - jepsen.util 847 :ok :read [12 45] INFO [2019-12-17 12:28:15,629] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [41 2]] INFO [2019-12-17 12:28:15,634] jepsen worker 54 - jepsen.util 864 :info :cas [18 [13 39]] :RequestCanceledException INFO [2019-12-17 12:28:15,646] jepsen worker 54 - jepsen.util 954 :invoke :read [18 nil] INFO [2019-12-17 12:28:15,647] jepsen worker 62 - jepsen.util 962 :ok :read [20 7] INFO [2019-12-17 12:28:15,647] jepsen worker 65 - jepsen.util 695 :ok :read [21 4] INFO [2019-12-17 12:28:15,650] jepsen worker 65 - jepsen.util 695 :invoke :cas [21 [0 38]] INFO [2019-12-17 12:28:15,650] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [25 37]] INFO [2019-12-17 12:28:15,657] jepsen worker 28 - jepsen.util 568 :fail :cas [9 [35 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,657] jepsen worker 28 - jepsen.util 568 :invoke :cas [9 [20 27]] INFO [2019-12-17 12:28:15,668] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:15,668] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4213,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4216,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:15,669] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4217,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:15,676] jepsen worker 31 - jepsen.util 931 :ok :read [10 23] INFO [2019-12-17 12:28:15,677] jepsen worker 31 - jepsen.util 931 :invoke :write [10 0] INFO [2019-12-17 12:28:15,684] jepsen worker 35 - jepsen.util 755 :ok :read [11 7] INFO [2019-12-17 12:28:15,684] jepsen worker 35 - jepsen.util 755 :invoke :cas [11 [31 37]] INFO [2019-12-17 12:28:15,695] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:15,695] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:15,703] jepsen worker 44 - jepsen.util 674 :fail :read [14 nil] :RequestTimeoutException INFO [2019-12-17 12:28:15,703] jepsen worker 44 - jepsen.util 674 :invoke :cas [14 [38 43]] INFO [2019-12-17 12:28:15,704] jepsen worker 29 - jepsen.util 749 :ok :read [9 9] INFO [2019-12-17 12:28:15,704] jepsen worker 29 - jepsen.util 749 :invoke :write [9 28] INFO [2019-12-17 12:28:15,707] jepsen worker 13 - jepsen.util 1003 :ok :write [4 46] INFO [2019-12-17 12:28:15,707] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [5 42]] INFO [2019-12-17 12:28:15,721] jepsen worker 32 - jepsen.util 842 :ok :read [10 23] INFO [2019-12-17 12:28:15,722] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [10 14]] INFO [2019-12-17 12:28:15,735] jepsen worker 89 - jepsen.util 989 :ok :read [29 48] INFO [2019-12-17 12:28:15,735] jepsen worker 89 - jepsen.util 989 :invoke :write [29 10] INFO [2019-12-17 12:28:15,749] jepsen worker 27 - jepsen.util 657 :ok :write [9 24] INFO [2019-12-17 12:28:15,749] jepsen worker 27 - jepsen.util 657 :invoke :write [9 21] INFO [2019-12-17 12:28:15,770] jepsen worker 74 - jepsen.util 704 :ok :read [24 37] INFO [2019-12-17 12:28:15,771] jepsen worker 74 - jepsen.util 704 :invoke :cas [24 [49 33]] INFO [2019-12-17 12:28:15,787] jepsen worker 36 - jepsen.util 936 :info :write [12 47] :RequestCanceledException INFO [2019-12-17 12:28:15,791] jepsen worker 47 - jepsen.util 1037 :ok :read [15 17] INFO [2019-12-17 12:28:15,791] jepsen worker 47 - jepsen.util 1037 :invoke :cas [15 [45 23]] INFO [2019-12-17 12:28:15,792] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:15,794] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [19 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,794] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [29 19]] INFO [2019-12-17 12:28:15,807] jepsen worker 18 - jepsen.util 828 :ok :read [6 32] INFO [2019-12-17 12:28:15,808] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:15,823] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [41 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,824] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:15,869] jepsen worker 29 - jepsen.util 749 :ok :write [9 28] INFO [2019-12-17 12:28:15,869] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [6 35]] INFO [2019-12-17 12:28:15,891] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,891] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [10 36]] INFO [2019-12-17 12:28:15,904] jepsen worker 60 - jepsen.util 1050 :fail :cas [20 [38 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,904] jepsen worker 60 - jepsen.util 1050 :invoke :cas [20 [11 41]] INFO [2019-12-17 12:28:15,911] jepsen worker 18 - jepsen.util 828 :ok :read [6 32] INFO [2019-12-17 12:28:15,911] jepsen worker 18 - jepsen.util 828 :invoke :cas [6 [21 17]] INFO [2019-12-17 12:28:15,932] jepsen worker 26 - jepsen.util 926 :info :write [8 6] :RequestCanceledException INFO [2019-12-17 12:28:15,934] jepsen worker 26 - jepsen.util 1016 :invoke :write [8 8] INFO [2019-12-17 12:28:15,942] jepsen worker 36 - jepsen.util 1026 :ok :read [12 45] INFO [2019-12-17 12:28:15,942] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [29 17]] INFO [2019-12-17 12:28:15,944] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [5 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,944] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:15,946] jepsen worker 29 - jepsen.util 749 :fail :cas [9 [6 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,947] jepsen worker 29 - jepsen.util 749 :invoke :cas [9 [9 38]] INFO [2019-12-17 12:28:15,949] jepsen worker 71 - jepsen.util 701 :ok :read [23 22] INFO [2019-12-17 12:28:15,949] jepsen worker 71 - jepsen.util 701 :invoke :write [23 16] INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4238,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4241,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:15,954] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4242,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:15,966] jepsen worker 18 - jepsen.util 828 :fail :cas [6 [21 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,966] jepsen worker 18 - jepsen.util 828 :invoke :read [6 nil] INFO [2019-12-17 12:28:15,984] jepsen worker 29 - jepsen.util 749 :fail :cas [9 [9 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,984] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:15,984] jepsen worker 58 - jepsen.util 778 :info :write [19 44] :RequestCanceledException INFO [2019-12-17 12:28:15,986] jepsen worker 58 - jepsen.util 868 :invoke :cas [19 [40 27]] INFO [2019-12-17 12:28:15,989] jepsen worker 43 - jepsen.util 1033 :ok :read [14 27] INFO [2019-12-17 12:28:15,989] jepsen worker 43 - jepsen.util 1033 :invoke :read [14 nil] INFO [2019-12-17 12:28:15,994] jepsen worker 80 - jepsen.util 980 :fail :cas [26 [38 0]] :ValueNotSwapFrom INFO [2019-12-17 12:28:15,994] jepsen worker 80 - jepsen.util 980 :invoke :write [26 29] INFO [2019-12-17 12:28:16,002] jepsen worker 17 - jepsen.util 827 :info :write [5 28] :RequestCanceledException INFO [2019-12-17 12:28:16,004] jepsen worker 17 - jepsen.util 917 :invoke :write [5 18] INFO [2019-12-17 12:28:16,015] jepsen worker 18 - jepsen.util 828 :ok :read [6 32] INFO [2019-12-17 12:28:16,015] jepsen worker 18 - jepsen.util 828 :invoke :write [6 47] INFO [2019-12-17 12:28:16,021] jepsen worker 6 - jepsen.util 816 :info :write [2 9] :RequestCanceledException INFO [2019-12-17 12:28:16,023] jepsen worker 6 - jepsen.util 906 :invoke :write [2 14] INFO [2019-12-17 12:28:16,028] jepsen worker 70 - jepsen.util 1060 :info :write [23 24] :RequestCanceledException INFO [2019-12-17 12:28:16,030] jepsen worker 70 - jepsen.util 1150 :invoke :write [23 6] INFO [2019-12-17 12:28:16,039] jepsen worker 29 - jepsen.util 749 :ok :read [9 28] INFO [2019-12-17 12:28:16,040] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:16,047] jepsen worker 5 - jepsen.util 545 :info :write [1 45] :RequestCanceledException INFO [2019-12-17 12:28:16,047] jepsen worker 0 - jepsen.util 990 :info :write [0 11] :RequestCanceledException INFO [2019-12-17 12:28:16,047] jepsen worker 14 - jepsen.util 734 :info :write [4 15] :RequestCanceledException INFO [2019-12-17 12:28:16,049] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:16,049] jepsen worker 0 - jepsen.util 1080 :invoke :read [0 nil] INFO [2019-12-17 12:28:16,049] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:16,081] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:16,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4256,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:16,082] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:16,082] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4259,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:16,082] jepsen worker 54 - jepsen.util 954 :ok :read [18 13] INFO [2019-12-17 12:28:16,082] jepsen worker 54 - jepsen.util 954 :invoke :write [18 26] INFO [2019-12-17 12:28:16,084] jepsen worker 9 - jepsen.util 729 :ok :write [3 5] INFO [2019-12-17 12:28:16,084] jepsen worker 9 - jepsen.util 729 :invoke :read [3 nil] INFO [2019-12-17 12:28:16,097] jepsen worker 72 - jepsen.util 792 :ok :write [24 42] INFO [2019-12-17 12:28:16,097] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [46 45]] INFO [2019-12-17 12:28:16,100] jepsen worker 87 - jepsen.util 807 :info :write [29 21] :RequestCanceledException INFO [2019-12-17 12:28:16,104] jepsen worker 87 - jepsen.util 897 :invoke :cas [29 [2 35]] INFO [2019-12-17 12:28:16,112] jepsen worker 43 - jepsen.util 1033 :ok :read [14 27] INFO [2019-12-17 12:28:16,112] jepsen worker 43 - jepsen.util 1033 :invoke :write [14 45] INFO [2019-12-17 12:28:16,118] jepsen worker 67 - jepsen.util 1237 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,118] jepsen worker 67 - jepsen.util 1237 :invoke :read [22 nil] INFO [2019-12-17 12:28:16,146] jepsen worker 65 - jepsen.util 695 :fail :cas [21 [0 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,146] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [18 31]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,146] jepsen worker 65 - jepsen.util 695 :invoke :cas [21 [18 32]] INFO [2019-12-17 12:28:16,146] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:16,147] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [29 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,147] jepsen worker 42 - jepsen.util 1122 :invoke :cas [14 [25 6]] INFO [2019-12-17 12:28:16,152] jepsen worker 20 - jepsen.util 830 :ok :write [6 1] INFO [2019-12-17 12:28:16,153] jepsen worker 20 - jepsen.util 830 :invoke :write [6 6] INFO [2019-12-17 12:28:16,175] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:16,183] jepsen nemesis - couchbase.util Rebalance check count: 26 INFO [2019-12-17 12:28:16,183] jepsen nemesis - couchbase.util Rebalance stuck count: 7 INFO [2019-12-17 12:28:16,211] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [37 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,215] jepsen worker 73 - jepsen.util 793 :invoke :write [24 0] INFO [2019-12-17 12:28:16,221] jepsen worker 41 - jepsen.util 581 :info :cas [13 [49 32]] :RequestCanceledException INFO [2019-12-17 12:28:16,227] jepsen worker 25 - jepsen.util 1015 :info :write [8 49] :RequestCanceledException INFO [2019-12-17 12:28:16,231] jepsen worker 50 - jepsen.util 950 :info :write [16 15] :RequestCanceledException INFO [2019-12-17 12:28:16,243] jepsen worker 79 - jepsen.util 709 :info :write [26 32] :RequestCanceledException INFO [2019-12-17 12:28:16,244] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:16,244] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4280,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:16,247] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:16,247] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4282,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:16,284] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:16,289] jepsen worker 25 - jepsen.util 1105 :invoke :cas [8 [11 11]] INFO [2019-12-17 12:28:16,293] jepsen worker 41 - jepsen.util 671 :invoke :cas [13 [6 8]] INFO [2019-12-17 12:28:16,295] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4287,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:16,296] jepsen worker 45 - jepsen.util 1035 :ok :write [15 26] INFO [2019-12-17 12:28:16,307] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 25] INFO [2019-12-17 12:28:16,310] jepsen worker 88 - jepsen.util 808 :info :write [29 38] :RequestCanceledException INFO [2019-12-17 12:28:16,311] jepsen worker 76 - jepsen.util 886 :info :write [25 22] :RequestCanceledException INFO [2019-12-17 12:28:16,311] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:16,340] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4289,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:16,341] jepsen worker 82 - jepsen.util 1162 :info :cas [27 [27 0]] :RequestCanceledException INFO [2019-12-17 12:28:16,345] jepsen worker 79 - jepsen.util 799 :invoke :read [26 nil] INFO [2019-12-17 12:28:16,349] jepsen worker 50 - jepsen.util 1040 :invoke :cas [16 [28 33]] INFO [2019-12-17 12:28:16,372] jepsen worker 76 - jepsen.util 976 :invoke :read [25 nil] INFO [2019-12-17 12:28:16,374] jepsen worker 82 - jepsen.util 1252 :invoke :cas [27 [8 37]] INFO [2019-12-17 12:28:16,381] jepsen worker 88 - jepsen.util 898 :invoke :write [29 4] INFO [2019-12-17 12:28:16,419] jepsen worker 61 - jepsen.util 961 :info :write [20 26] :RequestCanceledException INFO [2019-12-17 12:28:16,436] jepsen worker 15 - jepsen.util 735 :info :write [5 14] :RequestCanceledException INFO [2019-12-17 12:28:16,468] jepsen worker 40 - jepsen.util 1210 :info :cas [13 [38 42]] :RequestCanceledException INFO [2019-12-17 12:28:16,478] jepsen worker 16 - jepsen.util 826 :info :cas [5 [26 1]] :RequestCanceledException INFO [2019-12-17 12:28:16,492] jepsen worker 24 - jepsen.util 744 :info :write [8 44] :RequestCanceledException INFO [2019-12-17 12:28:16,509] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:16,509] jepsen worker 78 - jepsen.util 978 :ok :read [26 43] INFO [2019-12-17 12:28:16,526] jepsen worker 71 - jepsen.util 701 :ok :write [23 16] INFO [2019-12-17 12:28:16,541] jepsen worker 61 - jepsen.util 1051 :invoke :cas [20 [29 37]] INFO [2019-12-17 12:28:16,542] jepsen worker 15 - jepsen.util 825 :invoke :cas [5 [38 30]] INFO [2019-12-17 12:28:16,580] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:16,581] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4301,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:16,582] jepsen worker 78 - jepsen.util 978 :invoke :cas [26 [33 29]] INFO [2019-12-17 12:28:16,597] jepsen worker 66 - jepsen.util 696 :info :cas [22 [19 15]] :RequestCanceledException INFO [2019-12-17 12:28:16,607] jepsen worker 40 - jepsen.util 1300 :invoke :write [13 2] INFO [2019-12-17 12:28:16,611] jepsen worker 16 - jepsen.util 916 :invoke :cas [5 [31 21]] INFO [2019-12-17 12:28:16,614] jepsen worker 24 - jepsen.util 834 :invoke :write [8 28] INFO [2019-12-17 12:28:16,623] jepsen worker 3 - jepsen.util 633 :info :write [1 43] :RequestCanceledException INFO [2019-12-17 12:28:16,625] jepsen worker 66 - jepsen.util 786 :invoke :cas [22 [40 36]] INFO [2019-12-17 12:28:16,638] jepsen worker 2 - jepsen.util 632 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,638] jepsen worker 2 - jepsen.util 632 :invoke :cas [0 [10 23]] INFO [2019-12-17 12:28:16,638] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,639] jepsen worker 48 - jepsen.util 498 :invoke :read [16 nil] INFO [2019-12-17 12:28:16,639] jepsen worker 33 - jepsen.util 1023 :info :write [11 15] :RequestCanceledException INFO [2019-12-17 12:28:16,657] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:16,658] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4315,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:16,686] jepsen worker 3 - jepsen.util 723 :invoke :read [1 nil] INFO [2019-12-17 12:28:16,705] jepsen worker 33 - jepsen.util 1113 :invoke :cas [11 [21 6]] INFO [2019-12-17 12:28:16,717] jepsen worker 28 - jepsen.util 568 :fail :cas [9 [20 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,719] jepsen worker 28 - jepsen.util 568 :invoke :read [9 nil] INFO [2019-12-17 12:28:16,724] jepsen worker 56 - jepsen.util 776 :info :write [18 7] :RequestCanceledException INFO [2019-12-17 12:28:16,727] jepsen worker 38 - jepsen.util 758 :info :write [12 12] :RequestCanceledException INFO [2019-12-17 12:28:16,739] jepsen worker 55 - jepsen.util 1135 :ok :write [18 9] INFO [2019-12-17 12:28:16,740] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [5 9]] INFO [2019-12-17 12:28:16,751] jepsen worker 56 - jepsen.util 866 :invoke :read [18 nil] INFO [2019-12-17 12:28:16,752] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:16,797] jepsen worker 9 - jepsen.util 729 :ok :read [3 5] INFO [2019-12-17 12:28:16,798] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [4 35]] INFO [2019-12-17 12:28:16,807] jepsen worker 69 - jepsen.util 879 :ok :read [23 22] INFO [2019-12-17 12:28:16,808] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [10 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,808] jepsen worker 32 - jepsen.util 842 :invoke :write [10 27] INFO [2019-12-17 12:28:16,809] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:16,815] jepsen worker 60 - jepsen.util 1050 :fail :cas [20 [11 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,815] jepsen worker 60 - jepsen.util 1050 :invoke :cas [20 [43 23]] INFO [2019-12-17 12:28:16,827] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [46 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,827] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [10 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,827] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [10 12]] INFO [2019-12-17 12:28:16,827] jepsen worker 52 - jepsen.util 862 :invoke :read [17 nil] INFO [2019-12-17 12:28:16,831] jepsen worker 37 - jepsen.util 847 :ok :read [12 45] INFO [2019-12-17 12:28:16,831] jepsen worker 22 - jepsen.util 832 :ok :read [7 20] INFO [2019-12-17 12:28:16,833] jepsen worker 22 - jepsen.util 832 :invoke :write [7 2] INFO [2019-12-17 12:28:16,834] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [4 36]] INFO [2019-12-17 12:28:16,834] jepsen worker 4 - jepsen.util 724 :ok :read [1 49] INFO [2019-12-17 12:28:16,835] jepsen worker 4 - jepsen.util 724 :invoke :write [1 20] INFO [2019-12-17 12:28:16,835] jepsen worker 65 - jepsen.util 695 :fail :cas [21 [18 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,835] jepsen worker 1 - jepsen.util 631 :info :write [0 19] :RequestCanceledException INFO [2019-12-17 12:28:16,836] jepsen worker 65 - jepsen.util 695 :invoke :write [21 35] INFO [2019-12-17 12:28:16,836] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [25 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,836] jepsen worker 62 - jepsen.util 962 :invoke :cas [20 [13 13]] INFO [2019-12-17 12:28:16,841] jepsen worker 87 - jepsen.util 897 :fail :cas [29 [2 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,841] jepsen worker 87 - jepsen.util 897 :invoke :read [29 nil] INFO [2019-12-17 12:28:16,844] jepsen worker 1 - jepsen.util 721 :invoke :read [0 nil] INFO [2019-12-17 12:28:16,853] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:16,853] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4341,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:16,854] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:16,854] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4344,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:16,858] jepsen worker 13 - jepsen.util 1003 :ok :read [4 46] INFO [2019-12-17 12:28:16,859] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:16,861] jepsen worker 28 - jepsen.util 568 :ok :read [9 28] INFO [2019-12-17 12:28:16,862] jepsen worker 28 - jepsen.util 568 :invoke :write [9 6] INFO [2019-12-17 12:28:16,880] jepsen worker 9 - jepsen.util 729 :fail :cas [3 [4 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,880] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [5 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,880] jepsen worker 9 - jepsen.util 729 :invoke :cas [3 [14 33]] INFO [2019-12-17 12:28:16,880] jepsen worker 55 - jepsen.util 1135 :invoke :read [18 nil] INFO [2019-12-17 12:28:16,899] jepsen worker 53 - jepsen.util 953 :ok :write [17 23] INFO [2019-12-17 12:28:16,899] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:16,919] jepsen worker 84 - jepsen.util 624 :info :write [28 12] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 19 - jepsen.util 919 :info :write [6 9] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 63 - jepsen.util 603 :info :write [21 8] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 11 - jepsen.util 1001 :info :cas [3 [5 16]] :RequestCanceledException INFO [2019-12-17 12:28:16,922] jepsen worker 64 - jepsen.util 514 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,922] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [15 17]] INFO [2019-12-17 12:28:16,923] jepsen worker 84 - jepsen.util 714 :invoke :cas [28 [4 40]] INFO [2019-12-17 12:28:16,924] jepsen worker 11 - jepsen.util 1091 :invoke :read [3 nil] INFO [2019-12-17 12:28:16,924] jepsen worker 19 - jepsen.util 1009 :invoke :write [6 39] INFO [2019-12-17 12:28:16,924] jepsen worker 63 - jepsen.util 693 :invoke :cas [21 [33 21]] INFO [2019-12-17 12:28:16,945] jepsen worker 42 - jepsen.util 1122 :fail :cas [14 [25 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:16,945] jepsen worker 42 - jepsen.util 1122 :invoke :read [14 nil] INFO [2019-12-17 12:28:16,949] jepsen worker 86 - jepsen.util 806 :info :cas [28 [14 34]] :RequestCanceledException INFO [2019-12-17 12:28:16,951] jepsen worker 86 - jepsen.util 896 :invoke :cas [28 [29 40]] INFO [2019-12-17 12:28:16,962] jepsen worker 11 - jepsen.util 1091 :ok :read [3 5] INFO [2019-12-17 12:28:16,963] jepsen worker 11 - jepsen.util 1091 :invoke :write [3 23] INFO [2019-12-17 12:28:16,987] jepsen worker 85 - jepsen.util 895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:16,987] jepsen worker 85 - jepsen.util 895 :invoke :read [28 nil] INFO [2019-12-17 12:28:16,993] jepsen worker 65 - jepsen.util 695 :ok :write [21 35] INFO [2019-12-17 12:28:16,993] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,003] jepsen worker 10 - jepsen.util 910 :info :cas [3 [5 12]] :RequestCanceledException INFO [2019-12-17 12:28:17,005] jepsen worker 10 - jepsen.util 1000 :invoke :cas [3 [21 8]] INFO [2019-12-17 12:28:17,006] jepsen worker 64 - jepsen.util 514 :fail :cas [21 [15 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,006] jepsen worker 64 - jepsen.util 514 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,017] jepsen worker 46 - jepsen.util 946 :info :write [15 38] :RequestCanceledException INFO [2019-12-17 12:28:17,020] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [4 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,020] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [7 38]] INFO [2019-12-17 12:28:17,022] jepsen worker 46 - jepsen.util 1036 :invoke :cas [15 [21 22]] INFO [2019-12-17 12:28:17,025] jepsen worker 81 - jepsen.util 981 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:28:17,029] jepsen worker 33 - jepsen.util 1113 :fail :cas [11 [21 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,030] jepsen worker 33 - jepsen.util 1113 :invoke :cas [11 [37 27]] INFO [2019-12-17 12:28:17,030] jepsen worker 81 - jepsen.util 1071 :invoke :cas [27 [31 35]] INFO [2019-12-17 12:28:17,055] jepsen worker 65 - jepsen.util 695 :ok :read [21 35] INFO [2019-12-17 12:28:17,056] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,079] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:17,079] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4363,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4366,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:17,080] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4367,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:17,082] jepsen worker 10 - jepsen.util 1000 :fail :cas [3 [21 8]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,082] jepsen worker 10 - jepsen.util 1000 :invoke :write [3 45] INFO [2019-12-17 12:28:17,082] jepsen worker 46 - jepsen.util 1036 :fail :cas [15 [21 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,083] jepsen worker 46 - jepsen.util 1036 :invoke :cas [15 [42 49]] INFO [2019-12-17 12:28:17,084] jepsen worker 42 - jepsen.util 1122 :ok :read [14 27] INFO [2019-12-17 12:28:17,084] jepsen worker 42 - jepsen.util 1122 :invoke :write [14 27] INFO [2019-12-17 12:28:17,103] jepsen worker 30 - jepsen.util 840 :info :cas [10 [28 40]] :RequestCanceledException INFO [2019-12-17 12:28:17,105] jepsen worker 30 - jepsen.util 930 :invoke :write [10 10] INFO [2019-12-17 12:28:17,118] jepsen worker 39 - jepsen.util 939 :info :write [13 38] :RequestCanceledException INFO [2019-12-17 12:28:17,120] jepsen worker 39 - jepsen.util 1029 :invoke :write [13 9] INFO [2019-12-17 12:28:17,128] jepsen worker 68 - jepsen.util 518 :info :write [22 12] :RequestCanceledException INFO [2019-12-17 12:28:17,133] jepsen worker 68 - jepsen.util 608 :invoke :read [22 nil] INFO [2019-12-17 12:28:17,139] jepsen worker 56 - jepsen.util 866 :ok :read [18 9] INFO [2019-12-17 12:28:17,139] jepsen worker 56 - jepsen.util 866 :invoke :write [18 14] INFO [2019-12-17 12:28:17,146] jepsen worker 51 - jepsen.util 771 :info :write [17 9] :RequestCanceledException INFO [2019-12-17 12:28:17,146] jepsen worker 83 - jepsen.util 983 :info :write [27 22] :RequestCanceledException INFO [2019-12-17 12:28:17,148] jepsen worker 83 - jepsen.util 1073 :invoke :write [27 18] INFO [2019-12-17 12:28:17,148] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:17,151] jepsen worker 65 - jepsen.util 695 :ok :read [21 35] INFO [2019-12-17 12:28:17,151] jepsen worker 65 - jepsen.util 695 :invoke :write [21 39] INFO [2019-12-17 12:28:17,200] jepsen worker 46 - jepsen.util 1036 :fail :cas [15 [42 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:17,200] jepsen worker 46 - jepsen.util 1036 :invoke :write [15 41] INFO [2019-12-17 12:28:17,200] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:17,200] jepsen nemesis - couchbase.util Rebalance check count: 27 INFO [2019-12-17 12:28:17,200] jepsen nemesis - couchbase.util Rebalance stuck count: 8 INFO [2019-12-17 12:28:17,206] jepsen worker 12 - jepsen.util 822 :info :write [4 11] :RequestCanceledException INFO [2019-12-17 12:28:17,206] jepsen worker 7 - jepsen.util 1087 :fail :read [2 nil] :RequestTimeoutException INFO [2019-12-17 12:28:17,207] jepsen worker 7 - jepsen.util 1087 :invoke :write [2 36] INFO [2019-12-17 12:28:17,211] jepsen worker 12 - jepsen.util 912 :invoke :cas [4 [42 20]] INFO [2019-12-17 12:28:17,220] jepsen worker 77 - jepsen.util 797 :info :write [25 30] :RequestCanceledException INFO [2019-12-17 12:28:17,223] jepsen worker 87 - jepsen.util 897 :ok :read [29 48] INFO [2019-12-17 12:28:17,223] jepsen worker 87 - jepsen.util 897 :invoke :write [29 23] INFO [2019-12-17 12:28:17,224] jepsen worker 77 - jepsen.util 887 :invoke :write [25 10] INFO [2019-12-17 12:28:17,225] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4394,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4398,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4387,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:17,226] jepsen worker 23 - jepsen.util 743 :info :write [7 40] :RequestCanceledException INFO [2019-12-17 12:28:17,226] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4388,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:17,227] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:17,227] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4389,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:17,234] jepsen worker 23 - jepsen.util 833 :invoke :cas [7 [11 10]] INFO [2019-12-17 12:28:17,279] jepsen worker 65 - jepsen.util 695 :ok :write [21 39] INFO [2019-12-17 12:28:17,298] jepsen worker 73 - jepsen.util 793 :ok :write [24 0] INFO [2019-12-17 12:28:17,308] jepsen worker 5 - jepsen.util 635 :ok :read [1 49] INFO [2019-12-17 12:28:17,332] jepsen worker 88 - jepsen.util 898 :ok :write [29 4] INFO [2019-12-17 12:28:17,361] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:17,361] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [39 49]] INFO [2019-12-17 12:28:17,363] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [11 46]] INFO [2019-12-17 12:28:17,366] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:17,403] jepsen worker 4 - jepsen.util 724 :ok :write [1 20] INFO [2019-12-17 12:28:17,404] jepsen worker 75 - jepsen.util 885 :info :write [25 23] :RequestCanceledException INFO [2019-12-17 12:28:17,416] jepsen worker 20 - jepsen.util 830 :ok :write [6 6] INFO [2019-12-17 12:28:17,419] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [35 41]] INFO [2019-12-17 12:28:17,425] jepsen worker 20 - jepsen.util 830 :invoke :write [6 47] INFO [2019-12-17 12:28:17,438] jepsen worker 49 - jepsen.util 769 :info :write [16 42] :RequestCanceledException INFO [2019-12-17 12:28:17,439] jepsen worker 59 - jepsen.util 779 :info :write [19 0] :RequestCanceledException INFO [2019-12-17 12:28:17,440] jepsen worker 70 - jepsen.util 1150 :ok :write [23 6] INFO [2019-12-17 12:28:17,443] jepsen worker 57 - jepsen.util 867 :info :cas [19 [44 43]] :RequestCanceledException INFO [2019-12-17 12:28:17,445] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:17,445] jepsen worker 8 - jepsen.util 908 :info :cas [2 [19 7]] :RequestCanceledException INFO [2019-12-17 12:28:17,447] jepsen worker 75 - jepsen.util 975 :invoke :read [25 nil] INFO [2019-12-17 12:28:17,466] jepsen worker 11 - jepsen.util 1091 :ok :write [3 23] INFO [2019-12-17 12:28:17,477] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:17,493] jepsen worker 54 - jepsen.util 954 :ok :write [18 26] INFO [2019-12-17 12:28:17,493] jepsen worker 11 - jepsen.util 1091 :invoke :write [3 19] INFO [2019-12-17 12:28:17,502] jepsen worker 54 - jepsen.util 954 :invoke :write [18 39] INFO [2019-12-17 12:28:17,517] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4414,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:17,525] jepsen worker 59 - jepsen.util 869 :invoke :read [19 nil] INFO [2019-12-17 12:28:17,526] jepsen worker 45 - jepsen.util 1035 :ok :write [15 25] INFO [2019-12-17 12:28:17,526] jepsen worker 57 - jepsen.util 957 :invoke :cas [19 [9 28]] INFO [2019-12-17 12:28:17,536] jepsen worker 45 - jepsen.util 1035 :invoke :cas [15 [3 48]] INFO [2019-12-17 12:28:17,540] jepsen worker 32 - jepsen.util 842 :ok :write [10 27] INFO [2019-12-17 12:28:17,543] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [22 2]] INFO [2019-12-17 12:28:17,545] jepsen worker 49 - jepsen.util 859 :invoke :cas [16 [47 31]] INFO [2019-12-17 12:28:17,554] jepsen worker 56 - jepsen.util 866 :ok :write [18 14] INFO [2019-12-17 12:28:17,558] jepsen worker 56 - jepsen.util 866 :invoke :write [18 39] INFO [2019-12-17 12:28:17,558] jepsen worker 8 - jepsen.util 998 :invoke :cas [2 [25 49]] INFO [2019-12-17 12:28:17,558] jepsen worker 21 - jepsen.util 1011 :info :write [7 49] :RequestCanceledException INFO [2019-12-17 12:28:17,596] jepsen worker 89 - jepsen.util 989 :ok :write [29 10] INFO [2019-12-17 12:28:17,600] jepsen worker 89 - jepsen.util 989 :invoke :write [29 37] INFO [2019-12-17 12:28:17,637] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:17,670] jepsen worker 34 - jepsen.util 934 :info :write [11 20] :RequestCanceledException INFO [2019-12-17 12:28:17,708] jepsen worker 31 - jepsen.util 931 :info :write [10 0] :RequestCanceledException INFO [2019-12-17 12:28:17,742] jepsen worker 34 - jepsen.util 1024 :invoke :cas [11 [30 27]] INFO [2019-12-17 12:28:17,782] jepsen worker 31 - jepsen.util 1021 :invoke :cas [10 [28 17]] INFO [2019-12-17 12:28:17,827] jepsen worker 47 - jepsen.util 1037 :info :cas [15 [45 23]] :RequestCanceledException INFO [2019-12-17 12:28:17,832] jepsen worker 44 - jepsen.util 674 :info :cas [14 [38 43]] :RequestCanceledException INFO [2019-12-17 12:28:17,850] jepsen worker 27 - jepsen.util 657 :info :write [9 21] :RequestCanceledException INFO [2019-12-17 12:28:17,866] jepsen worker 74 - jepsen.util 704 :info :cas [24 [49 33]] :RequestCanceledException INFO [2019-12-17 12:28:17,890] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:17,905] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4437,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:17,913] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:17,930] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4440,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:17,964] jepsen worker 69 - jepsen.util 879 :ok :read [23 16] INFO [2019-12-17 12:28:17,993] jepsen worker 69 - jepsen.util 879 :invoke :write [23 4] INFO [2019-12-17 12:28:18,009] jepsen worker 44 - jepsen.util 764 :invoke :cas [14 [29 29]] INFO [2019-12-17 12:28:18,010] jepsen worker 26 - jepsen.util 1016 :info :write [8 8] :RequestCanceledException INFO [2019-12-17 12:28:18,038] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:18,039] jepsen worker 47 - jepsen.util 1127 :invoke :cas [15 [35 33]] INFO [2019-12-17 12:28:18,041] jepsen worker 58 - jepsen.util 868 :info :cas [19 [40 27]] :RequestCanceledException INFO [2019-12-17 12:28:18,045] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4445,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:18,145] jepsen worker 80 - jepsen.util 980 :info :write [26 29] :RequestCanceledException INFO [2019-12-17 12:28:18,145] jepsen worker 74 - jepsen.util 794 :invoke :write [24 11] INFO [2019-12-17 12:28:18,146] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [45 11]] INFO [2019-12-17 12:28:18,146] jepsen worker 29 - jepsen.util 749 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:18,147] jepsen worker 18 - jepsen.util 828 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:28:18,148] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:18,149] jepsen worker 58 - jepsen.util 958 :invoke :read [19 nil] INFO [2019-12-17 12:28:18,152] jepsen worker 35 - jepsen.util 755 :fail :cas [11 [31 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:18,152] jepsen worker 35 - jepsen.util 755 :invoke :read [11 nil] INFO [2019-12-17 12:28:18,155] jepsen worker 6 - jepsen.util 906 :info :write [2 14] :RequestCanceledException INFO [2019-12-17 12:28:18,166] jepsen worker 64 - jepsen.util 514 :ok :read [21 35] INFO [2019-12-17 12:28:18,166] jepsen worker 64 - jepsen.util 514 :invoke :write [21 21] INFO [2019-12-17 12:28:18,176] jepsen worker 17 - jepsen.util 917 :info :write [5 18] :RequestCanceledException INFO [2019-12-17 12:28:18,182] jepsen worker 0 - jepsen.util 1080 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,879] jepsen worker 0 - jepsen.util 1080 :invoke :write [0 6] INFO [2019-12-17 12:28:19,879] jepsen worker 26 - jepsen.util 1106 :invoke :read [8 nil] INFO [2019-12-17 12:28:19,879] jepsen worker 18 - jepsen.util 918 :invoke :cas [6 [37 25]] INFO [2019-12-17 12:28:19,879] jepsen worker 17 - jepsen.util 1007 :invoke :read [5 nil] INFO [2019-12-17 12:28:19,879] jepsen worker 80 - jepsen.util 1070 :invoke :write [26 49] INFO [2019-12-17 12:28:19,881] jepsen worker 6 - jepsen.util 996 :invoke :write [2 15] INFO [2019-12-17 12:28:19,883] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [29 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,884] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [44 29]] INFO [2019-12-17 12:28:19,887] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [22 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,887] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:19,890] jepsen worker 5 - jepsen.util 635 :ok :read [1 20] INFO [2019-12-17 12:28:19,890] jepsen worker 51 - jepsen.util 861 :ok :read [17 9] INFO [2019-12-17 12:28:19,890] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:19,891] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:19,893] jepsen worker 55 - jepsen.util 1135 :ok :read [18 9] INFO [2019-12-17 12:28:19,893] jepsen worker 25 - jepsen.util 1105 :fail :cas [8 [11 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,893] jepsen worker 25 - jepsen.util 1105 :invoke :write [8 3] INFO [2019-12-17 12:28:19,893] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [45 34]] INFO [2019-12-17 12:28:19,893] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [39 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,895] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [21 44]] INFO [2019-12-17 12:28:19,897] jepsen worker 71 - jepsen.util 701 :ok :read [23 6] INFO [2019-12-17 12:28:19,897] jepsen worker 71 - jepsen.util 701 :invoke :write [23 5] INFO [2019-12-17 12:28:19,898] jepsen worker 60 - jepsen.util 1050 :fail :cas [20 [43 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,899] jepsen worker 60 - jepsen.util 1050 :invoke :write [20 10] INFO [2019-12-17 12:28:19,904] jepsen worker 14 - jepsen.util 824 :ok :read [4 11] INFO [2019-12-17 12:28:19,904] jepsen worker 31 - jepsen.util 1021 :fail :cas [10 [28 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,904] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:19,904] jepsen worker 3 - jepsen.util 723 :ok :read [1 43] INFO [2019-12-17 12:28:19,904] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 26] INFO [2019-12-17 12:28:19,907] jepsen worker 3 - jepsen.util 723 :invoke :write [1 21] INFO [2019-12-17 12:28:19,907] jepsen worker 78 - jepsen.util 978 :fail :cas [26 [33 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,908] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [11 46]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,911] jepsen worker 65 - jepsen.util 695 :ok :read [21 39] INFO [2019-12-17 12:28:19,912] jepsen worker 45 - jepsen.util 1035 :fail :cas [15 [3 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,912] jepsen worker 78 - jepsen.util 978 :invoke :read [26 nil] INFO [2019-12-17 12:28:19,913] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [10 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,913] jepsen worker 62 - jepsen.util 962 :fail :cas [20 [13 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,914] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:19,914] jepsen worker 53 - jepsen.util 953 :ok :read [17 23] INFO [2019-12-17 12:28:19,915] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:19,915] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [7 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,915] jepsen worker 13 - jepsen.util 1003 :ok :read [4 46] INFO [2019-12-17 12:28:19,916] jepsen worker 61 - jepsen.util 1051 :fail :cas [20 [29 37]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,916] jepsen worker 45 - jepsen.util 1035 :invoke :read [15 nil] INFO [2019-12-17 12:28:19,916] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [45 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,916] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [40 9]] INFO [2019-12-17 12:28:19,916] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [35 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,917] jepsen worker 62 - jepsen.util 962 :invoke :write [20 11] INFO [2019-12-17 12:28:19,917] jepsen worker 53 - jepsen.util 953 :invoke :read [17 nil] INFO [2019-12-17 12:28:19,917] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [20 10]] INFO [2019-12-17 12:28:19,917] jepsen worker 13 - jepsen.util 1003 :invoke :read [4 nil] INFO [2019-12-17 12:28:19,918] jepsen worker 61 - jepsen.util 1051 :invoke :read [20 nil] INFO [2019-12-17 12:28:19,919] jepsen worker 38 - jepsen.util 848 :ok :read [12 12] INFO [2019-12-17 12:28:19,919] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:19,920] jepsen worker 85 - jepsen.util 895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,920] jepsen worker 50 - jepsen.util 1040 :info :cas [16 [28 33]] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 39 - jepsen.util 1029 :info :write [13 9] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:19,920] jepsen worker 22 - jepsen.util 832 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 40 - jepsen.util 1300 :info :write [13 2] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 63 - jepsen.util 693 :info :cas [21 [33 21]] :RequestCanceledException INFO [2019-12-17 12:28:19,920] jepsen worker 21 - jepsen.util 1101 :fail :read [7 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,920] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:19,921] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:19,922] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:19,922] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [17 11]] INFO [2019-12-17 12:28:19,922] jepsen worker 43 - jepsen.util 1033 :info :write [14 45] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 85 - jepsen.util 895 :invoke :read [28 nil] INFO [2019-12-17 12:28:19,922] jepsen worker 49 - jepsen.util 859 :info :cas [16 [47 31]] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 81 - jepsen.util 1071 :info :cas [27 [31 35]] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 57 - jepsen.util 957 :info :cas [19 [9 28]] :RequestCanceledException INFO [2019-12-17 12:28:19,922] jepsen worker 23 - jepsen.util 833 :fail :cas [7 [11 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:19,922] jepsen worker 41 - jepsen.util 671 :info :cas [13 [6 8]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 12 - jepsen.util 912 :info :cas [4 [42 20]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 66 - jepsen.util 786 :info :cas [22 [40 36]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 11 - jepsen.util 1091 :info :write [3 19] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 70 - jepsen.util 1150 :fail :read [23 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 67 - jepsen.util 1237 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 2 - jepsen.util 632 :info :cas [0 [10 23]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 48 - jepsen.util 498 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 79 - jepsen.util 799 :fail :read [26 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 30 - jepsen.util 930 :info :write [10 10] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 9 - jepsen.util 729 :info :cas [3 [14 33]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 28 - jepsen.util 568 :info :write [9 6] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 52 - jepsen.util 862 :ok :read [17 9] INFO [2019-12-17 12:28:19,923] jepsen worker 15 - jepsen.util 825 :info :cas [5 [38 30]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 19 - jepsen.util 1009 :info :write [6 39] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 84 - jepsen.util 714 :info :cas [28 [4 40]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 33 - jepsen.util 1113 :info :cas [11 [37 27]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 46 - jepsen.util 1036 :info :write [15 41] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 42 - jepsen.util 1122 :info :write [14 27] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 77 - jepsen.util 887 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 59 - jepsen.util 869 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 82 - jepsen.util 1252 :info :cas [27 [8 37]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 83 - jepsen.util 1073 :info :write [27 18] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 1 - jepsen.util 721 :fail :read [0 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,923] jepsen worker 8 - jepsen.util 998 :info :cas [2 [25 49]] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 24 - jepsen.util 834 :info :write [8 28] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 10 - jepsen.util 1000 :info :write [3 45] :RequestCanceledException INFO [2019-12-17 12:28:19,923] jepsen worker 87 - jepsen.util 897 :info :write [29 23] :RequestCanceledException INFO [2019-12-17 12:28:19,924] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:19,924] jepsen worker 50 - jepsen.util 1130 :invoke :write [16 21] INFO [2019-12-17 12:28:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4513,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,925] jepsen worker 34 - jepsen.util 1024 :info :cas [11 [30 27]] :RequestCanceledException INFO [2019-12-17 12:28:19,925] jepsen worker 75 - jepsen.util 975 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,925] jepsen worker 76 - jepsen.util 976 :fail :read [25 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,925] jepsen worker 86 - jepsen.util 896 :info :cas [28 [29 40]] :RequestCanceledException INFO [2019-12-17 12:28:19,925] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4465,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:19,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4456,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:19,926] jepsen worker 7 - jepsen.util 1087 :info :write [2 36] :RequestCanceledException INFO [2019-12-17 12:28:19,926] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4464,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:19,926] jepsen worker 16 - jepsen.util 916 :info :cas [5 [31 21]] :RequestCanceledException INFO [2019-12-17 12:28:19,927] jepsen worker 20 - jepsen.util 830 :info :write [6 47] :RequestCanceledException INFO [2019-12-17 12:28:19,927] jepsen worker 63 - jepsen.util 783 :invoke :cas [21 [38 49]] INFO [2019-12-17 12:28:19,927] jepsen worker 68 - jepsen.util 608 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:19,928] jepsen worker 39 - jepsen.util 1119 :invoke :read [13 nil] INFO [2019-12-17 12:28:19,928] jepsen worker 40 - jepsen.util 1390 :invoke :write [13 32] INFO [2019-12-17 12:28:19,928] jepsen worker 22 - jepsen.util 922 :invoke :write [7 24] INFO [2019-12-17 12:28:19,928] jepsen worker 81 - jepsen.util 1161 :invoke :write [27 22] INFO [2019-12-17 12:28:19,929] jepsen worker 23 - jepsen.util 833 :invoke :write [7 2] INFO [2019-12-17 12:28:19,929] jepsen worker 49 - jepsen.util 949 :invoke :read [16 nil] INFO [2019-12-17 12:28:19,929] jepsen worker 57 - jepsen.util 1047 :invoke :read [19 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 43 - jepsen.util 1123 :invoke :read [14 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [3 13]] INFO [2019-12-17 12:28:19,930] jepsen worker 41 - jepsen.util 761 :invoke :write [13 13] INFO [2019-12-17 12:28:19,930] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 66 - jepsen.util 876 :invoke :read [22 nil] INFO [2019-12-17 12:28:19,930] jepsen worker 67 - jepsen.util 1237 :invoke :write [22 35] INFO [2019-12-17 12:28:19,931] jepsen worker 11 - jepsen.util 1181 :invoke :read [3 nil] INFO [2019-12-17 12:28:19,931] jepsen worker 48 - jepsen.util 498 :invoke :cas [16 [39 40]] INFO [2019-12-17 12:28:19,931] jepsen worker 79 - jepsen.util 799 :invoke :write [26 20] INFO [2019-12-17 12:28:19,932] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [2 49]] INFO [2019-12-17 12:28:19,932] jepsen worker 2 - jepsen.util 722 :invoke :cas [0 [43 28]] INFO [2019-12-17 12:28:19,932] jepsen worker 30 - jepsen.util 1020 :invoke :cas [10 [40 20]] INFO [2019-12-17 12:28:19,933] jepsen worker 28 - jepsen.util 658 :invoke :cas [9 [46 12]] INFO [2019-12-17 12:28:19,933] jepsen worker 9 - jepsen.util 819 :invoke :write [3 35] INFO [2019-12-17 12:28:19,933] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:19,933] jepsen worker 59 - jepsen.util 869 :invoke :read [19 nil] INFO [2019-12-17 12:28:19,933] jepsen worker 19 - jepsen.util 1099 :invoke :write [6 14] INFO [2019-12-17 12:28:19,933] jepsen worker 84 - jepsen.util 804 :invoke :write [28 17] INFO [2019-12-17 12:28:19,933] jepsen worker 33 - jepsen.util 1203 :invoke :write [11 39] INFO [2019-12-17 12:28:19,934] jepsen worker 46 - jepsen.util 1126 :invoke :write [15 6] INFO [2019-12-17 12:28:19,934] jepsen worker 1 - jepsen.util 721 :invoke :cas [0 [39 29]] INFO [2019-12-17 12:28:19,934] jepsen worker 42 - jepsen.util 1212 :invoke :write [14 7] INFO [2019-12-17 12:28:19,934] jepsen worker 77 - jepsen.util 977 :invoke :write [25 10] INFO [2019-12-17 12:28:19,935] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,935] jepsen worker 8 - jepsen.util 1088 :invoke :cas [2 [10 32]] INFO [2019-12-17 12:28:19,935] jepsen worker 83 - jepsen.util 1163 :invoke :write [27 9] INFO [2019-12-17 12:28:19,935] jepsen worker 24 - jepsen.util 924 :invoke :cas [8 [49 17]] INFO [2019-12-17 12:28:19,935] jepsen worker 82 - jepsen.util 1342 :invoke :write [27 35] INFO [2019-12-17 12:28:19,935] jepsen worker 75 - jepsen.util 975 :invoke :write [25 43] INFO [2019-12-17 12:28:19,935] jepsen worker 76 - jepsen.util 976 :invoke :cas [25 [17 5]] INFO [2019-12-17 12:28:19,936] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [45 10]] INFO [2019-12-17 12:28:19,936] jepsen worker 10 - jepsen.util 1090 :invoke :read [3 nil] INFO [2019-12-17 12:28:19,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:19,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:19,936] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:19,937] jepsen worker 68 - jepsen.util 608 :invoke :cas [22 [29 37]] INFO [2019-12-17 12:28:19,937] jepsen worker 34 - jepsen.util 1114 :invoke :read [11 nil] INFO [2019-12-17 12:28:19,939] jepsen worker 7 - jepsen.util 1177 :invoke :read [2 nil] INFO [2019-12-17 12:28:19,940] jepsen worker 16 - jepsen.util 1006 :invoke :read [5 nil] INFO [2019-12-17 12:28:19,941] jepsen worker 86 - jepsen.util 986 :invoke :write [28 9] INFO [2019-12-17 12:28:19,941] jepsen worker 20 - jepsen.util 920 :invoke :write [6 45] INFO [2019-12-17 12:28:19,942] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4514,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,944] jepsen worker 54 - jepsen.util 954 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:28:19,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4470,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:19,945] jepsen worker 89 - jepsen.util 989 :info :write [29 37] :RequestCanceledException INFO [2019-12-17 12:28:19,945] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4467,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:19,946] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4471,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:19,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,947] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:19,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:19,948] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:19,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4515,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,949] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4481,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:19,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4483,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:19,950] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4477,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:19,950] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4519,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,952] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:19,952] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4521,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:19,962] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:19,962] jepsen nemesis - couchbase.util Rebalance check count: 28 INFO [2019-12-17 12:28:19,962] jepsen nemesis - couchbase.util Rebalance stuck count: 9 INFO [2019-12-17 12:28:19,964] jepsen worker 54 - jepsen.util 1044 :invoke :read [18 nil] INFO [2019-12-17 12:28:19,965] jepsen worker 89 - jepsen.util 1079 :invoke :write [29 46] INFO [2019-12-17 12:28:19,980] jepsen worker 56 - jepsen.util 866 :info :write [18 39] :RequestCanceledException INFO [2019-12-17 12:28:19,996] jepsen worker 56 - jepsen.util 956 :invoke :write [18 49] INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4533,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,028] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4538,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4535,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4539,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4540,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4541,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4545,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,029] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":4290,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4549,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":4397,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4590,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:20,030] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4592,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:20,031] jepsen worker 18 - jepsen.util 918 :fail :cas [6 [37 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,031] jepsen worker 18 - jepsen.util 918 :invoke :write [6 43] INFO [2019-12-17 12:28:20,035] jepsen worker 44 - jepsen.util 764 :info :cas [14 [29 29]] :RequestCanceledException INFO [2019-12-17 12:28:20,039] jepsen worker 44 - jepsen.util 854 :invoke :cas [14 [32 49]] INFO [2019-12-17 12:28:20,050] jepsen worker 47 - jepsen.util 1127 :info :cas [15 [35 33]] :RequestCanceledException INFO [2019-12-17 12:28:20,056] jepsen worker 47 - jepsen.util 1217 :invoke :cas [15 [46 30]] INFO [2019-12-17 12:28:20,058] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,059] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:20,067] jepsen worker 51 - jepsen.util 861 :ok :read [17 9] INFO [2019-12-17 12:28:20,067] jepsen worker 51 - jepsen.util 861 :invoke :read [17 nil] INFO [2019-12-17 12:28:20,075] jepsen worker 5 - jepsen.util 635 :ok :read [1 43] INFO [2019-12-17 12:28:20,075] jepsen worker 5 - jepsen.util 635 :invoke :cas [1 [12 15]] INFO [2019-12-17 12:28:20,080] jepsen worker 65 - jepsen.util 695 :ok :read [21 39] INFO [2019-12-17 12:28:20,080] jepsen worker 65 - jepsen.util 695 :invoke :read [21 nil] INFO [2019-12-17 12:28:20,095] jepsen worker 30 - jepsen.util 1020 :fail :cas [10 [40 20]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,096] jepsen worker 30 - jepsen.util 1020 :invoke :write [10 11] INFO [2019-12-17 12:28:20,112] jepsen worker 73 - jepsen.util 793 :ok :read [24 0] INFO [2019-12-17 12:28:20,113] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [26 49]] INFO [2019-12-17 12:28:20,144] jepsen worker 36 - jepsen.util 1026 :fail :cas [12 [44 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,144] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:20,147] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [21 44]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,147] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [41 28]] INFO [2019-12-17 12:28:20,151] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [40 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,151] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,162] jepsen worker 27 - jepsen.util 747 :ok :read [9 28] INFO [2019-12-17 12:28:20,162] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [25 16]] INFO [2019-12-17 12:28:20,180] jepsen worker 10 - jepsen.util 1090 :ok :read [3 23] INFO [2019-12-17 12:28:20,181] jepsen worker 10 - jepsen.util 1090 :invoke :read [3 nil] INFO [2019-12-17 12:28:20,188] jepsen worker 35 - jepsen.util 755 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:20,188] jepsen worker 35 - jepsen.util 755 :invoke :read [11 nil] INFO [2019-12-17 12:28:20,190] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,190] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [10 38]] INFO [2019-12-17 12:28:20,205] jepsen worker 58 - jepsen.util 958 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:20,205] jepsen worker 58 - jepsen.util 958 :invoke :write [19 33] INFO [2019-12-17 12:28:20,217] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [26 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,217] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,232] jepsen worker 72 - jepsen.util 792 :ok :read [24 0] INFO [2019-12-17 12:28:20,232] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [38 14]] INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4636,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4639,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:20,249] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4640,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:20,251] jepsen worker 78 - jepsen.util 978 :ok :read [26 29] INFO [2019-12-17 12:28:20,251] jepsen worker 78 - jepsen.util 978 :invoke :write [26 35] INFO [2019-12-17 12:28:20,252] jepsen worker 29 - jepsen.util 749 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:20,253] jepsen worker 29 - jepsen.util 749 :invoke :write [9 22] INFO [2019-12-17 12:28:20,262] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [25 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,262] jepsen worker 27 - jepsen.util 747 :invoke :write [9 34] INFO [2019-12-17 12:28:20,273] jepsen worker 4 - jepsen.util 724 :ok :read [1 43] INFO [2019-12-17 12:28:20,273] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4628,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4629,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:20,274] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4630,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:20,287] jepsen worker 10 - jepsen.util 1090 :ok :read [3 23] INFO [2019-12-17 12:28:20,287] jepsen worker 10 - jepsen.util 1090 :invoke :cas [3 [5 33]] INFO [2019-12-17 12:28:20,305] jepsen worker 32 - jepsen.util 842 :fail :cas [10 [10 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,306] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:20,320] jepsen worker 73 - jepsen.util 793 :ok :read [24 0] INFO [2019-12-17 12:28:20,320] jepsen worker 73 - jepsen.util 793 :invoke :cas [24 [30 13]] INFO [2019-12-17 12:28:20,334] jepsen worker 72 - jepsen.util 792 :fail :cas [24 [38 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,335] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,368] jepsen worker 27 - jepsen.util 747 :ok :write [9 34] INFO [2019-12-17 12:28:20,368] jepsen worker 27 - jepsen.util 747 :invoke :cas [9 [38 9]] INFO [2019-12-17 12:28:20,376] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [17 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,376] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [46 23]] INFO [2019-12-17 12:28:20,394] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,394] jepsen worker 32 - jepsen.util 842 :invoke :read [10 nil] INFO [2019-12-17 12:28:20,404] jepsen worker 73 - jepsen.util 793 :fail :cas [24 [30 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,404] jepsen worker 73 - jepsen.util 793 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,415] jepsen worker 80 - jepsen.util 1070 :ok :write [26 49] INFO [2019-12-17 12:28:20,416] jepsen worker 80 - jepsen.util 1070 :invoke :write [26 47] INFO [2019-12-17 12:28:20,419] jepsen worker 51 - jepsen.util 861 :ok :read [17 9] INFO [2019-12-17 12:28:20,419] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [6 33]] INFO [2019-12-17 12:28:20,420] jepsen worker 72 - jepsen.util 792 :ok :read [24 0] INFO [2019-12-17 12:28:20,420] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,435] jepsen worker 61 - jepsen.util 1051 :ok :read [20 7] INFO [2019-12-17 12:28:20,435] jepsen worker 61 - jepsen.util 1051 :invoke :read [20 nil] INFO [2019-12-17 12:28:20,441] jepsen worker 89 - jepsen.util 1079 :ok :write [29 46] INFO [2019-12-17 12:28:20,441] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [21 23]] INFO [2019-12-17 12:28:20,452] jepsen worker 27 - jepsen.util 747 :fail :cas [9 [38 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,452] jepsen worker 27 - jepsen.util 747 :invoke :write [9 0] INFO [2019-12-17 12:28:20,468] jepsen worker 32 - jepsen.util 842 :ok :read [10 27] INFO [2019-12-17 12:28:20,468] jepsen worker 32 - jepsen.util 842 :invoke :cas [10 [15 43]] INFO [2019-12-17 12:28:20,469] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:20,469] jepsen worker 21 - jepsen.util 1101 :invoke :cas [7 [47 19]] INFO [2019-12-17 12:28:20,480] jepsen worker 14 - jepsen.util 824 :ok :read [4 11] INFO [2019-12-17 12:28:20,480] jepsen worker 14 - jepsen.util 824 :invoke :cas [4 [4 2]] INFO [2019-12-17 12:28:20,489] jepsen worker 61 - jepsen.util 1051 :ok :read [20 7] INFO [2019-12-17 12:28:20,489] jepsen worker 61 - jepsen.util 1051 :invoke :cas [20 [37 9]] INFO [2019-12-17 12:28:20,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestTimeoutException: CarrierBucketConfigRequest {"retried":0,"reason":"TIMEOUT","requestId":4439,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:20,493] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:20,493] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4660,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4663,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:20,494] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4664,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:20,507] jepsen worker 43 - jepsen.util 1123 :ok :read [14 27] INFO [2019-12-17 12:28:20,507] jepsen worker 43 - jepsen.util 1123 :invoke :cas [14 [20 23]] INFO [2019-12-17 12:28:20,512] jepsen worker 73 - jepsen.util 793 :ok :read [24 0] INFO [2019-12-17 12:28:20,512] jepsen worker 73 - jepsen.util 793 :invoke :write [24 7] INFO [2019-12-17 12:28:20,529] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [45 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,529] jepsen worker 87 - jepsen.util 987 :invoke :write [29 14] INFO [2019-12-17 12:28:20,532] jepsen worker 24 - jepsen.util 924 :fail :cas [8 [49 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,532] jepsen worker 24 - jepsen.util 924 :invoke :write [8 37] INFO [2019-12-17 12:28:20,537] jepsen worker 72 - jepsen.util 792 :ok :read [24 0] INFO [2019-12-17 12:28:20,537] jepsen worker 72 - jepsen.util 792 :invoke :read [24 nil] INFO [2019-12-17 12:28:20,559] jepsen worker 27 - jepsen.util 747 :ok :write [9 0] INFO [2019-12-17 12:28:20,559] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:20,587] jepsen worker 44 - jepsen.util 854 :fail :cas [14 [32 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,587] jepsen worker 44 - jepsen.util 854 :invoke :write [14 12] INFO [2019-12-17 12:28:20,595] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [20 10]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,595] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:20,597] jepsen worker 61 - jepsen.util 1051 :fail :cas [20 [37 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,597] jepsen worker 61 - jepsen.util 1051 :invoke :cas [20 [49 45]] INFO [2019-12-17 12:28:20,628] jepsen worker 88 - jepsen.util 898 :fail :cas [29 [41 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,628] jepsen worker 88 - jepsen.util 898 :invoke :write [29 14] INFO [2019-12-17 12:28:20,650] jepsen worker 13 - jepsen.util 1003 :ok :read [4 11] INFO [2019-12-17 12:28:20,650] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [10 49]] INFO [2019-12-17 12:28:20,651] jepsen worker 79 - jepsen.util 799 :ok :write [26 20] INFO [2019-12-17 12:28:20,651] jepsen worker 79 - jepsen.util 799 :invoke :cas [26 [19 3]] INFO [2019-12-17 12:28:20,679] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [45 34]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,679] jepsen worker 55 - jepsen.util 1135 :invoke :read [18 nil] INFO [2019-12-17 12:28:20,713] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:20,713] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4683,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:20,724] jepsen worker 36 - jepsen.util 1026 :ok :read [12 12] INFO [2019-12-17 12:28:20,724] jepsen worker 36 - jepsen.util 1026 :invoke :read [12 nil] INFO [2019-12-17 12:28:20,742] jepsen worker 74 - jepsen.util 794 :ok :write [24 11] INFO [2019-12-17 12:28:20,743] jepsen worker 74 - jepsen.util 794 :invoke :write [24 31] INFO [2019-12-17 12:28:20,749] jepsen worker 69 - jepsen.util 879 :ok :write [23 4] INFO [2019-12-17 12:28:20,749] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:20,773] jepsen worker 31 - jepsen.util 1021 :ok :write [10 26] INFO [2019-12-17 12:28:20,773] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 12] INFO [2019-12-17 12:28:20,823] jepsen worker 43 - jepsen.util 1123 :fail :cas [14 [20 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,824] jepsen worker 43 - jepsen.util 1123 :invoke :read [14 nil] INFO [2019-12-17 12:28:20,831] jepsen worker 78 - jepsen.util 978 :ok :write [26 35] INFO [2019-12-17 12:28:20,831] jepsen worker 78 - jepsen.util 978 :invoke :cas [26 [39 21]] INFO [2019-12-17 12:28:20,833] jepsen worker 61 - jepsen.util 1051 :fail :cas [20 [49 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,833] jepsen worker 61 - jepsen.util 1051 :invoke :write [20 18] INFO [2019-12-17 12:28:20,838] jepsen worker 53 - jepsen.util 953 :ok :read [17 9] INFO [2019-12-17 12:28:20,838] jepsen worker 53 - jepsen.util 953 :invoke :write [17 15] INFO [2019-12-17 12:28:20,860] jepsen worker 64 - jepsen.util 514 :ok :write [21 21] INFO [2019-12-17 12:28:20,860] jepsen worker 64 - jepsen.util 514 :invoke :cas [21 [30 14]] INFO [2019-12-17 12:28:20,869] jepsen worker 47 - jepsen.util 1217 :fail :cas [15 [46 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,869] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 31] INFO [2019-12-17 12:28:20,883] jepsen worker 26 - jepsen.util 1106 :ok :read [8 14] INFO [2019-12-17 12:28:20,883] jepsen worker 26 - jepsen.util 1106 :invoke :read [8 nil] INFO [2019-12-17 12:28:20,898] jepsen worker 54 - jepsen.util 1044 :ok :read [18 39] INFO [2019-12-17 12:28:20,898] jepsen worker 54 - jepsen.util 1044 :invoke :cas [18 [7 17]] INFO [2019-12-17 12:28:20,923] jepsen worker 29 - jepsen.util 749 :ok :write [9 22] INFO [2019-12-17 12:28:20,923] jepsen worker 29 - jepsen.util 749 :invoke :read [9 nil] INFO [2019-12-17 12:28:20,940] jepsen worker 14 - jepsen.util 824 :fail :cas [4 [4 2]] :ValueNotSwapFrom INFO [2019-12-17 12:28:20,941] jepsen worker 14 - jepsen.util 824 :invoke :cas [4 [18 42]] INFO [2019-12-17 12:28:20,958] jepsen worker 4 - jepsen.util 724 :ok :read [1 43] INFO [2019-12-17 12:28:20,958] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [11 45]] INFO [2019-12-17 12:28:20,975] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:20,975] jepsen nemesis - couchbase.util Rebalance check count: 29 INFO [2019-12-17 12:28:20,975] jepsen nemesis - couchbase.util Rebalance stuck count: 10 INFO [2019-12-17 12:28:20,993] jepsen worker 7 - jepsen.util 1177 :ok :read [2 15] INFO [2019-12-17 12:28:20,993] jepsen worker 7 - jepsen.util 1177 :invoke :write [2 35] INFO [2019-12-17 12:28:21,144] jepsen worker 87 - jepsen.util 987 :ok :write [29 14] INFO [2019-12-17 12:28:21,145] jepsen worker 87 - jepsen.util 987 :invoke :cas [29 [21 9]] INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4713,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:21,145] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4717,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:21,146] jepsen worker 35 - jepsen.util 755 :ok :read [11 7] INFO [2019-12-17 12:28:21,146] jepsen worker 35 - jepsen.util 755 :invoke :write [11 22] INFO [2019-12-17 12:28:21,151] jepsen worker 69 - jepsen.util 879 :ok :read [23 4] INFO [2019-12-17 12:28:21,152] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,154] jepsen worker 5 - jepsen.util 635 :fail :cas [1 [12 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,154] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:21,157] jepsen worker 26 - jepsen.util 1106 :ok :read [8 14] INFO [2019-12-17 12:28:21,158] jepsen worker 26 - jepsen.util 1106 :invoke :cas [8 [23 29]] INFO [2019-12-17 12:28:21,158] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [6 33]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,158] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [21 15]] INFO [2019-12-17 12:28:21,159] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [46 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,159] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [28 22]] INFO [2019-12-17 12:28:21,188] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [21 23]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,188] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [20 13]] INFO [2019-12-17 12:28:21,195] jepsen worker 56 - jepsen.util 956 :ok :write [18 49] INFO [2019-12-17 12:28:21,195] jepsen worker 56 - jepsen.util 956 :invoke :cas [18 [11 30]] INFO [2019-12-17 12:28:21,209] jepsen worker 21 - jepsen.util 1101 :fail :cas [7 [47 19]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,209] jepsen worker 21 - jepsen.util 1101 :invoke :cas [7 [45 28]] INFO [2019-12-17 12:28:21,214] jepsen worker 74 - jepsen.util 794 :ok :write [24 31] INFO [2019-12-17 12:28:21,214] jepsen worker 74 - jepsen.util 794 :invoke :read [24 nil] INFO [2019-12-17 12:28:21,215] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [10 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,216] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 0] INFO [2019-12-17 12:28:21,235] jepsen worker 55 - jepsen.util 1135 :ok :read [18 49] INFO [2019-12-17 12:28:21,235] jepsen worker 55 - jepsen.util 1135 :invoke :cas [18 [34 11]] INFO [2019-12-17 12:28:21,235] jepsen worker 74 - jepsen.util 794 :ok :read [24 31] INFO [2019-12-17 12:28:21,235] jepsen worker 74 - jepsen.util 794 :invoke :write [24 3] INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4728,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:21,252] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4729,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:21,255] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:21,255] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4736,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:21,274] jepsen worker 37 - jepsen.util 847 :ok :read [12 12] INFO [2019-12-17 12:28:21,274] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [9 27]] INFO [2019-12-17 12:28:21,324] jepsen worker 56 - jepsen.util 956 :fail :cas [18 [11 30]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,324] jepsen worker 56 - jepsen.util 956 :invoke :cas [18 [12 21]] INFO [2019-12-17 12:28:21,326] jepsen worker 78 - jepsen.util 978 :fail :cas [26 [39 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,326] jepsen worker 78 - jepsen.util 978 :invoke :read [26 nil] INFO [2019-12-17 12:28:21,359] jepsen worker 5 - jepsen.util 635 :ok :read [1 43] INFO [2019-12-17 12:28:21,359] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:21,372] jepsen worker 31 - jepsen.util 1021 :ok :write [10 12] INFO [2019-12-17 12:28:21,373] jepsen worker 31 - jepsen.util 1021 :invoke :cas [10 [12 4]] INFO [2019-12-17 12:28:21,374] jepsen worker 43 - jepsen.util 1123 :ok :read [14 27] INFO [2019-12-17 12:28:21,374] jepsen worker 43 - jepsen.util 1123 :invoke :read [14 nil] INFO [2019-12-17 12:28:21,387] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [28 22]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,388] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:21,430] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [20 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,430] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [7 5]] INFO [2019-12-17 12:28:21,436] jepsen worker 71 - jepsen.util 701 :ok :write [23 5] INFO [2019-12-17 12:28:21,437] jepsen worker 71 - jepsen.util 701 :invoke :write [23 27] INFO [2019-12-17 12:28:21,445] jepsen worker 26 - jepsen.util 1106 :fail :cas [8 [23 29]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,445] jepsen worker 26 - jepsen.util 1106 :invoke :cas [8 [3 6]] INFO [2019-12-17 12:28:21,453] jepsen worker 47 - jepsen.util 1217 :ok :write [15 31] INFO [2019-12-17 12:28:21,453] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 33] INFO [2019-12-17 12:28:21,471] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:21,471] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4755,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:21,471] jepsen worker 88 - jepsen.util 898 :ok :write [29 14] INFO [2019-12-17 12:28:21,471] jepsen worker 88 - jepsen.util 898 :invoke :write [29 18] INFO [2019-12-17 12:28:21,493] jepsen worker 74 - jepsen.util 794 :ok :write [24 3] INFO [2019-12-17 12:28:21,493] jepsen worker 74 - jepsen.util 794 :invoke :cas [24 [36 25]] INFO [2019-12-17 12:28:21,496] jepsen worker 79 - jepsen.util 799 :fail :cas [26 [19 3]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,496] jepsen worker 79 - jepsen.util 799 :invoke :write [26 0] INFO [2019-12-17 12:28:21,509] jepsen worker 74 - jepsen.util 794 :fail :cas [24 [36 25]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,509] jepsen worker 74 - jepsen.util 794 :invoke :cas [24 [39 41]] INFO [2019-12-17 12:28:21,510] jepsen worker 78 - jepsen.util 978 :ok :read [26 35] INFO [2019-12-17 12:28:21,510] jepsen worker 78 - jepsen.util 978 :invoke :cas [26 [43 16]] INFO [2019-12-17 12:28:21,511] jepsen worker 13 - jepsen.util 1003 :ok :write [4 0] INFO [2019-12-17 12:28:21,511] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 41] INFO [2019-12-17 12:28:21,529] jepsen worker 3 - jepsen.util 723 :ok :write [1 21] INFO [2019-12-17 12:28:21,530] jepsen worker 3 - jepsen.util 723 :invoke :cas [1 [22 13]] INFO [2019-12-17 12:28:21,533] jepsen worker 74 - jepsen.util 794 :fail :cas [24 [39 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,536] jepsen worker 74 - jepsen.util 794 :invoke :write [24 10] INFO [2019-12-17 12:28:21,541] jepsen worker 55 - jepsen.util 1135 :fail :cas [18 [34 11]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,541] jepsen worker 55 - jepsen.util 1135 :invoke :write [18 14] INFO [2019-12-17 12:28:21,565] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [11 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,566] jepsen worker 4 - jepsen.util 724 :invoke :cas [1 [13 4]] INFO [2019-12-17 12:28:21,598] jepsen worker 5 - jepsen.util 635 :ok :read [1 43] INFO [2019-12-17 12:28:21,599] jepsen worker 5 - jepsen.util 635 :invoke :write [1 47] INFO [2019-12-17 12:28:21,628] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [9 27]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,628] jepsen worker 37 - jepsen.util 847 :invoke :write [12 43] INFO [2019-12-17 12:28:21,643] jepsen worker 43 - jepsen.util 1123 :ok :read [14 27] INFO [2019-12-17 12:28:21,644] jepsen worker 43 - jepsen.util 1123 :invoke :write [14 9] INFO [2019-12-17 12:28:21,651] jepsen worker 51 - jepsen.util 861 :fail :cas [17 [21 15]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,651] jepsen worker 51 - jepsen.util 861 :invoke :write [17 35] INFO [2019-12-17 12:28:21,661] jepsen worker 38 - jepsen.util 848 :ok :read [12 12] INFO [2019-12-17 12:28:21,661] jepsen worker 38 - jepsen.util 848 :invoke :cas [12 [26 17]] INFO [2019-12-17 12:28:21,679] jepsen worker 31 - jepsen.util 1021 :ok :cas [10 [12 4]] INFO [2019-12-17 12:28:21,679] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 26] INFO [2019-12-17 12:28:21,681] jepsen worker 56 - jepsen.util 956 :fail :cas [18 [12 21]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,681] jepsen worker 56 - jepsen.util 956 :invoke :read [18 nil] INFO [2019-12-17 12:28:21,683] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [3 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,683] jepsen worker 12 - jepsen.util 1002 :invoke :cas [4 [43 4]] INFO [2019-12-17 12:28:21,690] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} INFO [2019-12-17 12:28:21,691] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4776,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":5,"completed":true} {"coreId":5} INFO [2019-12-17 12:28:21,692] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [7 5]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,692] jepsen worker 89 - jepsen.util 1079 :invoke :read [29 nil] INFO [2019-12-17 12:28:21,715] jepsen worker 26 - jepsen.util 1106 :fail :cas [8 [3 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,716] jepsen worker 26 - jepsen.util 1106 :invoke :read [8 nil] INFO [2019-12-17 12:28:21,717] jepsen worker 21 - jepsen.util 1101 :fail :cas [7 [45 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,717] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:21,730] jepsen worker 78 - jepsen.util 978 :fail :cas [26 [43 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,730] jepsen worker 78 - jepsen.util 978 :invoke :write [26 17] INFO [2019-12-17 12:28:21,741] jepsen worker 70 - jepsen.util 1150 :ok :read [23 5] INFO [2019-12-17 12:28:21,741] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,766] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [2 49]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,766] jepsen worker 52 - jepsen.util 862 :invoke :cas [17 [22 32]] INFO [2019-12-17 12:28:21,768] jepsen worker 26 - jepsen.util 1106 :ok :read [8 14] INFO [2019-12-17 12:28:21,768] jepsen worker 26 - jepsen.util 1106 :invoke :write [8 45] INFO [2019-12-17 12:28:21,771] jepsen worker 47 - jepsen.util 1217 :ok :write [15 33] INFO [2019-12-17 12:28:21,772] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 17] INFO [2019-12-17 12:28:21,781] jepsen worker 13 - jepsen.util 1003 :ok :write [4 41] INFO [2019-12-17 12:28:21,782] jepsen worker 13 - jepsen.util 1003 :invoke :cas [4 [13 28]] INFO [2019-12-17 12:28:21,830] jepsen worker 56 - jepsen.util 956 :ok :read [18 49] INFO [2019-12-17 12:28:21,830] jepsen worker 56 - jepsen.util 956 :invoke :write [18 2] INFO [2019-12-17 12:28:21,834] jepsen worker 31 - jepsen.util 1021 :ok :write [10 26] INFO [2019-12-17 12:28:21,834] jepsen worker 31 - jepsen.util 1021 :invoke :write [10 31] INFO [2019-12-17 12:28:21,834] jepsen worker 74 - jepsen.util 794 :ok :write [24 10] INFO [2019-12-17 12:28:21,835] jepsen worker 74 - jepsen.util 794 :invoke :write [24 27] INFO [2019-12-17 12:28:21,849] jepsen worker 53 - jepsen.util 953 :ok :write [17 15] INFO [2019-12-17 12:28:21,849] jepsen worker 53 - jepsen.util 953 :invoke :cas [17 [6 14]] INFO [2019-12-17 12:28:21,854] jepsen worker 71 - jepsen.util 701 :ok :write [23 27] INFO [2019-12-17 12:28:21,854] jepsen worker 71 - jepsen.util 701 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,856] jepsen worker 88 - jepsen.util 898 :ok :write [29 18] INFO [2019-12-17 12:28:21,856] jepsen worker 88 - jepsen.util 898 :invoke :cas [29 [3 4]] INFO [2019-12-17 12:28:21,856] jepsen worker 80 - jepsen.util 1070 :ok :write [26 47] INFO [2019-12-17 12:28:21,856] jepsen worker 80 - jepsen.util 1070 :invoke :write [26 25] INFO [2019-12-17 12:28:21,865] jepsen worker 47 - jepsen.util 1217 :ok :write [15 17] INFO [2019-12-17 12:28:21,865] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 13] INFO [2019-12-17 12:28:21,878] jepsen worker 36 - jepsen.util 1026 :ok :read [12 12] INFO [2019-12-17 12:28:21,878] jepsen worker 36 - jepsen.util 1026 :invoke :cas [12 [35 12]] INFO [2019-12-17 12:28:21,893] jepsen worker 69 - jepsen.util 879 :ok :read [23 27] INFO [2019-12-17 12:28:21,893] jepsen worker 69 - jepsen.util 879 :invoke :read [23 nil] INFO [2019-12-17 12:28:21,897] jepsen worker 52 - jepsen.util 862 :fail :cas [17 [22 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,898] jepsen worker 52 - jepsen.util 862 :invoke :write [17 17] INFO [2019-12-17 12:28:21,902] jepsen worker 5 - jepsen.util 635 :ok :write [1 47] INFO [2019-12-17 12:28:21,902] jepsen worker 5 - jepsen.util 635 :invoke :read [1 nil] INFO [2019-12-17 12:28:21,904] jepsen worker 3 - jepsen.util 723 :fail :cas [1 [22 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,904] jepsen worker 3 - jepsen.util 723 :invoke :cas [1 [12 13]] INFO [2019-12-17 12:28:21,909] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} INFO [2019-12-17 12:28:21,909] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4804,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":6,"completed":true} {"coreId":6} INFO [2019-12-17 12:28:21,935] jepsen worker 25 - jepsen.util 1105 :info :write [8 3] :RequestCanceledException INFO [2019-12-17 12:28:21,936] jepsen worker 25 - jepsen.util 1195 :invoke :write [8 6] INFO [2019-12-17 12:28:21,937] jepsen worker 14 - jepsen.util 824 :fail :cas [4 [18 42]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,937] jepsen worker 14 - jepsen.util 824 :invoke :read [4 nil] INFO [2019-12-17 12:28:21,952] jepsen worker 28 - jepsen.util 658 :info :cas [9 [46 12]] :RequestCanceledException INFO [2019-12-17 12:28:21,952] jepsen worker 17 - jepsen.util 1007 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,953] jepsen worker 17 - jepsen.util 1007 :invoke :cas [5 [9 23]] INFO [2019-12-17 12:28:21,954] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:21,957] jepsen worker 38 - jepsen.util 848 :fail :cas [12 [26 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,957] jepsen worker 38 - jepsen.util 848 :invoke :read [12 nil] INFO [2019-12-17 12:28:21,966] jepsen worker 55 - jepsen.util 1135 :ok :write [18 14] INFO [2019-12-17 12:28:21,966] jepsen worker 55 - jepsen.util 1135 :invoke :read [18 nil] INFO [2019-12-17 12:28:21,974] jepsen worker 28 - jepsen.util 748 :ok :read [9 22] INFO [2019-12-17 12:28:21,974] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [32 32]] INFO [2019-12-17 12:28:21,978] jepsen worker 53 - jepsen.util 953 :fail :cas [17 [6 14]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,979] jepsen worker 53 - jepsen.util 953 :invoke :write [17 17] INFO [2019-12-17 12:28:21,986] jepsen worker 12 - jepsen.util 1002 :fail :cas [4 [43 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:21,987] jepsen worker 12 - jepsen.util 1002 :invoke :read [4 nil] INFO [2019-12-17 12:28:21,988] jepsen worker 0 - jepsen.util 1080 :info :write [0 6] :RequestCanceledException INFO [2019-12-17 12:28:21,988] jepsen worker 6 - jepsen.util 996 :info :write [2 15] :RequestCanceledException INFO [2019-12-17 12:28:21,988] jepsen worker 85 - jepsen.util 895 :fail :read [28 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,988] jepsen worker 66 - jepsen.util 876 :fail :read [22 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,988] jepsen worker 15 - jepsen.util 915 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:21,988] jepsen worker 85 - jepsen.util 895 :invoke :write [28 36] INFO [2019-12-17 12:28:21,988] jepsen worker 66 - jepsen.util 876 :invoke :write [22 33] INFO [2019-12-17 12:28:21,988] jepsen worker 15 - jepsen.util 915 :invoke :read [5 nil] INFO [2019-12-17 12:28:21,989] jepsen worker 0 - jepsen.util 1170 :invoke :cas [0 [31 36]] INFO [2019-12-17 12:28:21,989] jepsen worker 6 - jepsen.util 1086 :invoke :write [2 12] INFO [2019-12-17 12:28:22,002] jepsen nemesis - couchbase.util Rebalance status: {:status running, :ns_1@172.28.128.125 {:progress 0.09765625}, :ns_1@172.28.128.126 {:progress 0.08992805755395683}, :ns_1@172.28.128.127 {:progress 0.1789883268482491}, :ns_1@172.28.128.128 {:progress 0.6867924528301887}} INFO [2019-12-17 12:28:22,002] jepsen nemesis - couchbase.util Rebalance check count: 30 INFO [2019-12-17 12:28:22,002] jepsen nemesis - couchbase.util Rebalance stuck count: 11 INFO [2019-12-17 12:28:22,002] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [32 32]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,002] jepsen worker 28 - jepsen.util 748 :invoke :cas [9 [9 36]] INFO [2019-12-17 12:28:22,003] jepsen worker 45 - jepsen.util 1035 :fail :read [15 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 8 - jepsen.util 1088 :info :cas [2 [10 32]] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 76 - jepsen.util 976 :info :cas [25 [17 5]] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 49 - jepsen.util 949 :fail :read [16 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 11 - jepsen.util 1181 :fail :read [3 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 34 - jepsen.util 1114 :fail :read [11 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,003] jepsen worker 50 - jepsen.util 1130 :info :write [16 21] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 68 - jepsen.util 608 :info :cas [22 [29 37]] :RequestCanceledException INFO [2019-12-17 12:28:22,003] jepsen worker 45 - jepsen.util 1035 :invoke :write [15 37] INFO [2019-12-17 12:28:22,004] jepsen worker 49 - jepsen.util 949 :invoke :cas [16 [13 4]] INFO [2019-12-17 12:28:22,004] jepsen worker 11 - jepsen.util 1181 :invoke :read [3 nil] INFO [2019-12-17 12:28:22,004] jepsen worker 34 - jepsen.util 1114 :invoke :write [11 17] INFO [2019-12-17 12:28:22,005] jepsen worker 76 - jepsen.util 1066 :invoke :cas [25 [12 16]] INFO [2019-12-17 12:28:22,005] jepsen worker 8 - jepsen.util 1178 :invoke :cas [2 [17 38]] INFO [2019-12-17 12:28:22,005] jepsen worker 50 - jepsen.util 1220 :invoke :read [16 nil] INFO [2019-12-17 12:28:22,005] jepsen worker 68 - jepsen.util 698 :invoke :cas [22 [30 4]] INFO [2019-12-17 12:28:22,020] jepsen worker 87 - jepsen.util 987 :fail :cas [29 [21 9]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,020] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:22,020] jepsen worker 87 - jepsen.util 987 :invoke :write [29 27] INFO [2019-12-17 12:28:22,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4828,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:22,021] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} INFO [2019-12-17 12:28:22,021] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4831,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":2,"completed":true} {"coreId":2} INFO [2019-12-17 12:28:22,021] jepsen worker 60 - jepsen.util 1050 :info :write [20 10] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 9 - jepsen.util 819 :info :write [3 35] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 62 - jepsen.util 962 :info :write [20 11] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 22 - jepsen.util 922 :info :write [7 24] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 41 - jepsen.util 761 :info :write [13 13] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 67 - jepsen.util 1237 :info :write [22 35] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 42 - jepsen.util 1212 :info :write [14 7] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 63 - jepsen.util 783 :info :cas [21 [38 49]] :RequestCanceledException INFO [2019-12-17 12:28:22,021] jepsen worker 20 - jepsen.util 920 :info :write [6 45] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 57 - jepsen.util 1047 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,022] jepsen worker 2 - jepsen.util 722 :info :cas [0 [43 28]] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 59 - jepsen.util 869 :fail :read [19 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,022] jepsen worker 83 - jepsen.util 1163 :info :write [27 9] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 1 - jepsen.util 721 :info :cas [0 [39 29]] :RequestCanceledException INFO [2019-12-17 12:28:22,022] jepsen worker 84 - jepsen.util 804 :info :write [28 17] :RequestCanceledException INFO [2019-12-17 12:28:22,023] jepsen worker 60 - jepsen.util 1140 :invoke :write [20 48] INFO [2019-12-17 12:28:22,023] jepsen worker 9 - jepsen.util 909 :invoke :write [3 18] INFO [2019-12-17 12:28:22,023] jepsen worker 62 - jepsen.util 1052 :invoke :cas [20 [24 41]] INFO [2019-12-17 12:28:22,023] jepsen worker 57 - jepsen.util 1047 :invoke :write [19 28] INFO [2019-12-17 12:28:22,023] jepsen worker 22 - jepsen.util 1012 :invoke :cas [7 [21 16]] INFO [2019-12-17 12:28:22,023] jepsen worker 41 - jepsen.util 851 :invoke :cas [13 [7 42]] INFO [2019-12-17 12:28:22,023] jepsen worker 42 - jepsen.util 1302 :invoke :read [14 nil] INFO [2019-12-17 12:28:22,023] jepsen worker 59 - jepsen.util 869 :invoke :write [19 18] INFO [2019-12-17 12:28:22,023] jepsen worker 67 - jepsen.util 1327 :invoke :write [22 10] INFO [2019-12-17 12:28:22,024] jepsen worker 63 - jepsen.util 873 :invoke :cas [21 [7 12]] INFO [2019-12-17 12:28:22,024] jepsen worker 20 - jepsen.util 1010 :invoke :write [6 39] INFO [2019-12-17 12:28:22,024] jepsen worker 2 - jepsen.util 812 :invoke :cas [0 [24 48]] INFO [2019-12-17 12:28:22,024] jepsen worker 1 - jepsen.util 811 :invoke :write [0 40] INFO [2019-12-17 12:28:22,024] jepsen worker 84 - jepsen.util 894 :invoke :cas [28 [26 0]] INFO [2019-12-17 12:28:22,025] jepsen worker 83 - jepsen.util 1253 :invoke :cas [27 [43 18]] INFO [2019-12-17 12:28:22,033] jepsen worker 39 - jepsen.util 1119 :fail :read [13 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,033] jepsen worker 48 - jepsen.util 498 :info :cas [16 [39 40]] :RequestCanceledException INFO [2019-12-17 12:28:22,033] jepsen worker 16 - jepsen.util 1006 :fail :read [5 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,033] jepsen worker 33 - jepsen.util 1203 :info :write [11 39] :RequestCanceledException INFO [2019-12-17 12:28:22,033] jepsen worker 39 - jepsen.util 1119 :invoke :cas [13 [14 45]] INFO [2019-12-17 12:28:22,033] jepsen worker 16 - jepsen.util 1006 :invoke :read [5 nil] INFO [2019-12-17 12:28:22,034] jepsen worker 28 - jepsen.util 748 :fail :cas [9 [9 36]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,034] jepsen worker 28 - jepsen.util 748 :invoke :write [9 8] INFO [2019-12-17 12:28:22,035] jepsen worker 48 - jepsen.util 588 :invoke :write [16 43] INFO [2019-12-17 12:28:22,035] jepsen worker 33 - jepsen.util 1293 :invoke :read [11 nil] INFO [2019-12-17 12:28:22,052] jepsen worker 46 - jepsen.util 1126 :info :write [15 6] :RequestCanceledException INFO [2019-12-17 12:28:22,052] jepsen worker 86 - jepsen.util 986 :info :write [28 9] :RequestCanceledException INFO [2019-12-17 12:28:22,052] jepsen worker 82 - jepsen.util 1342 :info :write [27 35] :RequestCanceledException INFO [2019-12-17 12:28:22,052] jepsen worker 40 - jepsen.util 1390 :info :write [13 32] :RequestCanceledException INFO [2019-12-17 12:28:22,053] jepsen worker 46 - jepsen.util 1216 :invoke :read [15 nil] INFO [2019-12-17 12:28:22,053] jepsen worker 86 - jepsen.util 1076 :invoke :read [28 nil] INFO [2019-12-17 12:28:22,053] jepsen worker 82 - jepsen.util 1432 :invoke :write [27 0] INFO [2019-12-17 12:28:22,053] jepsen worker 40 - jepsen.util 1480 :invoke :write [13 44] INFO [2019-12-17 12:28:22,054] jepsen worker 89 - jepsen.util 1079 :ok :read [29 18] INFO [2019-12-17 12:28:22,054] jepsen worker 89 - jepsen.util 1079 :invoke :cas [29 [14 35]] INFO [2019-12-17 12:28:22,060] jepsen worker 54 - jepsen.util 1044 :fail :cas [18 [7 17]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,060] jepsen worker 54 - jepsen.util 1044 :invoke :write [18 0] INFO [2019-12-17 12:28:22,072] jepsen worker 11 - jepsen.util 1181 :ok :read [3 19] INFO [2019-12-17 12:28:22,072] jepsen worker 11 - jepsen.util 1181 :invoke :write [3 14] INFO [2019-12-17 12:28:22,076] jepsen worker 69 - jepsen.util 879 :ok :read [23 27] INFO [2019-12-17 12:28:22,076] jepsen worker 69 - jepsen.util 879 :invoke :cas [23 [44 34]] INFO [2019-12-17 12:28:22,078] jepsen worker 70 - jepsen.util 1150 :ok :read [23 27] INFO [2019-12-17 12:28:22,078] jepsen worker 70 - jepsen.util 1150 :invoke :read [23 nil] INFO [2019-12-17 12:28:22,087] jepsen worker 77 - jepsen.util 977 :info :write [25 10] :RequestCanceledException INFO [2019-12-17 12:28:22,087] jepsen worker 81 - jepsen.util 1161 :info :write [27 22] :RequestCanceledException INFO [2019-12-17 12:28:22,087] jepsen worker 23 - jepsen.util 833 :info :write [7 2] :RequestCanceledException INFO [2019-12-17 12:28:22,088] jepsen worker 81 - jepsen.util 1251 :invoke :read [27 nil] INFO [2019-12-17 12:28:22,088] jepsen worker 77 - jepsen.util 1067 :invoke :write [25 35] INFO [2019-12-17 12:28:22,088] jepsen worker 23 - jepsen.util 923 :invoke :cas [7 [42 45]] INFO [2019-12-17 12:28:22,103] jepsen worker 4 - jepsen.util 724 :fail :cas [1 [13 4]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,103] jepsen worker 4 - jepsen.util 724 :invoke :read [1 nil] INFO [2019-12-17 12:28:22,105] jepsen worker 19 - jepsen.util 1099 :info :write [6 14] :RequestCanceledException INFO [2019-12-17 12:28:22,105] jepsen worker 75 - jepsen.util 975 :info :write [25 43] :RequestCanceledException INFO [2019-12-17 12:28:22,106] jepsen worker 19 - jepsen.util 1189 :invoke :read [6 nil] INFO [2019-12-17 12:28:22,106] jepsen worker 75 - jepsen.util 1065 :invoke :write [25 2] INFO [2019-12-17 12:28:22,119] jepsen worker 37 - jepsen.util 847 :ok :write [12 43] INFO [2019-12-17 12:28:22,119] jepsen worker 37 - jepsen.util 847 :invoke :cas [12 [48 48]] INFO [2019-12-17 12:28:22,127] jepsen worker 8 - jepsen.util 1178 :fail :cas [2 [17 38]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,127] jepsen worker 8 - jepsen.util 1178 :invoke :read [2 nil] INFO [2019-12-17 12:28:22,134] jepsen worker 18 - jepsen.util 918 :info :write [6 43] :RequestCanceledException INFO [2019-12-17 12:28:22,135] jepsen worker 65 - jepsen.util 695 :fail :read [21 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,135] jepsen worker 30 - jepsen.util 1020 :info :write [10 11] :RequestCanceledException INFO [2019-12-17 12:28:22,135] jepsen worker 65 - jepsen.util 695 :invoke :cas [21 [5 37]] INFO [2019-12-17 12:28:22,136] jepsen worker 30 - jepsen.util 1110 :invoke :read [10 nil] INFO [2019-12-17 12:28:22,136] jepsen worker 18 - jepsen.util 1008 :invoke :read [6 nil] INFO [2019-12-17 12:28:22,157] jepsen worker 76 - jepsen.util 1066 :fail :cas [25 [12 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,157] jepsen worker 76 - jepsen.util 1066 :invoke :write [25 14] INFO [2019-12-17 12:28:22,164] jepsen worker 62 - jepsen.util 1052 :fail :cas [20 [24 41]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,164] jepsen worker 62 - jepsen.util 1052 :invoke :read [20 nil] INFO [2019-12-17 12:28:22,167] jepsen worker 28 - jepsen.util 748 :ok :write [9 8] INFO [2019-12-17 12:28:22,167] jepsen worker 28 - jepsen.util 748 :invoke :read [9 nil] INFO [2019-12-17 12:28:22,171] jepsen worker 30 - jepsen.util 1110 :ok :read [10 26] INFO [2019-12-17 12:28:22,172] jepsen worker 30 - jepsen.util 1110 :invoke :read [10 nil] INFO [2019-12-17 12:28:22,194] jepsen worker 78 - jepsen.util 978 :ok :write [26 17] INFO [2019-12-17 12:28:22,194] jepsen worker 51 - jepsen.util 861 :ok :write [17 35] INFO [2019-12-17 12:28:22,194] jepsen worker 78 - jepsen.util 978 :invoke :write [26 13] INFO [2019-12-17 12:28:22,194] jepsen worker 51 - jepsen.util 861 :invoke :cas [17 [33 42]] INFO [2019-12-17 12:28:22,207] jepsen worker 71 - jepsen.util 701 :ok :read [23 27] INFO [2019-12-17 12:28:22,207] jepsen worker 71 - jepsen.util 701 :invoke :write [23 21] INFO [2019-12-17 12:28:22,221] jepsen worker 22 - jepsen.util 1012 :fail :cas [7 [21 16]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,221] jepsen worker 22 - jepsen.util 1012 :invoke :cas [7 [26 22]] INFO [2019-12-17 12:28:22,231] jepsen worker 28 - jepsen.util 748 :ok :read [9 8] INFO [2019-12-17 12:28:22,232] jepsen worker 28 - jepsen.util 748 :invoke :write [9 21] INFO [2019-12-17 12:28:22,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:22,238] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4889,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:22,238] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} INFO [2019-12-17 12:28:22,239] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4890,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":1,"completed":true} {"coreId":1} INFO [2019-12-17 12:28:22,240] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:22,240] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4897,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:22,241] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} INFO [2019-12-17 12:28:22,241] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4902,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":3,"completed":true} {"coreId":3} INFO [2019-12-17 12:28:22,282] jepsen worker 30 - jepsen.util 1110 :ok :read [10 26] INFO [2019-12-17 12:28:22,283] jepsen worker 30 - jepsen.util 1110 :invoke :write [10 21] INFO [2019-12-17 12:28:22,305] jepsen worker 13 - jepsen.util 1003 :fail :cas [4 [13 28]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,305] jepsen worker 62 - jepsen.util 1052 :ok :read [20 7] INFO [2019-12-17 12:28:22,306] jepsen worker 13 - jepsen.util 1003 :invoke :write [4 1] INFO [2019-12-17 12:28:22,306] jepsen worker 62 - jepsen.util 1052 :invoke :write [20 29] INFO [2019-12-17 12:28:22,306] jepsen worker 58 - jepsen.util 958 :info :write [19 33] :RequestCanceledException INFO [2019-12-17 12:28:22,313] jepsen worker 58 - jepsen.util 1048 :invoke :cas [19 [18 6]] INFO [2019-12-17 12:28:22,337] jepsen worker 89 - jepsen.util 1079 :fail :cas [29 [14 35]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,337] jepsen worker 89 - jepsen.util 1079 :invoke :read [29 nil] INFO [2019-12-17 12:28:22,363] jepsen worker 74 - jepsen.util 794 :ok :write [24 27] INFO [2019-12-17 12:28:22,363] jepsen worker 74 - jepsen.util 794 :invoke :read [24 nil] INFO [2019-12-17 12:28:22,365] jepsen worker 31 - jepsen.util 1021 :ok :write [10 31] INFO [2019-12-17 12:28:22,366] jepsen worker 47 - jepsen.util 1217 :ok :write [15 13] INFO [2019-12-17 12:28:22,366] jepsen worker 31 - jepsen.util 1021 :invoke :cas [10 [44 23]] INFO [2019-12-17 12:28:22,366] jepsen worker 47 - jepsen.util 1217 :invoke :cas [15 [16 6]] INFO [2019-12-17 12:28:22,379] jepsen worker 21 - jepsen.util 1101 :ok :read [7 20] INFO [2019-12-17 12:28:22,379] jepsen worker 21 - jepsen.util 1101 :invoke :read [7 nil] INFO [2019-12-17 12:28:22,380] jepsen worker 23 - jepsen.util 923 :fail :cas [7 [42 45]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,380] jepsen worker 23 - jepsen.util 923 :invoke :cas [7 [37 34]] INFO [2019-12-17 12:28:22,383] jepsen worker 5 - jepsen.util 635 :ok :read [1 47] INFO [2019-12-17 12:28:22,383] jepsen worker 5 - jepsen.util 635 :invoke :write [1 21] INFO [2019-12-17 12:28:22,384] jepsen worker 10 - jepsen.util 1090 :info :cas [3 [5 33]] :RequestCanceledException INFO [2019-12-17 12:28:22,390] jepsen worker 10 - jepsen.util 1180 :invoke :write [3 12] INFO [2019-12-17 12:28:22,418] jepsen worker 28 - jepsen.util 748 :ok :write [9 21] INFO [2019-12-17 12:28:22,418] jepsen worker 28 - jepsen.util 748 :invoke :write [9 13] INFO [2019-12-17 12:28:22,419] jepsen worker 20 - jepsen.util 1010 :ok :write [6 39] INFO [2019-12-17 12:28:22,419] jepsen worker 20 - jepsen.util 1010 :invoke :write [6 35] INFO [2019-12-17 12:28:22,422] jepsen worker 38 - jepsen.util 848 :ok :read [12 43] INFO [2019-12-17 12:28:22,423] jepsen worker 38 - jepsen.util 848 :invoke :write [12 20] INFO [2019-12-17 12:28:22,427] jepsen worker 42 - jepsen.util 1302 :ok :read [14 27] INFO [2019-12-17 12:28:22,427] jepsen worker 42 - jepsen.util 1302 :invoke :read [14 nil] INFO [2019-12-17 12:28:22,453] jepsen worker 55 - jepsen.util 1135 :ok :read [18 14] INFO [2019-12-17 12:28:22,552] jepsen worker 32 - jepsen.util 842 :info :cas [10 [15 43]] :RequestCanceledException INFO [2019-12-17 12:28:22,552] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:22,553] jepsen worker 63 - jepsen.util 873 :fail :cas [21 [7 12]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4921,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4922,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:22,554] jepsen worker 63 - jepsen.util 873 :invoke :write [21 12] INFO [2019-12-17 12:28:22,554] cb-events - com.couchbase.request [com.couchbase.request][RequestNotRetriedEvent] Request CarrierBucketConfigRequest not retried per RetryStrategy (Reason: ENDPOINT_NOT_WRITABLE) {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} INFO [2019-12-17 12:28:22,555] cb-events - com.couchbase.config [com.couchbase.config][BucketConfigRefreshFailedEvent] Reason: INDIVIDUAL_REQUEST_FAILED, Type: KV, Cause: com.couchbase.client.core.error.RequestCanceledException: CarrierBucketConfigRequest {"retried":0,"reason":"NO_MORE_RETRIES (ENDPOINT_NOT_WRITABLE)","requestId":4923,"timeoutMs":2500,"service":{"bucket":"default","scope":"_default","collection":"_default","type":"kv","key":""},"cancelled":true,"coreId":4,"completed":true} {"coreId":4} INFO [2019-12-17 12:28:22,555] jepsen worker 74 - jepsen.util 794 :ok :read [24 27] INFO [2019-12-17 12:28:22,555] jepsen worker 74 - jepsen.util 794 :invoke :write [24 13] INFO [2019-12-17 12:28:22,555] jepsen worker 32 - jepsen.util 932 :invoke :cas [10 [0 49]] INFO [2019-12-17 12:28:22,556] jepsen worker 19 - jepsen.util 1189 :ok :read [6 47] INFO [2019-12-17 12:28:22,557] jepsen worker 19 - jepsen.util 1189 :invoke :write [6 43] INFO [2019-12-17 12:28:22,559] jepsen worker 55 - jepsen.util 1135 :invoke :write [18 14] INFO [2019-12-17 12:28:22,565] jepsen worker 46 - jepsen.util 1216 :ok :read [15 13] INFO [2019-12-17 12:28:22,565] jepsen worker 46 - jepsen.util 1216 :invoke :read [15 nil] INFO [2019-12-17 12:28:22,567] jepsen worker 4 - jepsen.util 724 :ok :read [1 47] INFO [2019-12-17 12:28:22,568] jepsen worker 4 - jepsen.util 724 :invoke :write [1 43] INFO [2019-12-17 12:28:22,568] jepsen worker 3 - jepsen.util 723 :fail :cas [1 [12 13]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,569] jepsen worker 3 - jepsen.util 723 :invoke :read [1 nil] INFO [2019-12-17 12:28:22,571] jepsen worker 37 - jepsen.util 847 :fail :cas [12 [48 48]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,571] jepsen worker 37 - jepsen.util 847 :invoke :read [12 nil] INFO [2019-12-17 12:28:22,572] jepsen worker 47 - jepsen.util 1217 :fail :cas [15 [16 6]] :ValueNotSwapFrom INFO [2019-12-17 12:28:22,572] jepsen worker 47 - jepsen.util 1217 :invoke :write [15 40] INFO [2019-12-17 12:28:22,588] jepsen worker 73 - jepsen.util 793 :info :write [24 7] :RequestCanceledException INFO [2019-12-17 12:28:22,588] jepsen worker 72 - jepsen.util 792 :fail :read [24 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,588] jepsen worker 27 - jepsen.util 747 :fail :read [9 nil] :RequestTimeoutException INFO [2019-12-17 12:28:22,588] jepsen worker 27 - jepsen.util 747 :invoke :read [9 nil] INFO [2019-12-17 12:28:22,588] jepsen worker 72 - jepsen.util 792 :invoke :cas [24 [0 49]] INFO [2019-12-17 12:28:22,589] jepsen worker 73 - jepsen.util 883 :invoke :write [24 42] INFO [2019-12-17 12:28:22,590] jepsen worker 8 - jepsen.util 1178 :ok :read [2 15] INFO [2019-12-17 12:28:22,591] jepsen worker 8 - jepsen.util 1178 :invoke :cas [2 [16 4]] INFO [2019-12-17 12:28:22,603] jepsen worker 65 - jepsen.util 695 :fail :cas [21 [5 37]] :ValueNotSwapF