XDCR fails, even on empty bucket

When trying to setup replication to elasticsearch, most of the time it generates errors: [Vb Rep] Error replicating vbucket 610. Please see logs for details.

Sometimes it does work. I'm inserting the same data every time. The amount that gets replicated varies, sometime nothing, other times just a couple of document, sometimes half the data. And very rarely all data is replicated.

If I flush the bucket, and re-create the index in elasticsearch, it sometimes fails even when I just setup the replication on the empty bucket.

No logs are generated in elasticsearch.

Couchbase error / debug log:
[error_logger:error,2014-06-19T17:00:29.768,ns_1@example.com:error_logger<0.6.0>:ale_error_logger_handler:log_msg:119]** Generic server <0.20587.6> terminating

• • Last message in was {'EXIT',<0.20607.6>,
    {http_request_failed,"POST",
    "http://administrator:*****@x.x.x.x:9091/testbucket%2f142%3b07063f2e0e0e41de85436c7c1f7c7449/_revs_diff",
    {error,
    Unknown macro: {code,503}

    }}}

  • When Server state == [{data,
    [{"State",
    {rep_state,
    {rep,
    <<"f4268e62702130298bf87f17cc481219/testbucket/testbucket">>,
    <<"testbucket">>,
    <<"/remoteClusters/f4268e62702130298bf87f17cc481219/buckets/testbucket">>,
    "capi",
    [ {optimistic_replication_threshold,256}

    ,
    

    {worker_batch_size,500}

    ,
    

    {failure_restart_interval,30}

    ,
    

    {doc_batch_size_kb,2048}

    ,
    

    {checkpoint_interval,1800}

    ,
    

    {max_concurrent_reps,32}

    ,
    

    {connection_timeout,180}

    ,
    

    {worker_processes,4}

    ,
    

    {http_connections,20}

    ,
    

    {retries_per_request,2}

    ,
    

    Unknown macro: {socket_options, [{keepalive,true},{nodelay,false}]},
    {supervisor_max_r,25},
    {supervisor_max_t,5},
    {trace_dump_invprob,1000}]},
    {rep_vb_status,142,<0.20587.6>,replicating,0,0,
    0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,
    {dict,0,16,16,8,80,48,
    {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[], []},
    {{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
    []}}}},
    {1403,190029,8270},
    nil,nil,<0.7494.6>,<0.7490.6>,
    <<"testbucket/142">>,
    <<"*****@x.x.x.x:9091/testbucket%2f142%3b07063f2e0e0e41de85436c7c1f7c7449">>,
    {db,<0.6240.6>,<0.6241.6>,nil,
    <<"1403189739030319">>,<0.6237.6>,<0.6242.6>,
    {db_header,11,1, <<0,0,0,0,65,132,0,0,0,0,0,64,0,0,0,0,1,0,0,0,0, 0,0,0,0,0,1,86>>, <<0,0,0,0,65,196,0,0,0,0,0,62,0,0,0,0,1>>, <<0,0,0,0,48,46,0,0,0,0,0,138>>, 0,nil,nil},
    1,
    {btree,<0.6237.6>,
    {16772,<<0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,86>>,64},
    #Fun<couch_db_updater.7.108595492>,
    #Fun<couch_db_updater.8.83286466>,
    #Fun<couch_btree.1.72034400>,
    #Fun<couch_db_updater.9.132659170>,1279,2558,
    true},
    {btree,<0.6237.6>,
    {16836,<<0,0,0,0,1>>,62},
    #Fun<couch_db_updater.10.23517937>,
    #Fun<couch_db_updater.11.59258139>,
    #Fun<couch_db_updater.6.7170572>,
    #Fun<couch_db_updater.12.109813745>,1279,2558,
    true},
    {btree,<0.6237.6>,
    {12334,<<>>,138},
    identity,identity,#Fun<couch_btree.1.72034400>,
    nil,1279,2558,true},
    1,<<"testbucket/142">>,
    "/data/couchbase/testbucket/142.couch.1",[],nil,
    {user_ctx,null,[],undefined},
    nil,
    [before_header,after_header,on_file_open],
    []},
    {httpdb,
    "*****@x.x.x.x:9091/testbucket%2f142%3b07063f2e0e0e41de85436c7c1f7c7449/",
    nil,
    [{"Accept","application/json"},
    {"User-Agent","CouchDB/1.2.0a-a425d97-git"}],
    180000,
    [{connect_options,
    [{keepalive,true},{nodelay,false}]},
    {connect_timeout,180000},
    {send_retry,0}],
    2,250,xdc_lhttpc_pool,20},
    {db,<0.25502.5>,<0.25503.5>,nil,
    <<"1403189707110801">>,<0.25498.5>,<0.25505.5>,
    {db_header,11,0,nil,nil,nil,0,nil,nil},
    0,
    {btree,<0.25498.5>,nil, #Fun<couch_db_updater.7.108595492>, #Fun<couch_db_updater.8.83286466>, #Fun<couch_btree.1.72034400>, #Fun<couch_db_updater.9.132659170>,1279,2558, true},
    {btree,<0.25498.5>,nil, #Fun<couch_db_updater.10.23517937>, #Fun<couch_db_updater.11.59258139>, #Fun<couch_db_updater.6.7170572>, #Fun<couch_db_updater.12.109813745>,1279,2558, true},
    {btree,<0.25498.5>,nil,identity,identity, #Fun<couch_btree.1.72034400>,nil,1279,2558, true},
    0,<<"testbucket/master">>,
    "/data/couchbase/testbucket/master.couch.1",[],
    nil,
    {user_ctx,null,[],undefined},
    nil,
    [before_header,after_header,on_file_open],
    [create]},
    {httpdb,
    "*****@x.x.x.x:9091/testbucket%2Fmaster%3B07063f2e0e0e41de85436c7c1f7c7449/",
    nil,
    [{"Accept","application/json"},
    {"User-Agent","CouchDB/1.2.0a-a425d97-git"}],
    180000,
    [{connect_options,
    [{keepalive,true},{nodelay,false}]}

    ,
    

    {connect_timeout,180000}

    ,
    

    {send_retry,0}

    ],
    2,250,xdc_lhttpc_pool,20},
    <<"ea145fd3618cc1743a1d23f967d5edc5">>,
    <<"6fe65c68467f49f39ccb242d5936b69d">>,0,
    undefined,0,0,
    [1],
    0,"Thu, 19 Jun 2014 15:00:29 GMT",
    

    {1403191829011369,#Ref<0.0.15.95320>}

    ,
    

    {1403,190029,11535}

    ,
    

    {1403,190029,8311}

    ,
    [<0.20607.6>],
    <0.20601.6>,false}}]}]

  • Reason for termination ==
  • {http_request_failed,"POST",
    "http://administrator:*****@x.x.x.x:9091/testbucket%2f142%3b07063f2e0e0e41de85436c7c1f7c7449/_revs_diff",
    
    Unknown macro: {error,{code,503}}

    }