GoXDCR: Node cannot be added to cluster after reboot (and firewall turned off)

Build
-------
3.5.0-1270

• In previous test, we rebooted .62 (hard shutdown of Couchbase server) and firewall was disabled afterwards, replication happened fine. We then removed .62 from .57 cluster
• In next test, we tried to add .62 to .57 again, .62 was unresponsive and requests timed-out.

Seeing errors like -

Error adding node: 172.23.105.62 to the cluster:172.23.105.57 - ["Prepare join failed. Timeout connecting to \"172.23.105.62\" on port 8091. This could be due to an incorrect host/port combination or a firewall in place between the servers."]

Easily reproducible after running -

./testrunner -i INI_FILE.ini -p get-cbcollect-info=True,get-logs=False,stop-on-failure=False,replication_type=xmem,enable_goxdcr=True,checkpoint_interval=3600 -t xdcr.biXDCR.bidirectional.replication_while_rebooting_a_non_master_src_dest_node,items=10000,rdirection=bidirection,sasl_buckets=1,ctopology=chain

Attaching logs.

[ns_server:error,2015-02-17T16:36:51.798,ns_1@172.23.105.62:<0.28306.0>:menelaus_web:loop:182]Server error during processing: ["web request failed",

{path,"/engageCluster2"}

,

{type,exit}

,
{what,
{timeout,
{gen_server,call,
[ns_cluster,
{engage_cluster,
[

{<<"requestedTargetNodeHostname">>, <<"172.23.105.62">>}

,
{<<"availableStorage">>,
{struct,
[{<<"hdd">>,
[{struct,
[

{<<"path">>,<<"/">>}

,

{<<"sizeKBytes">>,35377908}

,

{<<"usagePercent">>,47}

]},
{struct,
[

{<<"path">>,<<"/dev/shm">>}

,

{<<"sizeKBytes">>,7790528}

,

{<<"usagePercent">>,0}

]},
{struct,
[

{<<"path">>,<<"/boot">>}

,

{<<"sizeKBytes">>,495844}

,

{<<"usagePercent">>,7}

]},
{struct,
[

{<<"path">>,<<"/data">>}

,

{<<"sizeKBytes">>,423166440}

,

{<<"usagePercent">>,1}

]}]}]}},

{<<"memoryQuota">>,8114}

,
{<<"storageTotals">>,
{struct,
[{<<"ram">>,
{struct,
[

{<<"total">>,15955001344}

,

{<<"quotaTotal">>,8508145664}

,

{<<"quotaUsed">>,0}

,

{<<"used">>,9733283840}

,

{<<"usedByData">>,0},
{<<"quotaUsedPerNode">>,0},
{<<"quotaTotalPerNode">>, 8508145664}]}},
{<<"hdd">>,
{struct,
[{<<"total">>,36226977792},
{<<"quotaTotal">>,36226977792},
{<<"used">>,17026679562},
{<<"usedByData">>,0}

,

{<<"free">>,19200298230}

]}}]}},
{<<"storage">>,
{struct,
[

{<<"ssd">>,[]}

,
{<<"hdd">>,
[{struct,
[

{<<"path">>, <<"/opt/couchbase/var/lib/couchbase/data">>}

,

{<<"index_path">>, <<"/opt/couchbase/var/lib/couchbase/data">>}

,

{<<"quotaMb">>,<<"none">>}

,

{<<"state">>,<<"ok">>}

]}]}]}},
{<<"systemStats">>,
{struct,
[

{<<"cpu_utilization_rate">>, 1.0335917312661498}

,

{<<"swap_total">>,10984873984}

,

{<<"swap_used">>,0}

,

{<<"mem_total">>,15955001344}

,

{<<"mem_free">>,14928711680}

]}},
{<<"interestingStats">>,{struct,[]}},

{<<"uptime">>,<<"14895">>}

,

{<<"memoryTotal">>,15955001344}

,

{<<"memoryFree">>,14928711680}

,

{<<"mcdMemoryReserved">>,12172}

,

{<<"mcdMemoryAllocated">>,12172}

,

{<<"couchApiBase">>, <<"http://172.23.105.57:8092/">>}

,

{<<"otpCookie">>, <<"kqgzbpabfcphmrux">>}

,

{<<"clusterMembership">>,<<"active">>}

,

{<<"recoveryType">>,<<"none">>}

,

{<<"status">>,<<"healthy">>}

,

{<<"otpNode">>, <<"ns_1@172.23.105.57">>}

,

{<<"thisNode">>,true}

,

{<<"hostname">>, <<"172.23.105.57:8091">>}

,

{<<"clusterCompatibility">>,196610}

,

{<<"version">>, <<"3.5.0-1270-rel-enterprise">>}

,

{<<"os">>, <<"x86_64-unknown-linux-gnu">>}

,
{<<"ports">>,
{struct,
[

{<<"sslProxy">>,11214}

,

{<<"httpsMgmt">>,18091}

,

{<<"httpsCAPI">>,18092}

,

{<<"proxy">>,11211}

,

{<<"direct">>,11210}

]}},

{<<"services">>,[<<"kv">>]}

]},
30000]}}},
{trace,
[{gen_server,call,3,
[

{file,"gen_server.erl"}

,

{line,188}

]},
{menelaus_web,handle_engage_cluster2,1,
[

{file,"src/menelaus_web.erl"},
{line,871}]},
{request_throttler,do_request,3,
[{file,"src/request_throttler.erl"},
{line,59}]},
{menelaus_web,loop,2,
[{file,"src/menelaus_web.erl"}

,

{line,161}

]},
{mochiweb_http,headers,5,
[

{file, "/home/couchbase/jenkins/workspace/sherlock-unix/couchdb/src/mochiweb/mochiweb_http.erl"}

,

{line,94}

]},
{proc_lib,init_p_do_apply,3,
[

{file,"proc_lib.erl"}

,

{line,239}

]}]}]