nodes statuses are not synchronized in pools/default and /nodeStatuses

found when worked with failover tests. the issue is also related to MB-10479( and possible MB-10212)

steps:
turn on firewall on 10.3.4.146

the status for 10.3.4.146 in nodeStatuses is "unhealthy"
the status for 10.3.4.146 in /pools/default is "healthy"

http://10.3.4.144:8091/nodeStatuses

{"10.3.4.144:8091":

{"status":"healthy","gracefulFailoverPossible":true,"otpNode":"ns_1@10.3.4.144","replication":0}

,"10.3.4.145:8091":

{"status":"healthy","gracefulFailoverPossible":true,"otpNode":"ns_1@10.3.4.145","replication":0}

,"10.3.4.146:8091":{"status":"unhealthy","otpNode":"ns_1@10.3.4.146","replication":1}}

http://10.3.4.144:8091/pools/default
{"storageTotals":{"ram":

{"total":12120231936,"quotaTotal":4307550208,"quotaUsed":2868903936,"used":6327033856,"usedByData":10377984,"quotaUsedPerNode":1434451968,"quotaTotalPerNode":2153775104}

,"hdd":{"total":168982376448,"quotaTotal":168982376448,"used":17461512231,"usedByData":2433998,"free":150394315041}},"serverGroupsUri":"/pools/default/serverGroups?v=56764646","name":"default","alerts":[],"alertsSilenceURL":"/controller/resetAlerts?token=0&uuid=9e267c7fc310cf185e319d728ce39eea","nodes":[{"systemStats":

{"cpu_utilization_rate":1.015228426395939,"swap_total":6140452864,"swap_used":233431040,"mem_total":4040077312,"mem_free":3214118912}

,"interestingStats":

{"cmd_get":0,"couch_docs_actual_disk_size":841180,"couch_docs_data_size":782918,"couch_views_actual_disk_size":0,"couch_views_data_size":0,"curr_items":8,"curr_items_tot":14,"ep_bg_fetched":0,"get_hits":0,"mem_used":3541552,"ops":0,"vb_replica_curr_items":6}

,"uptime":"885","memoryTotal":4040077312,"memoryFree":3214118912,"mcdMemoryReserved":3082,"mcdMemoryAllocated":3082,"couchApiBase":"http://10.3.4.144:8092/","otpCookie":"cdlvoudlxfagzmkm","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"ns_1@10.3.4.144","thisNode":true,"hostname":"10.3.4.144:8091","clusterCompatibility":196608,"version":"3.0.0-451-rel-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"sslDirect":11207,"proxy":11211,"direct":11210}},{"systemStats":

{"cpu_utilization_rate":1.511335012594458,"swap_total":6140452864,"swap_used":102400,"mem_total":4040077312,"mem_free":3297746944}

,"interestingStats":

{"cmd_get":0,"couch_docs_actual_disk_size":801356,"couch_docs_data_size":758248,"couch_views_actual_disk_size":0,"couch_views_data_size":0,"curr_items":6,"curr_items_tot":16,"ep_bg_fetched":0,"get_hits":0,"mem_used":3420784,"ops":0,"vb_replica_curr_items":10}

,"uptime":"774","memoryTotal":4040077312,"memoryFree":3297746944,"mcdMemoryReserved":3082,"mcdMemoryAllocated":3082,"couchApiBase":"http://10.3.4.146:8092/","otpCookie":"cdlvoudlxfagzmkm","clusterMembership":"active","recoveryType":"none","status":"unhealthy","otpNode":"ns_1@10.3.4.146","hostname":"10.3.4.146:8091","clusterCompatibility":196608,"version":"3.0.0-451-rel-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"sslDirect":11207,"proxy":11211,"direct":11210}},{"systemStats":

{"cpu_utilization_rate":0.5050505050505051,"swap_total":6140452864,"swap_used":102400,"mem_total":4040077312,"mem_free":3314155520}

,"interestingStats":

{"cmd_get":0,"couch_docs_actual_disk_size":791462,"couch_docs_data_size":733402,"couch_views_actual_disk_size":0,"couch_views_data_size":0,"curr_items":6,"curr_items_tot":10,"ep_bg_fetched":0,"get_hits":0,"mem_used":3415648,"ops":0,"vb_replica_curr_items":4}

,"uptime":"882","memoryTotal":4040077312,"memoryFree":3314155520,"mcdMemoryReserved":3082,"mcdMemoryAllocated":3082,"couchApiBase":"http://10.3.4.145:8092/","otpCookie":"cdlvoudlxfagzmkm","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"ns_1@10.3.4.145","hostname":"10.3.4.145:8091","clusterCompatibility":196608,"version":"3.0.0-451-rel-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"sslDirect":11207,"proxy":11211,"direct":11210}}],"buckets":

{"uri":"/pools/default/buckets?v=35622459&uuid=9e267c7fc310cf185e319d728ce39eea","terseBucketsBase":"/pools/default/b/","terseStreamingBucketsBase":"/pools/default/bs/"}

,"remoteClusters":

{"uri":"/pools/default/remoteClusters?uuid=9e267c7fc310cf185e319d728ce39eea","validateURI":"/pools/default/remoteClusters?just_validate=1"}

,"controllers":{"addNode":

{"uri":"/controller/addNode?uuid=9e267c7fc310cf185e319d728ce39eea"}

,"rebalance":

{"uri":"/controller/rebalance?uuid=9e267c7fc310cf185e319d728ce39eea","requireDeltaRecoveryURI":"/controller/rebalance?uuid=9e267c7fc310cf185e319d728ce39eea&requireDeltaRecovery=true"}

,"failOver":

{"uri":"/controller/failOver?uuid=9e267c7fc310cf185e319d728ce39eea"}

,"startGracefulFailover":

{"uri":"/controller/startGracefulFailover?uuid=9e267c7fc310cf185e319d728ce39eea"}

,"reAddNode":

{"uri":"/controller/reAddNode?uuid=9e267c7fc310cf185e319d728ce39eea"}

,"ejectNode":

{"uri":"/controller/ejectNode?uuid=9e267c7fc310cf185e319d728ce39eea"}

,"setRecoveryType":

{"uri":"/controller/setRecoveryType?uuid=9e267c7fc310cf185e319d728ce39eea"}

,"setAutoCompaction":

{"uri":"/controller/setAutoCompaction?uuid=9e267c7fc310cf185e319d728ce39eea","validateURI":"/controller/setAutoCompaction?just_validate=1"}

,"replication":

{"createURI":"/controller/createReplication?uuid=9e267c7fc310cf185e319d728ce39eea","validateURI":"/controller/createReplication?just_validate=1"}

,"setFastWarmup":{"uri":"/controller/setFastWarmup?uuid=9e267c7fc310cf185e319d728ce39eea","validateURI":"/controller/setFastWarmup?just_validate=1"}},"rebalanceStatus":"none","rebalanceProgressUri":"/pools/default/rebalanceProgress","stopRebalanceUri":"/controller/stopRebalance?uuid=9e267c7fc310cf185e319d728ce39eea","nodeStatusesUri":"/nodeStatuses","maxBucketCount":10,"autoCompactionSettings":{"parallelDBAndViewCompaction":false,"databaseFragmentationThreshold":

{"percentage":30,"size":"undefined"}

,"viewFragmentationThreshold":{"percentage":30,"size":"undefined"}},"fastWarmupSettings":

{"fastWarmupEnabled":true,"minMemoryThreshold":10,"minItemsThreshold":10}

,"tasks":

{"uri":"/pools/default/tasks?v=133172395"}

,"counters":

{"rebalance_success":1,"rebalance_start":1}

}