Uploaded image for project: 'Couchbase Server'
  1. Couchbase Server
  2. MB-10910

Rebalance with views after failover fails due to "wait_checkpoint_persisted_failed"

    XMLWordPrintable

Details

    • Bug
    • Resolution: Duplicate
    • Critical
    • 3.0
    • 3.0
    • ns_server, view-engine
    • Security Level: Public
    • 3.0.0-594

      Platform = Physical
      OS = CentOS 6.5
      CPU = Intel Xeon E5-2630
      Memory = 64 GB
      Disk = 2 x SSD

    Description

      Rebalance after failover, 3 -> 4 nodes, 1 bucket x 100M x 2KB, DGM, 1 x 1 views, 10K ops/sec, 400 qps

      Rebalance exited with reason {unexpected_exit,
      {'EXIT',<0.16719.261>,
      {wait_checkpoint_persisted_failed,"bucket-1",
      892,2204,
      [{'ns_1@172.23.96.18',
      {'EXIT',
      {{{{badmatch,
      {error,
      {{function_clause,
      [{couch_set_view_group,handle_info,
      [

      {'DOWN',#Ref<16800.0.690.88151>, process,<16800.29051.86>,normal}

      ,
      {state,
      {"/ssd2",<<"bucket-1">>,
      {set_view_group,
      <<185,152,194,156,196,175,136,
      156,249,192,91,24,8,146,109,
      97>>,
      nil,<<"bucket-1">>,
      <<"_design/A">>,[],
      [{set_view,0,
      <<"\n function(doc, meta)

      {\n emit(doc.city, null);\n }\n ">>,
      undefined,
      {mapreduce_view,
      [<<"id_by_city">>],
      nil,[],[]}}],
      nil,nil,
      {set_view_index_header,2,0,0, 0,0,[],nil,[],false,[],nil, [],[]},
      main,nil,nil,nil,[],
      mapreduce_view,".view",prod,
      couch_set_view_stats_prod,0,
      nil}},
      <16800.28934.86>,
      {set_view_group,
      <<185,152,194,156,196,175,136,
      156,249,192,91,24,8,146,109,97>>,
      <16800.28930.86>,
      <<"bucket-1">>,<<"_design/A">>,
      [],
      [{set_view,0,
      <<"\n function(doc, meta) {n emit(doc.city, null);n }

      \n ">>,
      #Ref<16800.0.690.85787>,
      {mapreduce_view,
      [<<"id_by_city">>],
      {btree,<16800.28930.86>,

      {872023493, <<0,0,94,204,32,255,254, 254,191,207,240,120,0,0, 0,3,255,128,0,0,0,0,0,0, 0,0,3,199,65,254,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0>>, 115226299}

      ,
      identity,identity,
      #Fun<mapreduce_view.14.59760005>,
      #Fun<mapreduce_view.13.116710578>,
      7168,6144,true},
      [],[]}}],
      {btree,<16800.28930.86>,

      {757316813, <<0,0,94,204,32,255,254,254, 191,207,240,120,0,0,0,3,255, 128,0,0,0,0,0,0,0,0,3,199, 65,254,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0>>, 98005067}

      ,
      identity,identity,
      #Fun<couch_btree.1.34546481>,
      #Fun<couch_set_view_group.15.54553809>,
      7168,6144,true},
      <16800.28933.86>,
      {set_view_index_header,2,1024,
      179769313486231580793728973728767334576404131515618582199733964182346700693311000189374238233220773621413668943491054022143327228131710028156464112654519069829469374388748562738720335524163543296024799521113246188546084449778835326689833933543997895438175469077364855610760285917179803342821150185391109701632,
      7484401160755199293711447201500438591369787779463561409980691526969024494823315590706649929017925717627609195968322657071845869263978468466858275373827242994228699112960690437335703692189245416448148245614649742243301249193480738942045004516930273328675008179153418008263268875886110850416640,
      0,
      [

      {820,0}

      ,

      {821,0}

      ,

      {822,0}

      ,

      {823,0}

      ,

      {824,0}

      ,

      {825,98576}

      ,

      {826,98613}

      ,

      {827,98636}

      ,

      {828,98649}

      ,

      {829,98744}

      ,

      {830,98776}

      ,

      {831,98779}

      ,

      {832,98834}

      ,

      {833,0}

      ,

      {834,0}

      ,

      {835,0}

      ,

      {836,0}

      ,

      {837,0}

      ,

      {838,98538}

      ,

      {839,0}

      ,

      {840,98596}

      ,

      {841,98622}

      ,

      {842,98599}

      ,

      {843,0}

      ,

      {844,0}

      ,

      {845,0}

      ,

      {846,98663}

      ,

      {847,98725}

      ,

      {848,98727}

      ,

      {849,98793}

      ,

      {850,0}

      ,

      {851,0}

      ,

      {852,0}

      ,

      {891,0}

      ,

      {892,0}

      ,

      {893,0}

      ,

      {894,0}

      ,

      {895,0}

      ,

      {896,0}

      ,

      {897,0}

      ,

      {898,0}

      ,

      {899,0}

      ,

      {900,0}

      ,

      {901,0}

      ,

      {902,0}

      ,

      {903,0}

      ,

      {904,0}

      ,

      {905,0}

      ,

      {906,0}

      ,

      {907,0}

      ,

      {908,0}

      ,

      {909,0}

      ,

      {910,0}

      ,

      {911,0}

      ,

      {912,0}

      ,

      {913,0}

      ,

      {914,0}

      ,

      {915,0}

      ,

      {916,0}

      ,

      {917,0}

      ,

      {918,0}

      ,

      {919,0}

      ,

      {920,0}

      ,

      {921,0}

      ,

      {922,0}

      ,

      {923,0}

      ,

      {924,0}

      ,

      {925,0}

      ,

      {926,0}

      ,

      {927,98576}

      ,

      {928,98605}

      ,

      {929,98654}

      ,

      {930,98651}

      ,

      {931,98704}

      ,

      {932,98721}

      ,

      {933,98738}

      ,

      {934,98713}

      ,

      {935,98782}

      ,

      {936,98779}

      ,

      {937,98760}

      ,

      {968,0}

      ,

      {969,0}

      ,

      {970,0}

      ,

      {971,98877}

      ,

      {972,98914}

      ,

      {973,98961}

      ,

      {974,99028}

      ,

      {975,0}

      ,

      {976,0}

      ,

      {977,0}

      ,

      {978,0}

      ,

      {979,0}

      ,

      {980,98539}

      ,

      {981,98574}

      ,

      {982,98586}

      ,

      {983,98664}

      ,

      {984,98658}

      ,

      {985,98722}

      ,

      {986,98704}

      ,

      {987,98676}

      ,

      {988,0}

      ,

      {989,0}

      ,

      {990,98535}

      ,

      {991,98594}

      ,

      {992,98591}

      ,

      {993,98649}

      ,

      {994,98653}

      ,

      {995,98711}

      ,

      {996,98717}

      ,

      {997,98774}

      ,

      {998,0}

      ,

      {999,98396}

      ,

      {1000,0}

      ,

      {1001,98395}

      ,

      {1002,98430}

      ,

      {1003,98456}

      ,

      {1004,98502}

      ,

      {1005,98528}

      ,

      {1006,98570}

      ,

      {1007,98602}

      ,

      {1008,0}

      ,

      {1009,98412}

      ,

      {1010,98354}

      ,

      {1011,98346}

      ,

      {1012,98366}

      ,

      {1013,98360}

      ,

      {1014,98358}

      ,

      {1015,98378}

      ,

      {1016,98345}

      ,

      {1017,98405}

      ,

      {1018,98375}

      ,

      {1019,98428}

      ,

      {1020,98454}

      ,

      {1021,98460}

      ,

      {1022,98494}

      ,

      {1023,98521}

      ],
      <<0,0,45,35,188,205,0,0,5,215,
      112,75,0,0,94,204,32,255,254,
      254,191,207,240,120,0,0,0,3,
      255,128,0,0,0,0,0,0,0,0,3,
      199,65,254,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0>>,
      [<<0,0,51,250,5,197,0,0,6,222,
      54,187,0,0,94,204,32,255,
      254,254,191,207,240,120,0,
      0,0,3,255,128,0,0,0,0,0,0,
      0,0,3,199,65,254,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0,0,0,0,0,0,0,
      0,0,0,0,0,0,0>>],
      true,[],nil,[],
      [{820,[

      {0,0}]},
      {821,[{0,0}

      ]},
      {822,[

      {0,0}]},
      {823,[{0,0}

      ]},
      {824,[

      {0,0}]},
      {825,[{0,0}

      ]},
      {826,[

      {0,0}]},
      {827,[{0,0}

      ]},
      {828,[

      {0,0}]},
      {829,[{0,0}

      ]},
      {830,[

      {0,0}]},
      {831,[{0,0}

      ]},
      {832,[

      {0,0}]},
      {833,[{0,0}

      ]},
      {834,[

      {0,0}]},
      {835,[{0,0}

      ]},
      {836,[

      {0,0}]},
      {837,[{0,0}

      ]},
      {838,[

      {0,0}]},
      {839,[{0,0}

      ]},
      {840,[

      {0,0}]},
      {841,[{0,0}

      ]},
      {842,[

      {0,0}]},
      {843,[{0,0}

      ]},
      {844,[

      {0,0}]},
      {845,[{0,0}

      ]},
      {846,[

      {0,0}]},
      {847,[{0,0}

      ]},
      {848,[

      {0,0}]},
      {849,[{0,0}

      ]},
      {850,[

      {0,0}]},
      {851,[{0,0}

      ]},
      {852,[

      {0,0}]},
      {891,[{0,0}

      ]},
      {892,[

      {0,0}]},
      {893,[{0,0}

      ]},
      {894,[

      {0,0}]},
      {895,[{0,0}

      ]},
      {896,[

      {0,0}]},
      {897,[{0,0}

      ]},
      {898,[

      {0,0}]},
      {899,[{0,0}

      ]},
      {900,[

      {0,0}]},
      {901,[{0,0}

      ]},
      {902,[

      {0,0}]},
      {903,[{0,0}

      ]},
      {904,[

      {0,0}]},
      {905,[{0,0}

      ]},
      {906,[

      {0,0}]},
      {907,[{0,0}

      ]},
      {908,[

      {0,0}]},
      {909,[{0,0}

      ]},
      {910,[

      {0,0}]},
      {911,[{0,0}

      ]},
      {912,[

      {0,0}]},
      {913,[{0,0}

      ]},
      {914,[

      {0,0}]},
      {915,[{0,0}

      ]},
      {916,[

      {0,0}]},
      {917,[{0,0}

      ]},
      {918,[

      {0,0}]},
      {919,[{0,0}

      ]},
      {920,[

      {0,0}]},
      {921,[{0,0}

      ]},
      {922,[

      {0,0}]},
      {923,[{0,0}

      ]},
      {924,[

      {0,0}]},
      {925,[{0,0}

      ]},
      {926,[

      {0,0}]},
      {927,[{0,0}

      ]},
      {928,[

      {0,0}]},
      {929,[{0,0}

      ]},
      {930,[

      {0,0}]},
      {931,[{0,0}

      ]},
      {932,[

      {0,0}]},
      {933,[{0,0}

      ]},
      {934,[

      {0,0}]},
      {935,[{0,0}

      ]},
      {936,[

      {0,0}]},
      {937,[{0,0}

      ]},
      {968,[

      {0,0}]},
      {969,[{0,0}

      ]},
      {970,[

      {0,0}]},
      {971,[{0,0}

      ]},
      {972,[

      {0,0}]},
      {973,[{0,0}

      ]},
      {974,[

      {0,0}]},
      {975,[{0,0}

      ]},
      {976,[

      {0,0}]},
      {977,[{0,0}

      ]},
      {978,[

      {0,0}]},
      {979,[{0,0}

      ]},
      {980,[

      {0,0}]},
      {981,[{0,0}

      ]},
      {982,[

      {0,0}]},
      {983,[{0,0}

      ]},
      {984,[

      {0,0}]},
      {985,[{0,0}

      ]},
      {986,[

      {0,0}]},
      {987,[{0,0}

      ]},
      {988,[

      {0,0}]},
      {989,[{0,0}

      ]},
      {990,[

      {0,0}]},
      {991,[{0,0}

      ]},
      {992,[

      {0,0}]},
      {993,[{0,0}

      ]},
      {994,[

      {0,0}]},
      {995,[{0,0}

      ]},
      {996,[

      {0,0}]},
      {997,[{0,0}

      ]},
      {998,[

      {0,0}]},
      {999,[{0,0}

      ]},
      {1000,[

      {0,0}]},
      {1001,[{0,0}

      ]},
      {1002,[

      {0,0}]},
      {1003,[{0,0}

      ]},
      {1004,[

      {0,0}]},
      {1005,[{0,0}

      ]},
      {1006,[

      {0,0}]},
      {1007,[{0,0}

      ]},
      {1008,[

      {0,0}]},
      {1014,[{0,0}

      ]},
      {1015,[

      {0,0}]},
      {1016,[{0,0}

      ]},
      {1017,[

      {206630118054190,0}

      ]},
      {1018,[

      {58851646531965,0}

      ]},
      {1019,
      [

      {118920551874785,98345}

      ,

      {103801475651071,0}

      ]},
      {1020,[

      {0,0}

      ]},
      {1021,[

      {14726742042937,0}

      ]},
      {1022,
      [

      {210477493706492,98367}

      ,

      {85820590538866,0}

      ]},
      {1023,
      [

      {92085762404913,98351}

      ,

      {277016319418037,0}

      ]}]},
      main,nil,<16800.28934.86>,nil,
      "/ssd2/@indexes/bucket-1/main_b998c29cc4af889cf9c05b1808926d61.view.1",
      mapreduce_view,".view",prod,
      couch_set_view_stats_prod,
      872062976,<16800.28949.86>},
      nil,false,not_running,nil,nil,
      nil,0,[],nil,false,undefined,
      true,true,
      [171,172,173,174,175,176,177,
      178,179,180,181,182,183,184,
      185,186,187,188,189,190,191,
      192,193,194,195,196,197,198,
      199,200,201,202,203,204,205,
      206,207,208,209,210,211,212,
      213,214,215,216,217,218,219,
      220,221,222,223,224,225,226,
      227,228,229,230,231,232,233,
      234,235,236,237,238,239,240,
      241,242,243,244,245,246,247,
      248,249,250,251,252,253,254,
      255,427,428,429,430,431,432,
      433,434,435,436,437,438,439,
      440,441,442,443,444,445,446,
      447,448,449,450,451,452,453,
      454,455,456,457,458,459,460,
      461,462,463,464,465,466,467,
      468,469,470,471,472,473,474,
      475,476,477,478,479,480,481,
      482,483,484,485,486,487,488,
      489,490,491,492,493,494,495,
      496,497,498,499,500,501,502,
      503,504,505,506,507,508,509,
      510,511,682,683,684,685,686,
      687,688,689,690,691,692,693,
      694,695,696,697,698,699,700,
      701,702,703,704,705,706,707,
      708,709,710,711,712,713,714,
      715,716,717,718,719,720,721,
      722,723,724,725,726,727,728,
      729,730,731,732,733,734,735,
      736,737,738,739,740,741,742,
      743,744,745,746,747,748,749,
      750,751,752,753,754,755,756,
      757,758,759,760,761,762,763,
      764,765,766,767],
      [],
      {dict,0,16,16,8,80,48,

      {[],[],[],[],[],[],[],[],[],[], [],[],[],[],[],[]}

      ,
      {{[],[],[],[],[],[],[],[],[],
      [],[],[],[],[],[],[]}}},
      nil,3000}]},

      {gen_server,handle_msg,5},
      {proc_lib,init_p_do_apply,3}]},
      {gen_server,call,
      [<16800.28929.86>,
      {monitor_partition_update,891, #Ref<16800.0.695.30261>, <16800.10511.87>},
      infinity]}}}},
      [{capi_set_view_manager,handle_call,3},
      {gen_server,handle_msg,5}

      ,

      {gen_server,init_it,6}

      ,

      {proc_lib,init_p_do_apply,3}

      ]},
      {gen_server,call,
      ['capi_set_view_manager-bucket-1',

      {wait_index_updated,891}

      ,
      infinity]}},
      {gen_server,call,
      [

      {'janitor_agent-bucket-1', 'ns_1@172.23.96.18'}

      ,
      {if_rebalance,<0.15317.163>,
      {wait_checkpoint_persisted,892,2204}},
      infinity]}}}}]}}}

      Attachments

        Issue Links

          No reviews matched the request. Check your Options in the drop-down menu of this sections header.

          Activity

            People

              pavelpaulau Pavel Paulau (Inactive)
              pavelpaulau Pavel Paulau (Inactive)
              Votes:
              0 Vote for this issue
              Watchers:
              0 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Gerrit Reviews

                  There are no open Gerrit changes

                  PagerDuty