Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.1.0
-
Untriaged
-
1
-
Unknown
-
Tools 2022-Jan
Description
7.1.0-2097
Test:
-test tests/integration/neo/test_neo_couchstore_milestone4.yml -scope tests/integration/neo/scope_couchstore.yml
Scale 3
Iteration 3
merge task failed with error:
{
|
"task_name": "merge",
|
"status": "failed",
|
"start": "2022-01-21T12:00:04.878600571-08:00",
|
"end": "2022-01-21T12:00:14.489103792-08:00",
|
"node_runs": [
|
{
|
"node_id": "34e6902271c90b80be588f7ac0ed0ec0",
|
"status": "failed",
|
"start": "2022-01-21T12:00:04.974299181-08:00",
|
"end": "2022-01-21T12:00:14.32516012-08:00",
|
"error": "exit status 2",
|
"progress": 60.11610519834638,
|
"stats": {
|
"id": "1a5e9f36-522d-451c-8f75-2949a49d18b2",
|
"current_transfer": 4,
|
"total_transfers": 5,
|
"transfers": [
|
{
|
"description": "(1/5) Merging backup 2022-01-20T06_00_17.418670777-08_00",
|
"stats": {
|
"started_at": 1642795205050365700,
|
"finished_at": 1642795208106192100,
|
"buckets": {
|
"N1QL_SYSTEM_BUCKET": {
|
"total_items": 282,
|
"started_at": 1642795206423106300,
|
"finished_at": 1642795206438655200,
|
"complete": true
|
},
|
"bucket1": {
|
"total_items": 3389095,
|
"started_at": 1642795207269386500,
|
"finished_at": 1642795207279254800,
|
"complete": true
|
},
|
"bucket2": {
|
"total_items": 7567672,
|
"started_at": 1642795208089988400,
|
"finished_at": 1642795208101586200,
|
"complete": true
|
},
|
"bucket3": {
|
"total_items": 4819892,
|
"started_at": 1642795207526243800,
|
"finished_at": 1642795207542487800,
|
"complete": true
|
},
|
"bucket4": {
|
"total_items": 1681440,
|
"started_at": 1642795206714425000,
|
"finished_at": 1642795206753977600,
|
"complete": true
|
},
|
"bucket5": {
|
"total_items": 1675457,
|
"started_at": 1642795205716358000,
|
"finished_at": 1642795205743187700,
|
"complete": true
|
},
|
"bucket6": {
|
"total_items": 1719725,
|
"started_at": 1642795205953569300,
|
"finished_at": 1642795205987285800,
|
"complete": true
|
},
|
"bucket7": {
|
"total_items": 480872,
|
"started_at": 1642795207805383200,
|
"finished_at": 1642795207842094300,
|
"complete": true
|
},
|
"bucket8": {
|
"started_at": 1642795205410075000,
|
"finished_at": 1642795205431282000,
|
"complete": true
|
},
|
"bucket9": {
|
"started_at": 1642795206183521000,
|
"finished_at": 1642795206191153400,
|
"complete": true
|
},
|
"default": {
|
"total_items": 8162601,
|
"started_at": 1642795207003046100,
|
"finished_at": 1642795207016598300,
|
"complete": true
|
}
|
},
|
"complete": true
|
},
|
"progress": 100,
|
"eta": "2022-01-21T12:00:08.111504916-08:00"
|
},
|
{
|
"description": "(2/5) Merging backup 2022-01-20T12_03_21.656783634-08_00",
|
"stats": {
|
"started_at": 1642795208113725000,
|
"finished_at": 1642795210455183600,
|
"buckets": {
|
"N1QL_SYSTEM_BUCKET": {
|
"total_items": 466,
|
"started_at": 1642795210438900500,
|
"finished_at": 1642795210451620900,
|
"complete": true
|
},
|
"bucket1": {
|
"total_items": 4832790,
|
"started_at": 1642795210055779000,
|
"finished_at": 1642795210068014600,
|
"complete": true
|
},
|
"bucket2": {
|
"total_items": 5390619,
|
"started_at": 1642795209211276500,
|
"finished_at": 1642795209227260700,
|
"complete": true
|
},
|
"bucket3": {
|
"total_items": 3350291,
|
"started_at": 1642795210243730200,
|
"finished_at": 1642795210254633700,
|
"complete": true
|
},
|
"bucket4": {
|
"total_items": 1834673,
|
"started_at": 1642795209469433300,
|
"finished_at": 1642795209503023400,
|
"complete": true
|
},
|
"bucket5": {
|
"total_items": 1834673,
|
"started_at": 1642795208345840000,
|
"finished_at": 1642795208381837600,
|
"complete": true
|
},
|
"bucket6": {
|
"total_items": 1834673,
|
"started_at": 1642795208589927200,
|
"finished_at": 1642795208627360300,
|
"complete": true
|
},
|
"bucket7": {
|
"total_items": 656702,
|
"started_at": 1642795208807754200,
|
"finished_at": 1642795208845987600,
|
"complete": true
|
},
|
"bucket8": {
|
"started_at": 1642795209013285000,
|
"finished_at": 1642795209028571600,
|
"complete": true
|
},
|
"bucket9": {
|
"started_at": 1642795209678304800,
|
"finished_at": 1642795209694304300,
|
"complete": true
|
},
|
"default": {
|
"total_items": 15920936,
|
"started_at": 1642795209870977500,
|
"finished_at": 1642795209881298400,
|
"complete": true
|
}
|
},
|
"complete": true
|
},
|
"progress": 100,
|
"eta": "2022-01-21T12:00:10.46270384-08:00"
|
},
|
{
|
"description": "(3/5) Merging backup 2022-01-20T18_00_19.069142212-08_00",
|
"stats": {
|
"started_at": 1642795210468116500,
|
"finished_at": 1642795212994859800,
|
"buckets": {
|
"N1QL_SYSTEM_BUCKET": {
|
"total_items": 461,
|
"started_at": 1642795212375692000,
|
"finished_at": 1642795212385993000,
|
"complete": true
|
},
|
"bucket1": {
|
"total_items": 6919749,
|
"started_at": 1642795210929659000,
|
"finished_at": 1642795210950684200,
|
"complete": true
|
},
|
"bucket2": {
|
"total_items": 9050529,
|
"started_at": 1642795212203428600,
|
"finished_at": 1642795212215114800,
|
"complete": true
|
},
|
"bucket3": {
|
"total_items": 4390541,
|
"started_at": 1642795211794796300,
|
"finished_at": 1642795211807726600,
|
"complete": true
|
},
|
"bucket4": {
|
"total_items": 1750522,
|
"started_at": 1642795210732619300,
|
"finished_at": 1642795210774980600,
|
"complete": true
|
},
|
"bucket5": {
|
"total_items": 1662375,
|
"started_at": 1642795211216097800,
|
"finished_at": 1642795211266192400,
|
"complete": true
|
},
|
"bucket6": {
|
"total_items": 1660639,
|
"started_at": 1642795211558782000,
|
"finished_at": 1642795211591400400,
|
"complete": true
|
},
|
"bucket7": {
|
"total_items": 616652,
|
"started_at": 1642795212609011000,
|
"finished_at": 1642795212638817500,
|
"complete": true
|
},
|
"bucket8": {
|
"started_at": 1642795212812676600,
|
"finished_at": 1642795212823164700,
|
"complete": true
|
},
|
"bucket9": {
|
"started_at": 1642795211994522400,
|
"finished_at": 1642795212008612900,
|
"complete": true
|
},
|
"default": {
|
"total_items": 9387686,
|
"started_at": 1642795212980445200,
|
"finished_at": 1642795212990252000,
|
"complete": true
|
}
|
},
|
"complete": true
|
},
|
"progress": 100,
|
"eta": "2022-01-21T12:00:13.003821319-08:00"
|
},
|
{
|
"description": "(4/5) Merging backup 2022-01-21T00_00_22.380884795-08_00",
|
"stats": {
|
"started_at": 1642795213010470100,
|
"buckets": {
|
"bucket1": {
|
"total_items": 3310077,
|
"started_at": 1642795213569857500,
|
"finished_at": 1642795213582648000,
|
"complete": true
|
},
|
"bucket2": {
|
"total_items": 6620432,
|
"started_at": 1642795214099170600,
|
"finished_at": 1642795214110769200,
|
"complete": true
|
},
|
"bucket3": {
|
"total_items": 4611301,
|
"started_at": 1642795213764019700,
|
"finished_at": 1642795213775593200,
|
"complete": true
|
},
|
"bucket8": {
|
"started_at": 1642795213233090000,
|
"finished_at": 1642795213241357600,
|
"complete": true
|
},
|
"bucket9": {
|
"started_at": 1642795213922406700,
|
"finished_at": 1642795213932587000,
|
"complete": true
|
},
|
"default": {
|
"total_items": 8691197,
|
"started_at": 1642795213392571100,
|
"finished_at": 1642795213405520100,
|
"complete": true
|
}
|
}
|
},
|
"progress": 0.5805259917319026,
|
"eta": "2022-01-21T12:26:20.183533609-08:00"
|
}
|
],
|
"progress": 60.11610519834638,
|
"eta": "2022-01-21T12:00:17.183533609-08:00"
|
},
|
"error_code": 2
|
}
|
],
|
"error": "exit status 2",
|
"error_code": 2,
|
"type": "MERGE",
|
"show": true
|
}
|
Following panic observed in CBM logs:
panic: runtime error: index out of range [6] with length 6 [recovered]
|
panic: runtime error: index out of range [6] with length 6
|
|
|
goroutine 1745 [running]:
|
encoding/json.(*encodeState).marshal.func1()
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:328 +0x6e
|
panic({0x17096a0, 0xc000d7c870})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/runtime/panic.go:1038 +0x215
|
encoding/json.mapEncoder.encode({0xc002e67470}, 0xc002516380, {0x158db20, 0xc0021cd650, 0x6}, {0xa, 0x0})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:800 +0x706
|
encoding/json.structEncoder.encode({{{0xc0000df8c0, 0xc002e4a140, 0x0}, 0xc00018c150}}, 0xc002516380, {0x16bfa60, 0xc0021cd640, 0xc002e67608}, {0x0, 0x1})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:761 +0x1f4
|
encoding/json.ptrEncoder.encode({0x14fcc20}, 0xc002516380, {0x16d4600, 0xc002347990, 0x0}, {0x8, 0x0})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:945 +0x27e
|
encoding/json.structEncoder.encode({{{0xc0000df440, 0xc000040750, 0x30}, 0xc00018c1b0}}, 0xc002516380, {0x16bf2e0, 0xc002347980, 0xc002e55200}, {0x0, 0x1})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:761 +0x1f4
|
encoding/json.ptrEncoder.encode({0xb}, 0xc002516380, {0x14b13e0, 0xc0019ccbf8, 0x68}, {0x17, 0x0})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:945 +0x27e
|
encoding/json.arrayEncoder.encode({0x68}, 0xc002516380, {0x14d07e0, 0xc002e8c020, 0xc000040750}, {0x30, 0x0})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:916 +0xb5
|
encoding/json.sliceEncoder.encode({0x14fcc20}, 0xc002516380, {0x14d07e0, 0xc002e8c020, 0x2}, {0xc, 0x0})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:889 +0x2e6
|
encoding/json.structEncoder.encode({{{0xc0004a0900, 0xc000e5db98, 0x487809}, 0xc00018c210}}, 0xc002516380, {0x1717c60, 0xc002e8c000, 0xc000e5dbd8}, {0x0, 0x1})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:761 +0x1f4
|
encoding/json.(*encodeState).reflectValue(0xc000e5dc48, {0x1717c60, 0xc002e8c000, 0x0}, {0xe0, 0x1b})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:360 +0x78
|
encoding/json.(*encodeState).marshal(0xc002e8c000, {0x1717c60, 0xc002e8c000}, {0xd8, 0xdc})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:332 +0xfa
|
encoding/json.Marshal({0x1717c60, 0xc002e8c000})
|
/home/couchbase/.cbdepscache/exploded/x86_64/go-1.17.6/go/src/encoding/json/encode.go:161 +0x45
|
github.com/couchbase/backup/plan.(*ProgressJSON).displayProgress(0xc0005a4bb0, 0x0, 0x8c)
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/backup/plan/progress_json.go:209 +0x5ec
|
github.com/couchbase/backup/plan.updateHook.func1()
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/backup/plan/progress_reporter.go:105 +0x175
|
created by github.com/couchbase/backup/plan.updateHook
|
/home/couchbase/jenkins/workspace/couchbase-server-unix/backup/plan/progress_reporter.go:93 +0x12f
|
Cluster config:
########## Cluster config ##################
|
###### n1ql : 2 ===== > [172.23.104.137:8091 172.23.99.11:8091] ###########
|
###### index : 6 ===== > [172.23.104.155:8091 172.23.104.70:8091 172.23.120.245:8091 172.23.123.28:8091 172.23.96.251:8091 172.23.96.252:8091] ###########
|
###### cbas : 4 ===== > [172.23.104.157:8091 172.23.105.168:8091 172.23.120.107:8091 172.23.97.239:8091] ###########
|
###### fts : 2 ===== > [172.23.104.5:8091 172.23.105.111:8091] ###########
|
###### eventing : 3 ===== > [172.23.104.67:8091 172.23.96.148:8091 172.23.97.122:8091] ###########
|
###### backup : 1 ===== > [172.23.104.69:8091] ###########
|
###### kv : 10 ===== > [172.23.105.107:8091 172.23.108.103:8091 172.23.121.117:8091 172.23.96.253:8091 172.23.97.119:8091 172.23.97.121:8091 172.23.98.135:8091 172.23.99.20:8091 172.23.99.21:8091 172.23.99.25:8091] ###########
|
Attaching CBM logs