Details
-
Bug
-
Resolution: Unresolved
-
Minor
-
Columnar 1.0.0
-
1.0.0-2203
-
Untriaged
-
0
-
Unknown
Description
During a scale up operation (8 nodes to 16 nodes), it was seen that a second rebalance was triggered despite a successful first rebalance. A similar issue was observed here and here. It was supposed to have been fixed in 1.0.0-2189. While this had no functional impact, it could be worth investigating to see why this happened.
Rebalance 1 was successfully completed at 2024-07-10T07:33:35.060.
{"stageInfo":{"analytics":{"totalProgress":100,"perNodeProgress":{"ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1},"startTime":"2024-07-10T07:25:53.534Z","completedTime":"2024-07-10T07:33:35.034Z","timeTaken":461500},"data":{"totalProgress":100,"perNodeProgress":{"ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1},"startTime":"2024-07-10T07:25:53.311Z","completedTime":"2024-07-10T07:25:53.534Z","timeTaken":222}},"rebalanceId":"862f84e959a79a5aa35326a501c56b36","nodesInfo":{"active_nodes":["ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com"],"keep_nodes":["ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com"],"eject_nodes":[],"delta_nodes":[],"failed_nodes":[]},"masterNode":"ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","startTime":"2024-07-10T07:25:53.309Z","completedTime":"2024-07-10T07:33:35.060Z","timeTaken":461751,"completionMessage":"Rebalance completed successfully."} |
Rebalance 2 gets triggered at 2024-07-10T07:33:38.541Z.
{"stageInfo":{"analytics":{"totalProgress":100,"perNodeProgress":{"ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1},"startTime":"2024-07-10T07:33:38.541Z","completedTime":"2024-07-10T07:33:39.664Z","timeTaken":1123},"data":{"totalProgress":100,"perNodeProgress":{"ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1,"ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com":1},"startTime":"2024-07-10T07:33:38.383Z","completedTime":"2024-07-10T07:33:38.541Z","timeTaken":157}},"rebalanceId":"1d99e56f021785b509b079681a824d05","nodesInfo":{"active_nodes":["ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com"],"keep_nodes":["ns_1@svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","ns_1@svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com"],"eject_nodes":[],"delta_nodes":[],"failed_nodes":[]},"masterNode":"ns_1@svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com","startTime":"2024-07-10T07:33:38.381Z","completedTime":"2024-07-10T07:33:39.691Z","timeTaken":1309,"completionMessage":"Rebalance completed successfully."} |
cbcollect ->
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-001.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-002.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-003.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-004.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-005.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-006.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-007.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-008.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-009.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-010.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-011.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-012.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-013.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-014.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-015.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnarJul9/collectinfo-2024-07-10T075916-ns_1%40svc-da-node-016.adewm3olqtoa4vfw.sandbox.nonprod-project-avengers.com.zip