[BP to MH - MB-37345] - [System Test]: Pausing failed with ERR_REBALANCE_ONGOING

[2019-12-24T03:34:48-08:00, sequoiatools/eventing:6.5:f2c80b] /eventing.py 172.23.98.135 8096 mad-hatter/bucket_op_curl_integration.json Administrator password pause true

→ 

Error occurred on container - sequoiatools/eventing:6.5:[/eventing.py 172.23.98.135 8096 mad-hatter/bucket_op_curl_integration.json Administrator password pause true]

docker logs f2c80b

docker start f2c80b

#Traceback (most recent call last):

-  File "/eventing.py", line 371, in <module>

    EventingOperations().run()

'  File "/eventing.py", line 29, in run

I    response = self.perform_eventing_lifecycle_operation(app_definition)

H  File "/eventing.py", line 82, in perform_eventing_lifecycle_operation

4    raise Exception("Failed to deploy application")

(Exception: Failed to deploy application

App definition :

	{u'appcode': u'function OnUpdate(doc, meta) {\n    dst_bucket[meta.id]="curl bucket op";\n    var request = {\n\tpath : \'job/test_suite_executor/api/json?tree=jobs[component]\'\n    };\n    if (meta.id.includes("_")){\n        var id = meta.id.split("_")[1];\n        if(parseInt(id)%5==0){\n        while(true){\n            try {\n            \tvar response = curl("GET", server, request);\n            \tlog(\'response body received from server:\', response.body);\n            \tlog(\'response headers received from server:\', response.headers);\n            \tlog(\'response status received from server:\', response.status);\n            \tvar res= new Uint8Array(response.body);\n            \tif(response.status == 200){\n            \t         \tbreak; \n                \t    }\n                }catch (e) {\n        \t      log(\'error:\', e);\n                }\n            }\n        }\n    }\n}\n\n\n\nfunction OnDelete(meta) {\n    delete dst_bucket[meta.id];\n    var request = {\n\tpath : \'job/test_suite_executor/api/json?tree=jobs[component]\'\n    };\n    if (meta.id.includes("_")){\n        var id = meta.id.split("_")[1];\n        if(parseInt(id)%5==0){\n        while(true){\n            try {\n            \tvar response = curl("GET", server, request);\n            \tlog(\'response body received from server:\', response.body);\n            \tlog(\'response headers received from server:\', response.headers);\n            \tlog(\'response status received from server:\', response.status);\n            \tvar res= new Uint8Array(response.body);\n            \tif(response.status == 200){\n            \t         \tbreak; \n                \t    }\n                }catch (e) {\n        \t      log(\'error:\', e);\n                }\n            }\n        }\n    }\n}', u'appname': u'bucket_op_curl', u'settings': {u'language_compatibility': u'6.5.0', u'execution_timeout': 20, u'log_level': u'INFO', u'processing_status': False, u'dcp_stream_boundary': u'from_now', u'deadline_timeout': 62, u'deployment_status': False}, u'depcfg': {u'buckets': [{u'access': u'rw', u'alias': u'dst_bucket', u'bucket_name': u'ITEM'}], u'source_bucket': u'default', u'curl': [{u'auth_type': u'no-auth', u'username': u'', u'bearer_key': u'', u'hostname': u'http://qa.sc.couchbase.com/', u'validate_ssl_certificate': False, u'value': u'server', u'allow_cookies': False, u'password': u''}], u'metadata_bucket': u'NEW_ORDER'}}

{

" "name": "ERR_REBALANCE_ONGOING",

 "code": 36,

 "description": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed",

 "attributes": null,

 "runtime_info": {

  "code": 36,

  "info": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed"

 }

z} {'date': 'Tue, 24 Dec 2019 11:34:52 GMT', 'status': '406', 'content-length': '406', 'content-type': 'application/json'}

False {

" "name": "ERR_REBALANCE_ONGOING",

 "code": 36,

 "description": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed",

 "attributes": null,

 "runtime_info": {

  "code": 36,

  "info": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed"

 }

z} {'date': 'Tue, 24 Dec 2019 11:34:52 GMT', 'status': '406', 'content-length': '406', 'content-type': 'application/json'}

[2019-12-24T03:35:01-08:00, sequoiatools/eventing:6.5:b52c9d] /eventing.py 172.23.98.135 8096 mad-hatter/timer_op_n1ql.json Administrator password pause true

→ 

Error occurred on container - sequoiatools/eventing:6.5:[/eventing.py 172.23.98.135 8096 mad-hatter/timer_op_n1ql.json Administrator password pause true]

docker logs b52c9d

docker start b52c9d

#Traceback (most recent call last):

App definition :

|{u'using_timer': True, u'appname': u'timer_op', u'settings': {u'language_compatibility': u'6.5.0', u'execution_timeout': 60, u'log_level': u'INFO', u'processing_status': False, u'dcp_stream_boundary': u'from_now', u'deadline_timeout': 62, u'deployment_status': False}, u'appcode': u'function OnUpdate(doc, meta) {\n    var expiry = new Date();\n    expiry.setSeconds(expiry.getSeconds() + 30);\n    if (meta.id.includes("_")){\n        var id = meta.id.split("_")[1];\n        if(parseInt(id)%5==0){\n            var context = {docID : meta.id, random_text : \'e6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0R7Aumoe6cZZGHuh0\'};\n            createTimer(timerCallback,  expiry, meta.id, context);\n        }\n    }\n}\nfunction OnDelete(meta) {\n    var expiry = new Date();\n    expiry.setSeconds(expiry.getSeconds() + 30);\n    if (meta.id.includes("_")){\n        var id = meta.id.split("_")[1];\n        if(parseInt(id)%5==0){\n            var context = {docID : meta.id};\n            createTimer(NDtimerCallback,  expiry, meta.id, context);\n        }\n    }\n}\nfunction NDtimerCallback(context) {\n    var select_query = SELECT * FROM default USE KEYS[$context.docID];\n    for (var r of select_query) {\n    }\n    log("deleting: ",context.docID);\n    delete dst_bucket[context.docID];\n}\nfunction timerCallback(context) {\n    var select_query = SELECT * FROM default USE KEYS[$context.docID];\n    for (var r of select_query) {\n        log("creating: ",context.docID);\n        dst_bucket[context.docID] = \'from timerCallback\';\n    }\n}\n', u'src_mutation': False, u'depcfg': {u'buckets': [{u'access': u'rw', u'alias': u'dst_bucket', u'bucket_name': u'WAREHOUSE'}], u'source_bucket': u'default', u'curl': [], u'metadata_bucket': u'NEW_ORDER'}}

{

" "name": "ERR_REBALANCE_ONGOING",

 "code": 36,

 "description": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed",

 "attributes": null,

 "runtime_info": {

  "code": 36,

  "info": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed"

 }

z} {'date': 'Tue, 24 Dec 2019 11:35:05 GMT', 'status': '406', 'content-length': '406', 'content-type': 'application/json'}

False {

" "name": "ERR_REBALANCE_ONGOING",

 "code": 36,

 "description": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed",

 "attributes": null,

 "runtime_info": {

  "code": 36,

  "info": "Rebalance ongoing on some/all Eventing nodes, creating new functions, deployment or undeployment of existing functions is not allowed"

 }

z} {'date': 'Tue, 24 Dec 2019 11:35:05 GMT', 'status': '406', 'content-length': '406', 'content-type': 'application/json'}

-  File "/eventing.py", line 371, in <module>

    EventingOperations().run()

'  File "/eventing.py", line 29, in run

I    response = self.perform_eventing_lifecycle_operation(app_definition)

H  File "/eventing.py", line 82, in perform_eventing_lifecycle_operation

4    raise Exception("Failed to deploy application")

(Exception: Failed to deploy application 

[2019-12-24T03:18:01-08:00, sequoiatools/couchbase-cli:6.5:2e2286] server-add -c 172.23.108.103:8091 --server-add https://172.23.97.239 -u Administrator -p password --server-add-username Administrator --server-add-password password --services analytics

[2019-12-24T03:18:16-08:00, sequoiatools/couchbase-cli:6.5:f351d1] rebalance -c 172.23.108.103:8091 --server-remove 172.23.104.69 -u Administrator -p password

warning using 'json' filter:  unexpected end of JSON input []

[2019-12-24T03:24:30-08:00, sequoiatools/cmd:6c785c] 60 

[2019-12-24T07:46:21-08:00, sequoiatools/eventing:6.5:973910] /eventing.py 172.23.98.135 8096 mad-hatter/bucket_op_curl_integration.json Administrator password resume true

[2019-12-24T07:47:04-08:00, sequoiatools/eventing:6.5:da0b96] /eventing.py 172.23.98.135 8096 mad-hatter/timer_op_n1ql.json Administrator password resume true