[system test] Build index request timesout

1. setup the cluster, create indexes
2. One index ITEM(IT_ID) is never built,
3. After loading items on all the buckets – ITEM with 45K items
I issue "build index command on the pending index.

The build index command errors out with a timeout

cbq> build index on ITEM(IT_ID) using gsi ;
{
"requestID": "e9467596-a252-4e8b-84bf-34af27466b44",
"signature": null,
"results": [
],
"errors": [

{ "code": 5000, "msg": "BuildIndexes - cause: Request Timeout" }

],
"status": "errors",
"metrics":

{ "elapsedTime": "30.889091678s", "executionTime": "30.888380258s", "resultCount": 0, "resultSize": 0, "errorCount": 1 }

}

cbq> select * from system:indexes where state="pending";
{
"requestID": "347d86fd-7bc9-4d1f-b63c-44a3016b427a",
"signature":

{ "*": "*" }

,
"results": [
{
"indexes":

{ "datastore_id": "http://127.0.0.1:8091", "id": "2e02dd2cd6d57501", "index_key": [ "`I_ID`" ], "keyspace_id": "ITEM", "name": "IT_ID", "namespace_id": "default", "state": "pending", "using": "gsi" }

}
],
"status": "success",
"metrics":

{ "elapsedTime": "9.243727226s", "executionTime": "9.242942706s", "resultCount": 1, "resultSize": 410 }

}

attaching logs.

curl -uAdministrator:password 10.6.2.164:9102/getIndexStatus | jq '.'
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 4717 0 4717 0 0 14521 0 -::- -::- -::- 14558
{
"code": "success",
"status": [

{ "defnId": 12491595595400575000, "name": "seq_num", "bucket": "default", "secExprs": [ "`SequenceNumber`" ], "status": "Ready", "definition": "CREATE INDEX seq_num ON default(`SequenceNumber`) USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 13899584551521161000, "name": "px_history", "bucket": "HISTORY", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX px_history ON HISTORY USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 14290696383568634000, "name": "OR_O_ID_D_ID_W_ID", "bucket": "ORDERS", "secExprs": [ "`O_ID`", "`O_D_ID`", "`O_W_ID`", "`O_C_ID`" ], "status": "Ready", "definition": "CREATE INDEX OR_O_ID_D_ID_W_ID ON ORDERS(`O_ID`,`O_D_ID`,`O_W_ID`,`O_C_ID`) USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 15069888705414525000, "name": "CU_ID_D_ID_W_ID", "bucket": "CUSTOMER", "secExprs": [ "`C_ID`", "`C_D_ID`", "`C_W_ID`" ], "status": "Ready", "definition": "CREATE INDEX CU_ID_D_ID_W_ID ON CUSTOMER(`C_ID`,`C_D_ID`,`C_W_ID`) USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 15846627507486525000, "name": "PX_DISTRICT", "bucket": "DISTRICT", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_DISTRICT ON DISTRICT USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 3315455460350719500, "name": "IT_ID", "bucket": "ITEM", "secExprs": [ "`I_ID`" ], "status": "Created", "definition": "CREATE INDEX IT_ID ON ITEM(`I_ID`) USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 4332266952092018700, "name": "NO_D_ID_W_ID", "bucket": "NEW_ORDER", "secExprs": [ "`NO_O_ID`", "`NO_D_ID`", "`NO_W_ID`" ], "status": "Ready", "definition": "CREATE INDEX NO_D_ID_W_ID ON NEW_ORDER(`NO_O_ID`,`NO_D_ID`,`NO_W_ID`) USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 5385351889856855000, "name": "PX_WAREHOUSE", "bucket": "WAREHOUSE", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_WAREHOUSE ON WAREHOUSE USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 6069981027852458000, "name": "PX_STOCK", "bucket": "STOCK", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_STOCK ON STOCK USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 6088473239565867000, "name": "PX_ORDER_LINE", "bucket": "ORDER_LINE", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_ORDER_LINE ON ORDER_LINE USING GSI", "hosts": [ "10.6.2.164:8091" ] }

,

{ "defnId": 10166878466801975000, "name": "px_default", "bucket": "default", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX px_default ON default USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 14630686226012383000, "name": "WH_ID", "bucket": "WAREHOUSE", "secExprs": [ "`W_ID`" ], "status": "Ready", "definition": "CREATE INDEX WH_ID ON WAREHOUSE(`W_ID`) USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 15493462150554866000, "name": "PX_NEW_ORDER", "bucket": "NEW_ORDER", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_NEW_ORDER ON NEW_ORDER USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 16256725172173674000, "name": "DI_ID_W_ID", "bucket": "DISTRICT", "secExprs": [ "`D_ID`", "`D_W_ID`" ], "status": "Ready", "definition": "CREATE INDEX DI_ID_W_ID ON DISTRICT(`D_ID`,`D_W_ID`) USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 18067059215546972000, "name": "ST_W_ID_I_ID1", "bucket": "STOCK", "secExprs": [ "`S_W_ID`", "`S_I_ID`" ], "status": "Ready", "definition": "CREATE INDEX ST_W_ID_I_ID1 ON STOCK(`S_W_ID`,`S_I_ID`) USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 18266280185197386000, "name": "PX_ITEM", "bucket": "ITEM", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_ITEM ON ITEM USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 5055181566269432000, "name": "OL_O_ID_D_ID_W_ID", "bucket": "ORDER_LINE", "secExprs": [ "`OL_O_ID`", "`OL_D_ID`", "`OL_W_ID`" ], "status": "Ready", "definition": "CREATE INDEX OL_O_ID_D_ID_W_ID ON ORDER_LINE(`OL_O_ID`,`OL_D_ID`,`OL_W_ID`) USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 6058370070843462000, "name": "OR_W_ID_D_ID_C_ID", "bucket": "ORDERS", "secExprs": [ "`O_W_ID`", "`O_D_ID`", "`O_C_ID`" ], "status": "Ready", "definition": "CREATE INDEX OR_W_ID_D_ID_C_ID ON ORDERS(`O_W_ID`,`O_D_ID`,`O_C_ID`) USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 7365785464875836000, "name": "CU_W_ID_D_ID_LAST", "bucket": "CUSTOMER", "secExprs": [ "`C_W_ID`", "`C_D_ID`", "`C_LAST`" ], "status": "Ready", "definition": "CREATE INDEX CU_W_ID_D_ID_LAST ON CUSTOMER(`C_W_ID`,`C_D_ID`,`C_LAST`) USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 7460001598182829000, "name": "PX_CUSTOMER", "bucket": "CUSTOMER", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_CUSTOMER ON CUSTOMER USING GSI", "hosts": [ "10.6.2.167:8091" ] }

,

{ "defnId": 9098228072179851000, "name": "PX_ORDERS", "bucket": "ORDERS", "isPrimary": true, "status": "Ready", "definition": "CREATE PRIMARY INDEX PX_ORDERS ON ORDERS USING GSI", "hosts": [ "10.6.2.167:8091" ] }

]
}