Details
-
Bug
-
Resolution: Not a Bug
-
Critical
-
Columnar 1.0.0
-
Columnar Edition 1.0.0 build 2144
-
Untriaged
-
0
-
Unknown
-
Analytics Sprint 44
Description
Seen on 006 -
2024-06-14T06:15:02.037+00:00 FATA CBAS.nc.CcConnection [main] Registering with ClusterControllerRemoteProxy [svc-da-node-002.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com/10.0.3.211:9112] failed with exception |
org.apache.hyracks.api.exceptions.HyracksException: HYR0010: Node 14fece7d2954fdb2b6598304737bf1f9 does not exist
|
at org.apache.hyracks.api.exceptions.HyracksException.create(HyracksException.java:58) ~[hyracks-api.jar:1.0.0-2144] |
at org.apache.hyracks.control.cc.cluster.NodeManager.addNode(NodeManager.java:106) ~[hyracks-control-cc.jar:1.0.0-2144] |
at org.apache.hyracks.control.cc.work.RegisterNodeWork.doRun(RegisterNodeWork.java:67) ~[hyracks-control-cc.jar:1.0.0-2144] |
at org.apache.hyracks.control.common.work.SynchronizableWork.run(SynchronizableWork.java:43) ~[hyracks-control-common.jar:1.0.0-2144] |
at org.apache.hyracks.control.common.work.WorkQueue$WorkerThread.run(WorkQueue.java:127) ~[hyracks-control-common.jar:1.0.0-2144] |
2024-06-14T06:15:02.042+00:00 FATA CBAS.util.ExitUtil [main] JVM halting with status 15 (halting thread Thread[main,5,main], interrupted false) |
2024-06-14T06:15:02.110+00:00 FATA CBAS.util.ExitUtil [pool-2-thread-1] Thread dump at halt: |
"main" [tid=1 state=TIMED_WAITING lock=java.util.concurrent.FutureTask@4a9a824a] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) |
at java.base@17.0.11/java.util.concurrent.FutureTask.awaitDone(FutureTask.java:444) |
at java.base@17.0.11/java.util.concurrent.FutureTask.get(FutureTask.java:203) |
at app//org.apache.hyracks.util.ExitUtil.halt(ExitUtil.java:117) |
- <locked java.lang.Class@7e74a380> |
at app//org.apache.hyracks.util.ExitUtil.halt(ExitUtil.java:104) |
at app//org.apache.hyracks.control.nc.CcConnection.registerNode(CcConnection.java:80) |
- <locked org.apache.hyracks.control.nc.CcConnection@43d3ec7f> |
at app//org.apache.hyracks.control.nc.NodeControllerService.registerNode(NodeControllerService.java:417) |
at app//org.apache.hyracks.control.nc.NodeControllerService.addCc(NodeControllerService.java:362) |
- <locked java.lang.Object@288b8da8> |
at app//org.apache.hyracks.control.nc.NodeControllerService.start(NodeControllerService.java:300) |
at app//com.couchbase.analytics.control.AnalyticsDriver.startService(AnalyticsDriver.java:142) |
at app//com.couchbase.analytics.control.AnalyticsDriver.startServices(AnalyticsDriver.java:123) |
at app//com.couchbase.analytics.control.AnalyticsDriver.main(AnalyticsDriver.java:100) |
at app//com.couchbase.columnar.ColumnarDriver.main(ColumnarDriver.java:10) |
|
"Reference Handler" [tid=2 state=RUNNABLE] |
at java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method) |
at java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253) |
at java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215) |
|
"Finalizer" [tid=3 state=WAITING lock=java.lang.ref.ReferenceQueue$Lock@43bd511] |
at java.base@17.0.11/java.lang.Object.wait(Native Method) |
at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) |
at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) |
at java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172) |
|
"Signal Dispatcher" [tid=4 state=RUNNABLE] |
|
"Common-Cleaner" [tid=12 state=TIMED_WAITING lock=java.lang.ref.ReferenceQueue$Lock@58d761fd] |
at java.base@17.0.11/java.lang.Object.wait(Native Method) |
at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) |
at java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140) |
at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) |
at java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162) |
|
"Notification Thread" [tid=13 state=RUNNABLE] |
|
"Log4j2-AsyncAppenderEventDispatcher-1-AsyncInternalAccessLog" [tid=14 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2cc55182] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-2-AsyncDcpDebugLog" [tid=15 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@32d30e82] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-3-AsyncAccessLog" [tid=16 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@681ad694] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-4-AsyncOptimizerLog" [tid=17 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@ba19ae3] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-5-AsyncDebugLog" [tid=18 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@dee8354] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-6-AsyncInternalAccessLog" [tid=20 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@27632b75] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-7-AsyncDcpDebugLog" [tid=21 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@19f3e77b] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
|
"Log4j2-AsyncAppenderEventDispatcher-8-AsyncAccessLog" [tid=22 state=WAITING lock=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1e020024] |
at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) |
at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) |
at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) |
at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) |
at app//com.conversantmedia.util.concurrent.AbstractCondition.await(AbstractCondition.java:57) |
at app//com.conversantmedia.util.concurrent.DisruptorBlockingQueue.take(DisruptorBlockingQueue.java:240) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.dispatchAll(AsyncAppenderEventDispatcher.java:81) |
at app//org.apache.logging.log4j.core.appender.AsyncAppenderEventDispatcher.run(AsyncAppenderEventDispatcher.java:73) |
At the same time, I can see some certificate errors on node 002 -
2024-06-14T06:15:02.472+00:00 WARN CBAS.server.HttpServerHandler [nioEventLoopGroup-3-6] Failure handling HTTP Request |
io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate
|
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:499) ~[netty-codec-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.101.Final.jar:4.1.101.Final] |
at java.base/java.lang.Thread.run(Thread.java:840) [?:?] |
Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: bad_certificate
|
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?] |
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?] |
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:365) ~[?:?] |
at java.base/sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293) ~[?:?] |
at java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:204) ~[?:?] |
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:172) ~[?:?] |
at java.base/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736) ~[?:?] |
at java.base/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691) ~[?:?] |
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506) ~[?:?] |
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482) ~[?:?] |
at java.base/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679) ~[?:?] |
at io.netty.handler.ssl.JdkSslEngine.unwrap(JdkSslEngine.java:92) ~[netty-handler-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.ssl.JdkAlpnSslEngine.unwrap(JdkAlpnSslEngine.java:163) ~[netty-handler-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:310) ~[netty-handler-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1445) ~[netty-handler-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1338) ~[netty-handler-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1387) ~[netty-handler-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.101.Final.jar:4.1.101.Final] |
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.101.Final.jar:4.1.101.Final] |
... 17 more |
Not sure if they are connection issues or if there's some other reason.
Cluster was in the state of rebalancing( nodes changed from 8 to 4) at the time of log collection, hence you see 5 nodes.
cbcollect ->
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T061151-ns_1%40svc-da-node-001.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T061151-ns_1%40svc-da-node-002.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T061151-ns_1%40svc-da-node-004.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T061151-ns_1%40svc-da-node-006.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T061151-ns_1%40svc-da-node-008.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
cbcollect (older) ->
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-001.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-002.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-003.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-004.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-005.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-006.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-007.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip
https://cb-engineering.s3.amazonaws.com/SysTestColumnar13Jun/collectinfo-2024-06-14T050309-ns_1%40svc-da-node-008.smlgzhhxmxj2aadc.sandbox.nonprod-project-avengers.com.zip