Thread Dump MTN

org.apache.ignite.internal.processors.failure.
FailureProcessor - Thread dump at

2023/08/21 09:16:58 UTC
Thread [name="stat-mgmt-#26543", id=128686, state=TIMED_WAITING, blockCnt=0,
waitCnt=1]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@621d7
d, ownerName=null, ownerId=-1]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Ab
stractQueuedSynchronizer.java:2078)
at
java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Thread [name="hconnection-0x4fef5792-shared-pool3-t84055", id=128685,

state=TIMED_WAITING, blockCnt=1, waitCnt=2]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@d12a7
da, ownerName=null, ownerId=-1]
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at
Thread [name="hconnection-0x5a455ca3-shared-pool6-t6677", id=128670,

Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@40ae8
7fb, ownerName=null, ownerId=-1]
at
at
at
at
at
Thread [name="phoenix-1-thread-3231", id=128669, state=TIMED_WAITING, blockCnt=1,

waitCnt=3]
Lock [object=java.lang.Object@69beb985, ownerName=null, ownerId=-1]
at java.lang.Object.wait(Native Method)
at
org.apache.phoenix.job.AbstractRoundRobinQueue.poll(AbstractRoundRobinQueue.java:13
8)
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at
Thread [name="phoenix-1-thread-3230", id=128662, state=TIMED_WAITING, blockCnt=1,

waitCnt=3]
Lock [object=java.lang.Object@69beb985, ownerName=null, ownerId=-1]
at
org.apache.phoenix.job.AbstractRoundRobinQueue.poll(AbstractRoundRobinQueue.java:13
8)
at
at
at

Lock
at
at
at
at
at
Lock
at
at
at
at
at

Lock
at
at
at
at
at
Thread [name="pub-#26541", id=128605, state=WAITING, blockCnt=47, waitCnt=56]

at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304)
at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179)
at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:142)
at
o.a.i.i.processors.cache.GridCacheAdapter$20.op(GridCacheAdapter.java:2511)
at
at
o.a.i.i.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4284)
at
o.a.i.i.processors.cache.GridCacheAdapter.put0(GridCacheAdapter.java:2509)
at
o.a.i.i.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2487)
at
at
o.a.i.i.processors.cache.IgniteCacheProxyImpl.put(IgniteCacheProxyImpl.java:1332)
at
o.a.i.i.processors.cache.GatewayProtectedCacheProxy.put(GatewayProtectedCacheProxy.
java:867)
at
com.clouds.ignite.cache.utility.IgniteInsertAPI.insertInCache(IgniteInsertAPI.java:
298)
at
com.clouds.ignite.cache.utility.IgniteInsertAPI.insert(IgniteInsertAPI.java:117)
at com.clouds.ignite.transaction.TxMetrics.txCommits(TxMetrics.java:87)
at
com.clouds.ignite.transaction.ServerTransactionProcessor.commitTransaction(ServerTr
ansactionProcessor.java:88)
at
com.clouds.ignite.transaction.TransationProcesser.igniteTransaction(TransationProce
sser.java:50)
at
com.clouds.database.transaction.Transaction.initiatTransaction(Transaction.java:62)
at com.clouds.core.stream.ActionProcessor.process(ActionProcessor.java:135)
at
com.clouds.core.stream.CoreStreamProcessor.processCommand(CoreStreamProcessor.java:
188)
at
com.clouds.core.stream.CoreStreamProcessor.process(CoreStreamProcessor.java:113)
at
com.clouds.ignite.cascadeupdate.ModuleComputeJobLogic.executeComputeJobLogic(Module
ComputeJobLogic.java:61)
at
com.clouds.ignite.cascadeupdate.IgniteComputeExecutor.lambda$processModuleOnSingleN
ode$815299f$1(IgniteComputeExecutor.java:123)
at com.clouds.ignite.cascadeupdate.IgniteComputeExecutor$
$Lambda$2374/1577209067.run(Unknown Source)
at
o.a.i.i.processors.closure.GridClosureProcessor$C4.execute(GridClosureProcessor.jav
a:1940)
at o.a.i.i.processors.job.GridJobWorker$1.call(GridJobWorker.java:628)
at o.a.i.i.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7384)
at o.a.i.i.processors.job.GridJobWorker.execute0(GridJobWorker.java:622)
at o.a.i.i.processors.job.GridJobWorker.body(GridJobWorker.java:547)
at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125)
at
at
Locked synchronizers:
java.util.concurrent.ThreadPoolExecutor$Worker@6f62496a
at
at
at
at
at
at
at
at
java:867)
at
298)
at
at
at
sser.java:50)
at
at
188)
at
at
at
at
a:1940)
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@27ac2596
Thread [name="pub-#26539", id=128603, state=TIMED_WAITING, blockCnt=38, waitCnt=47]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4aa9e
7f6, ownerName=null, ownerId=-1]
at
at
at
at
at

Lock
at
at
at
at
at
Thread [name="pub-#26536", id=128586, state=TIMED_WAITING, blockCnt=134,

waitCnt=142]
Lock
at
at
at
at
at

Lock
at
at
at
at
at

at
at
at
at
at
at
at
at
java:867)
at
298)
at
at
at
sser.java:50)
at
at
188)
at
at
at
at
a:1940)
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@3452fd04
Thread [name="pub-#26532", id=128573, state=RUNNABLE, blockCnt=65, waitCnt=65]
at java.lang.Throwable.getStackTraceElement(Native Method)
at java.lang.Throwable.getOurStackTrace(Throwable.java:828)
- locked java.lang.Throwable@392e7eaf
at java.lang.Throwable.getStackTrace(Throwable.java:817)
at com.clouds.logger.Logger.delay(Logger.java:288)
at com.clouds.logger.baton.Baton.add(Baton.java:66)
at com.clouds.core.abstraction.CloudsDevice.<init>(CloudsDevice.java:116)
at com.clouds.core.abstraction.ListingTask.<init>(ListingTask.java:25)
at
com.clouds.maintenance.hub.UpdateMaintenanceHub.<init>(UpdateMaintenanceHub.java:52
)
at
com.clouds.maintenance.controller.HubController.getUpdateMaintenanceHubReference(Hu
bController.java:318)
at com.clouds.maintenance.controller.HubController$$EnhancerBySpringCGLIB$
$a3a7eaa5.CGLIB$getUpdateMaintenanceHubReference$5(<generated>)
$a3a7eaa5$$FastClassBySpringCGLIB$$428198b4.invoke(<generated>)
at
org.springframework.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:244)
at
org.springframework.context.annotation.ConfigurationClassEnhancer$BeanMethodInterce
ptor.intercept(ConfigurationClassEnhancer.java:363)
$a3a7eaa5.getUpdateMaintenanceHubReference(<generated>)
at sun.reflect.GeneratedMethodAccessor636.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:4
3)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(S
impleInstantiationStrategy.java:154)
at
org.springframework.beans.factory.support.ConstructorResolver.instantiate(Construct
orResolver.java:651)
at
org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFacto
ryMethod(ConstructorResolver.java:636)
at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instan
tiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1338)
at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.create
BeanInstance(AbstractAutowireCapableBeanFactory.java:1177)
at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCrea
teBean(AbstractAutowireCapableBeanFactory.java:557)
at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.create
Bean(AbstractAutowireCapableBeanFactory.java:517)
at
org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBea
nFactory.java:341)
at
org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanF
actory.java:212)
at
org.springframework.context.support.AbstractApplicationContext.getBean(AbstractAppl
icationContext.java:1120)
at
com.clouds.controller.util.CloudBeanManager.getCloudActionController(CloudBeanManag
er.java:81)
at
com.clouds.core.stream.ActionProcessor.getCloudActionController(ActionProcessor.jav
a:108)
at
sser.java:49)
at
at
188)
at
at
at
at
a:1940)
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@2bc3a22f
Thread [name="sys-#26516", id=128416, state=TIMED_WAITING, blockCnt=23, waitCnt=57]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3c6fc
14c, ownerName=null, ownerId=-1]
at
at
at
at
at
Thread [name="sys-#26447", id=127651, state=TIMED_WAITING, blockCnt=97,

waitCnt=244]
Lock
at
at
at
at
at

waitCnt=252]
Lock
at
at
at
at
at

waitCnt=262]
Lock
at
at
at
at
at

waitCnt=258]
Lock
at
at
at
at
at

waitCnt=246]
Lock
at
at
at
at
at

waitCnt=338]
Lock
at
at
at
at
at

waitCnt=381]
Lock
at
at
at
at
at
Thread [name="pool-8-thread-4", id=5859, state=WAITING, blockCnt=50, waitCnt=75]

Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2274a
c00, ownerName=null, ownerId=-1]
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Abstrac
tQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at

Lock
at
at
at
at
at
Thread [name="org.apache.hadoop.hdfs.PeerCache@35d31883", id=1626,

at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255)
at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
Thread
[name="org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner",
id=1614, state=WAITING, blockCnt=0, waitCnt=1]
Lock [object=java.lang.ref.ReferenceQueue$Lock@79074be2, ownerName=null,
ownerId=-1]
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
at
org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileS
ystem.java:3063)
Thread [name="MQTT Ping: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1048,

Lock [object=java.util.TaskQueue@2a02dde8, ownerName=null, ownerId=-1]
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
Thread [name="MQTT Call: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1047,

state=WAITING, blockCnt=792321, waitCnt=690282]
Lock [object=java.lang.Object@690f9013, ownerName=null, ownerId=-1]
at java.lang.Object.wait(Object.java:502)
at
org.eclipse.paho.mqttv5.client.internal.CommsCallback.run(CommsCallback.java:192)
Thread [name="MQTT Snd: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1046,

Lock [object=java.lang.Object@1b9db694, ownerName=null, ownerId=-1]
at
org.eclipse.paho.mqttv5.client.internal.ClientState.get(ClientState.java:833)
at
org.eclipse.paho.mqttv5.client.internal.CommsSender.run(CommsSender.java:130)
Thread [name="MQTT Rec: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1045,

state=RUNNABLE, blockCnt=1875, waitCnt=0]
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at
sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464)
at
sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.ja
va:68)
at
sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1346)
at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)
at
sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:962)
at
at java.io.DataInputStream.readByte(DataInputStream.java:265)
at
org.eclipse.paho.mqttv5.client.wire.MqttInputStream.readMqttWireMessage(MqttInputSt
ream.java:96)
at
org.eclipse.paho.mqttv5.client.internal.CommsReceiver.run(CommsReceiver.java:139)
java.util.concurrent.locks.ReentrantLock$NonfairSync@4e741e4a
Thread [name="tcp-disco-sock-reader-[48268ee8 10.233.86.2:37935]-#9-#84", id=899,
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
- locked java.io.BufferedInputStream@60b210f3
at
o.a.i.marshaller.jdk.JdkMarshallerInputStreamWrapper.read(JdkMarshallerInputStreamW
rapper.java:53)
at
java.io.ObjectInputStream$PeekInputStream.read(ObjectInputStream.java:2814)
at
java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2830)
at
java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:330
7)
at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:934)
at java.io.ObjectInputStream.<init>(ObjectInputStream.java:396)
at
o.a.i.marshaller.jdk.JdkMarshallerObjectInputStream.<init>(JdkMarshallerObjectInput
Stream.java:43)
at o.a.i.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:122)
at
o.a.i.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMar
shaller.java:92)
at o.a.i.i.util.IgniteUtils.unmarshal(IgniteUtils.java:10663)
at
o.a.i.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:7021)
at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:58)
Thread [name="hive-consumer-health-checkup-thread", id=812, state=TIMED_WAITING,

blockCnt=1, waitCnt=4490]
Lock [object=java.util.TaskQueue@6fe4c746, ownerName=null, ownerId=-1]
Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-9", id=624, state=WAITING,

Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6f947
633, ownerName=null, ownerId=-1]
at
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThr
eadPoolExecutor.java:1088)
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at
Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-6", id=620, state=TIMED_WAITING,

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at
Thread [name="ReadOnlyZKClient-192.168.24.25:2181@0x077725ff", id=595,

Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11cb2
at
at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
at
org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:326)
at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$
Thread [name="Timer for 'Phoenix' metrics system", id=594, state=TIMED_WAITING,

Lock [object=java.util.TaskQueue@44130f20, ownerName=null, ownerId=-1]
Thread [name="QueryLogger-thread-0", id=591, state=WAITING, blockCnt=0, waitCnt=1]

Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@9b1b0
at
at
com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
at
com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java
:56)
at
com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
Thread [name="DestroyJavaVM", id=586, state=RUNNABLE, blockCnt=0, waitCnt=0]

Lock
cc9, ownerName=null, ownerId=-1]
at
at
at
at
at
at
Thread [name="http-nio-8081-Acceptor", id=580, state=RUNNABLE, blockCnt=0,

waitCnt=0]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
at
- locked java.lang.Object@c8d0318
at
org.apache.tomcat.util.net.NioEndpoint.serverSocketAccept(NioEndpoint.java:467)
at
org.apache.tomcat.util.net.NioEndpoint.serverSocketAccept(NioEndpoint.java:73)
at org.apache.tomcat.util.net.Acceptor.run(Acceptor.java:95)
Thread [name="http-nio-8081-ClientPoller", id=579, state=RUNNABLE, blockCnt=0,

waitCnt=0]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked sun.nio.ch.Util$3@469ccf20
- locked java.util.Collections$UnmodifiableSet@28740ff7
- locked sun.nio.ch.EPollSelectorImpl@5831b782
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.tomcat.util.net.NioEndpoint$Poller.run(NioEndpoint.java:712)
Thread [name="http-nio-8081-exec-10", id=577, state=WAITING, blockCnt=0, waitCnt=1]

Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4e6e9
cf4, ownerName=null, ownerId=-1]
at
at
at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:107)
at
at
at
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at

Lock
at
at
at
at
at
at
Thread [name="http-nio-8081-BlockPoller", id=567, state=RUNNABLE, blockCnt=0,

waitCnt=0]
- locked sun.nio.ch.Util$3@3c40ea5f
- locked java.util.Collections$UnmodifiableSet@1c05d738
- locked sun.nio.ch.EPollSelectorImpl@148668f8
at
org.apache.tomcat.util.net.NioBlockingSelector$BlockPoller.run(NioBlockingSelector.
java:313)
Thread [name="container-0", id=552, state=TIMED_WAITING, blockCnt=0, waitCnt=26894]

at org.apache.catalina.core.StandardServer.await(StandardServer.java:570)
at
org.springframework.boot.web.embedded.tomcat.TomcatWebServer$1.run(TomcatWebServer.
java:181)
Thread [name="Catalina-utility-2", id=551, state=TIMED_WAITING, blockCnt=0,
waitCnt=331143]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@31e6d
abb, ownerName=null, ownerId=-1]
at
at
at
at
at
at
at
Thread [name="Catalina-utility-1", id=550, state=WAITING, blockCnt=0,

waitCnt=331068]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@31e6d
abb, ownerName=null, ownerId=-1]
at
at
at
at
at
at
at
Thread [name="MQTT Ping: BACKEND-CONSUMER-HUB/HUB_TO_MTN/#-v0.0.88-57-28", id=549,

Lock [object=java.util.TaskQueue@3ee823c5, ownerName=null, ownerId=-1]
Thread [name="MQTT Call: BACKEND-CONSUMER-HUB/HUB_TO_MTN/#-v0.0.88-57-28", id=548,
Lock [object=java.lang.Object@164626c5, ownerName=null, ownerId=-1]
at
org.eclipse.paho.mqttv5.client.internal.CommsCallback.run(CommsCallback.java:192)
Thread [name="MQTT Snd: BACKEND-CONSUMER-HUB/HUB_TO_MTN/#-v0.0.88-57-28", id=547,

Lock [object=java.lang.Object@2c7591c5, ownerName=null, ownerId=-1]
at
org.eclipse.paho.mqttv5.client.internal.ClientState.get(ClientState.java:833)
at
org.eclipse.paho.mqttv5.client.internal.CommsSender.run(CommsSender.java:130)
Thread [name="MQTT Rec: BACKEND-CONSUMER-HUB/HUB_TO_MTN/#-v0.0.88-57-28", id=546,

at
sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464)
at
sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.ja
va:68)
at
sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1346)
at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)
at
at
at java.io.DataInputStream.readByte(DataInputStream.java:265)
at
org.eclipse.paho.mqttv5.client.wire.MqttInputStream.readMqttWireMessage(MqttInputSt
ream.java:96)
at
org.eclipse.paho.mqttv5.client.internal.CommsReceiver.run(CommsReceiver.java:139)
java.util.concurrent.locks.ReentrantLock$NonfairSync@41de504d
Thread [name="kafka-coordinator-heartbeat-thread | test-mtn-clz-com-v0.0.88-
RTH_CLZ_COM-TEST_123", id=544, state=TIMED_WAITING, blockCnt=81105,
waitCnt=4830480]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@2b41b5f2,
ownerName=null, ownerId=-1]
at
org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run
(AbstractCoordinator.java:1482)
waitCnt=4825853]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@54485d54,
at

waitCnt=4822187]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@731f2916,
at

HUB_CLZ_COM_BCKGRND_PROCESS-TEST_123", id=541, state=TIMED_WAITING, blockCnt=79480,
waitCnt=4786781]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@73e46509,
at

DAPI_CLZ_COM_POST_PROCESS-TEST_123", id=540, state=TIMED_WAITING, blockCnt=80979,
waitCnt=4824070]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@55aca0a2,
at
Thread [name="test-mtn-clz-com-v0.0.88-RTH_CLZ_COM-TEST_123-a865a3b0-d26a-4428-
b19a-11b5f1e4111f-CleanupThread", id=539, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4318e
5be, ownerName=null, ownerId=-1]
at
at
at
at
at
at

RTH_CLZ_COM_DELETE-TEST_123", id=534, state=TIMED_WAITING, blockCnt=80755,
waitCnt=4819402]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@49c53352,
at

waitCnt=4819829]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@1959db4e,
at

waitCnt=4824242]
Lock
at

waitCnt=4817604]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@2d71f4f4,
at

RTH_CLZ_COM_BCKGRND_PROCESS-TEST_123", id=535, state=TIMED_WAITING, blockCnt=80810,
waitCnt=4821546]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@4d4e1dab,
at

waitCnt=4778018]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@1a1a2ce7,
at

waitCnt=4775220]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@35fe1daa,
at
b19a-11b5f1e4111f-StreamThread-3", id=511, state=RUNNABLE, blockCnt=1659630,
waitCnt=14968]
- locked sun.nio.ch.Util$3@7a77583b
- locked java.util.Collections$UnmodifiableSet@48f9ec5a
- locked sun.nio.ch.EPollSelectorImpl@6f9f9c15
at org.apache.kafka.common.network.Selector.select(Selector.java:873)
at org.apache.kafka.common.network.Selector.poll(Selector.java:465)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:560)
at
org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetw
orkClient.java:280)
at
orkClient.java:251)
at
org.apache.kafka.clients.consumer.KafkaConsumer.pollForFetches(KafkaConsumer.java:1
311)
at
org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1247)
at
at
org.apache.kafka.streams.processor.internals.StreamThread.pollRequests(StreamThread
.java:971)
at
org.apache.kafka.streams.processor.internals.StreamThread.pollPhase(StreamThread.ja
va:919)
at
org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java
:737)
at
org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java
:588)
at
org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:550
)
java.util.concurrent.locks.ReentrantLock$FairSync@2b583085
waitCnt=12981]
- locked sun.nio.ch.Util$3@5c70f593
- locked java.util.Collections$UnmodifiableSet@1c86d8a6
- locked sun.nio.ch.EPollSelectorImpl@1b1a6c50
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@3f27cb8d
waitCnt=17492]
- locked sun.nio.ch.Util$3@8f63412
- locked java.util.Collections$UnmodifiableSet@537184f2
- locked sun.nio.ch.EPollSelectorImpl@799825d2
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@736e0b0b
Thread [name="test-mtn-clz-com-v0.0.88-DAPI_CLZ_COM_POST_PROCESS-TEST_123-e9aa56bf-
eb0b-4e7c-a4c1-a118f8c5058a-CleanupThread", id=530, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@170a8
9b5, ownerName=null, ownerId=-1]
at
at
at
at
at
at
eb0b-4e7c-a4c1-a118f8c5058a-StreamThread-3", id=509, state=RUNNABLE,
- locked sun.nio.ch.Util$3@31a8ae5d
- locked java.util.Collections$UnmodifiableSet@8474ade
- locked sun.nio.ch.EPollSelectorImpl@57ce0930
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@330ede8b
- locked sun.nio.ch.Util$3@6f4010ae
- locked java.util.Collections$UnmodifiableSet@1b6e915d
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6f262690
- locked sun.nio.ch.Util$3@e6d32c4
- locked java.util.Collections$UnmodifiableSet@614786c7
- locked sun.nio.ch.EPollSelectorImpl@ecc17cb
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@151252cf
waitCnt=4819214]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@2a147dba,
at
Thread [name="test-mtn-clz-com-v0.0.88-RTH_CLZ_COM_DELETE-TEST_123-f99ad426-bccb-
4e1b-9907-4af104390725-CleanupThread", id=527, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
10b, ownerName=null, ownerId=-1]
at
at
at
at
at
at

waitCnt=4820698]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@24f453d8,
at
Thread [name="test-mtn-clz-com-v0.0.88-HUB_CLZ_COM_BCKGRND_PROCESS-TEST_123-
142d3669-5783-4079-9ccb-f5a76a7c9218-CleanupThread", id=525, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67702
1d2, ownerName=null, ownerId=-1]
at
at
at
at
at
at
4e1b-9907-4af104390725-StreamThread-3", id=506, state=RUNNABLE, blockCnt=1695644,
waitCnt=14063]
- locked sun.nio.ch.Util$3@4e0b08df
- locked java.util.Collections$UnmodifiableSet@2917e8c8
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@2e16cabc
waitCnt=12796]
- locked sun.nio.ch.Util$3@1a271190
- locked java.util.Collections$UnmodifiableSet@c7cbffb
- locked sun.nio.ch.EPollSelectorImpl@1a3b5714
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@72a0ba51
waitCnt=14129]
- locked sun.nio.ch.Util$3@37b9416a
- locked java.util.Collections$UnmodifiableSet@15b06d9c
- locked sun.nio.ch.EPollSelectorImpl@7c5a8bb6
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@7695bd2c
waitCnt=4834473]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@6e00792b,
at
142d3669-5783-4079-9ccb-f5a76a7c9218-StreamThread-3", id=500, state=RUNNABLE,
at
com.clouds.kafka.broadcast.BroadcastHelper.isOverSizePayload(BroadcastHelper.java:2
8)
at
com.clouds.kafka.broadcast.GeneralCommandBroadcast.sendCommandResponse(GeneralComma
ndBroadcast.java:59)
at
com.clouds.core.operations.Broadcaster.broadcastResponse(Broadcaster.java:187)
at com.clouds.core.operations.Broadcaster.broadcast(Broadcaster.java:252)
at
com.clouds.maintenance.utility.BroadcastUtility.performBroadcastOperations(Broadcas
tUtility.java:135)
at
com.clouds.maintenance.utility.BroadcastUtility.getAllInviteeUsersForBroadcastNew(B
roadcastUtility.java:748)
at
com.clouds.maintenance.hub.UpdateHubDetails.devise(UpdateHubDetails.java:238)
at com.clouds.core.stream.ActionProcessor.devise(ActionProcessor.java:173)
at
199)
at
at
com.clouds.stream.core.processor.kafka.CoreKafkaIgniteComputeStream.process(CoreKaf
kaIgniteComputeStream.java:72)
at
at
org.apache.kafka.streams.processor.internals.ProcessorAdapter.process(ProcessorAdap
ter.java:71)
at
org.apache.kafka.streams.processor.internals.ProcessorNode.process(ProcessorNode.ja
va:157)
at
org.apache.kafka.streams.processor.internals.ProcessorContextImpl.forwardInternal(P
rocessorContextImpl.java:290)
at
org.apache.kafka.streams.processor.internals.ProcessorContextImpl.forward(Processor
ContextImpl.java:269)
at
at
at
com.clouds.stream.core.validator.KafkaStreamRequestReceiver.process(KafkaStreamRequ
estReceiver.java:64)
at
at
ter.java:71)
at
va:157)
at
at
at
at
org.apache.kafka.streams.processor.internals.SourceNode.process(SourceNode.java:84)
at
org.apache.kafka.streams.processor.internals.StreamTask.lambda$doProcess$1(StreamTa
sk.java:793)
at org.apache.kafka.streams.processor.internals.StreamTask$
at
org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasur
eLatency(StreamsMetricsImpl.java:872)
at
org.apache.kafka.streams.processor.internals.StreamTask.doProcess(StreamTask.java:7
93)
at
org.apache.kafka.streams.processor.internals.StreamTask.process(StreamTask.java:724
)
at
org.apache.kafka.streams.processor.internals.TaskExecutor.processTask(TaskExecutor.
java:100)
at
org.apache.kafka.streams.processor.internals.TaskExecutor.process(TaskExecutor.java
:81)
at
org.apache.kafka.streams.processor.internals.TaskManager.process(TaskManager.java:1
182)
at
:770)
at
:588)
at
)
at java.lang.Throwable.fillInStackTrace(Native Method)
at java.lang.Throwable.fillInStackTrace(Throwable.java:784)
- locked java.lang.Throwable@3c99f0de
at java.lang.Throwable.<init>(Throwable.java:251)
at com.clouds.logger.Logger.delay(Logger.java:242)
at
com.clouds.apibuilder.util.RequestFactory.getUtility(RequestFactory.java:48)
at
com.clouds.apibuilder.util.PayloadIdentifier.isCommandPayload(PayloadIdentifier.jav
a:31)
at
com.clouds.core.operations.Broadcaster.checkVisibility(Broadcaster.java:400)
at
tUtility.java:135)
at
at
at
199)
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)
- locked java.lang.Throwable@41565436
at
org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:177)
at
org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:7
8)
at
org.apache.logging.log4j.core.impl.Log4jLogEvent.getSource(Log4jLogEvent.java:693)
at org.graylog2.log4j2.GelfAppender.append(GelfAppender.java:127)
at
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderContro
l.java:156)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.
java:129)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(A
ppenderControl.java:120)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.j
ava:84)
at
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:5
43)
at
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java
:502)
at
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:485)
at
at
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCo
mpletionReliabilityStrategy.java:82)
at org.apache.logging.log4j.core.Logger.log(Logger.java:161)
at
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2198)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger
.java:2152)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:21
35)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2011)
at
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
at
org.apache.logging.log4j.spi.AbstractLogger.warn(AbstractLogger.java:2659)
at com.clouds.logger.CloudLogger.warn(CloudLogger.java:137)
at com.clouds.logger.Logger.warn(Logger.java:215)
at
at
tUtility.java:135)
at
at
at
199)
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)

MTN_CLZ_COM-TEST_123", id=522, state=TIMED_WAITING, blockCnt=80394,
waitCnt=4811093]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@1fdb07a9,
at

waitCnt=4811438]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@19b7fd19,
at

waitCnt=4811994]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@7f2f314c,
at
DAPI_CLZ_COM_DELETE-TEST_123", id=518, state=TIMED_WAITING, blockCnt=80912,
waitCnt=4820278]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@120bc6a3,
at

waitCnt=4819201]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@552a6d04,
at
Thread [name="test-mtn-clz-com-v0.0.88-RTH_CLZ_COM_BCKGRND_PROCESS-TEST_123-
9b3ec258-e059-4979-a978-796e345adeb5-CleanupThread", id=516, state=TIMED_WAITING,
Lock
d8d, ownerName=null, ownerId=-1]
at
at
at
at
at
at

MTN_CLZ_COM_DELETE-TEST_123", id=515, state=TIMED_WAITING, blockCnt=76672,
waitCnt=4573935]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@75f0e7ea,
at
9b3ec258-e059-4979-a978-796e345adeb5-StreamThread-3", id=483, state=RUNNABLE,
- locked sun.nio.ch.Util$3@571c0d0d
- locked java.util.Collections$UnmodifiableSet@7fb32b63
- locked sun.nio.ch.EPollSelectorImpl@c5f3b4b
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@7be6eb9b
- locked sun.nio.ch.Util$3@7f5b3912
- locked java.util.Collections$UnmodifiableSet@48182791
- locked sun.nio.ch.EPollSelectorImpl@2e16f5c2
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@122bdb3d
- locked sun.nio.ch.Util$3@5be60628
- locked java.util.Collections$UnmodifiableSet@7dd327ef
- locked sun.nio.ch.EPollSelectorImpl@747ea36b
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@77ef6bd0
waitCnt=4821639]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@39821929,
at

waitCnt=4818579]
Lock
at

waitCnt=4819729]
Lock
at
Thread [name="test-mtn-clz-com-v0.0.88-MTN_CLZ_COM-TEST_123-68e561cb-c3c0-4fa8-
a191-b39fdbe68776-CleanupThread", id=508, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
e8f, ownerName=null, ownerId=-1]
at
at
at
at
at
at
a191-b39fdbe68776-StreamThread-3", id=487, state=RUNNABLE, blockCnt=1731482,
waitCnt=57671]
- locked sun.nio.ch.Util$3@6ddfe0c3
- locked java.util.Collections$UnmodifiableSet@6e36c16
- locked sun.nio.ch.EPollSelectorImpl@62ebdf7a
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6088ad50
a191-b39fdbe68776-StreamThread-2", id=400, state=RUNNABLE, blockCnt=1717495,
waitCnt=58646]
- locked sun.nio.ch.Util$3@638f5db7
- locked java.util.Collections$UnmodifiableSet@714460fd
- locked sun.nio.ch.EPollSelectorImpl@6cda211f
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6fb40cbb
a191-b39fdbe68776-StreamThread-1", id=353, state=WAITING, blockCnt=1708220,
waitCnt=60942]
at
at
at
at
at
at
at
at
java:867)
at
298)
at
at
com.clouds.stream.core.validator.KafkaStreamRequestReceiver.repeatRequestChecking(K
afkaStreamRequestReceiver.java:100)
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)
Thread [name="test-mtn-clz-com-v0.0.88-DAPI_CLZ_COM_DELETE-TEST_123-1f43f117-62f8-
49c6-be42-73aa89020063-CleanupThread", id=505, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@fcd20
at
at
at
at
at
at
49c6-be42-73aa89020063-StreamThread-3", id=497, state=RUNNABLE, blockCnt=1699888,
waitCnt=14545]
- locked java.util.Collections$UnmodifiableSet@498daead
- locked sun.nio.ch.EPollSelectorImpl@1b1c7ce9
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@5f53f52d
waitCnt=13159]
- locked sun.nio.ch.Util$3@26f3744c
- locked java.util.Collections$UnmodifiableSet@477bb2c8
- locked sun.nio.ch.EPollSelectorImpl@6327944c
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@7384039
waitCnt=15106]
- locked sun.nio.ch.Util$3@1b2457f5
- locked java.util.Collections$UnmodifiableSet@26c801c9
- locked sun.nio.ch.EPollSelectorImpl@3c3f1a2a
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@5f43f40a
HUB_CLZ_COM-TEST_123", id=503, state=TIMED_WAITING, blockCnt=80421,
waitCnt=4813012]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@8eec1a,
at
Thread [name="test-mtn-clz-com-v0.0.88-MTN_CLZ_COM_DELETE-TEST_123-da8a730f-6145-
437e-9171-e7d733239469-CleanupThread", id=502, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
at
at
at
at
at
at
437e-9171-e7d733239469-StreamThread-3", id=490, state=RUNNABLE, blockCnt=1739580,
waitCnt=12942]
- locked sun.nio.ch.Util$3@63e56c15
- locked java.util.Collections$UnmodifiableSet@5ac70463
- locked sun.nio.ch.EPollSelectorImpl@3c6790bb
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@3cd1c62e
waitCnt=12911]
- locked sun.nio.ch.Util$3@784d2357
- locked java.util.Collections$UnmodifiableSet@30806d8f
- locked sun.nio.ch.EPollSelectorImpl@528693e0
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
waitCnt=16419]
- locked sun.nio.ch.Util$3@52e42343
- locked java.util.Collections$UnmodifiableSet@43a2e5d3
- locked sun.nio.ch.EPollSelectorImpl@707d143d
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6e183220
MTN_CLZ_COM_BCKGRND_PROCESS-TEST_123", id=499, state=TIMED_WAITING, blockCnt=80368,
waitCnt=4807026]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@19bc5519,
at

waitCnt=4811879]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@7f163aac,
at

waitCnt=4807532]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@88ff768,
at

waitCnt=4813221]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@cb33616,
at

HUB_CLZ_COM_DELETE-TEST_123", id=494, state=TIMED_WAITING, blockCnt=80901,
waitCnt=4819223]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@77dcbe04,
at

waitCnt=4572823]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@18b1412e,
at
waitCnt=4818532]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@4e92adfc,
at

waitCnt=4812147]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@5b0431be,
at
Thread [name="test-mtn-clz-com-v0.0.88-HUB_CLZ_COM-TEST_123-c3c77b25-6c36-4d2a-
9782-07837f90f8f4-CleanupThread", id=489, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49ba8
at
at
at
at
at
at
9782-07837f90f8f4-StreamThread-3", id=476, state=WAITING, blockCnt=1808947,
waitCnt=74987]
at
at
at
at
at
at
at
at
java:867)
at
298)
at
at
com.clouds.stream.core.validator.KafkaStreamRequestReceiver.repeatRequestChecking(K
afkaStreamRequestReceiver.java:100)
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)
9782-07837f90f8f4-StreamThread-2", id=396, state=RUNNABLE, blockCnt=1793814,
waitCnt=78436]
- locked sun.nio.ch.Util$3@16829041
- locked java.util.Collections$UnmodifiableSet@650f47e9
- locked sun.nio.ch.EPollSelectorImpl@31d7aae6
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@5162f4fc
9782-07837f90f8f4-StreamThread-1", id=348, state=RUNNABLE, blockCnt=1734191,
waitCnt=134625]
- locked java.lang.Throwable@29939c1d
at
org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:177)
at
org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:7
8)
at
org.apache.logging.log4j.core.impl.Log4jLogEvent.getSource(Log4jLogEvent.java:693)
at org.graylog2.log4j2.GelfAppender.append(GelfAppender.java:127)
at
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderContro
l.java:156)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.
java:129)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(A
ppenderControl.java:120)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.j
ava:84)
at
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:5
43)
at
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java
:502)
at
at
at
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCo
mpletionReliabilityStrategy.java:82)
at org.apache.logging.log4j.core.Logger.log(Logger.java:161)
at
org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2198)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger
.java:2152)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:21
35)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2011)
at
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
at
org.apache.logging.log4j.spi.AbstractLogger.warn(AbstractLogger.java:2659)
at com.clouds.logger.CloudLogger.warn(CloudLogger.java:137)
at com.clouds.logger.Logger.warn(Logger.java:215)
at
com.clouds.kafka.broadcast.BroadcastHelper.logPayloadDetails(BroadcastHelper.java:8
6)
at
com.clouds.kafka.broadcast.GeneralCommandBroadcast.sendCommandResponse(GeneralComma
ndBroadcast.java:60)
at
at
tUtility.java:135)
at
at
com.clouds.maintenance.hub.information.ResetHubRebootProgressBar.devise(ResetHubReb
ootProgressBar.java:209)
at
199)
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)
Thread [name="test-mtn-clz-com-v0.0.88-MTN_CLZ_COM_BCKGRND_PROCESS-TEST_123-
e01a8bfe-68c6-45e7-8ac6-4457e81b2396-CleanupThread", id=486, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7913e
91b, ownerName=null, ownerId=-1]
at
at
at
at
at
at
e01a8bfe-68c6-45e7-8ac6-4457e81b2396-StreamThread-3", id=466, state=RUNNABLE,
- locked sun.nio.ch.Util$3@2b1101f6
- locked java.util.Collections$UnmodifiableSet@34be2b72
- locked sun.nio.ch.EPollSelectorImpl@38323799
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@660b5a43
- locked sun.nio.ch.Util$3@df437f6
- locked java.util.Collections$UnmodifiableSet@2ec9890f
- locked sun.nio.ch.EPollSelectorImpl@6be9c21f
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@3e05ad56
- locked sun.nio.ch.Util$3@42cebe04
- locked java.util.Collections$UnmodifiableSet@6ea96a2c
- locked sun.nio.ch.EPollSelectorImpl@cca0707
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@34bef6ec
MTN_CLZ_COM_POST_PROCESS-TEST_123", id=484, state=TIMED_WAITING, blockCnt=80825,
waitCnt=4824874]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@45a34145,
at
Thread [name="test-mtn-clz-com-v0.0.88-HUB_CLZ_COM_DELETE-TEST_123-ed0d4009-2adb-
4b5c-9a7c-c99fcf9a598b-CleanupThread", id=482, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@231e2
3db, ownerName=null, ownerId=-1]
at
at
at
at
at
at
4b5c-9a7c-c99fcf9a598b-StreamThread-3", id=458, state=RUNNABLE, blockCnt=1734363,
waitCnt=12907]
- locked sun.nio.ch.Util$3@2916c5bb
- locked java.util.Collections$UnmodifiableSet@1e9f6003
- locked sun.nio.ch.EPollSelectorImpl@3bc281f6
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@5fa87f81
waitCnt=13010]
- locked sun.nio.ch.Util$3@676c4bb6
- locked java.util.Collections$UnmodifiableSet@78c861fd
- locked sun.nio.ch.EPollSelectorImpl@1d32999f
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@37057cd3
waitCnt=16387]
- locked sun.nio.ch.Util$3@32d076fe
- locked java.util.Collections$UnmodifiableSet@37e4aae9
- locked sun.nio.ch.EPollSelectorImpl@c98f261
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
waitCnt=4819820]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@2b972026,
at

waitCnt=4822528]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@436ca4e7,
at

RTH_CLZ_COM_POST_PROCESS-TEST_123", id=478, state=TIMED_WAITING, blockCnt=80570,
waitCnt=4822531]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@1c9a6ca6,
at

waitCnt=4818666]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@4ec89415,
at

DAPI_CLZ_COM_BCKGRND_PROCESS-TEST_123", id=475, state=TIMED_WAITING,
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@5ccc1bfa,
at

waitCnt=4829968]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@7aa1d718,
at
Thread [name="test-mtn-clz-com-v0.0.88-MTN_CLZ_COM_POST_PROCESS-TEST_123-f06ae088-
7a74-4591-897a-963cbd18b620-CleanupThread", id=473, state=TIMED_WAITING,
Lock
bb6, ownerName=null, ownerId=-1]
at
at
at
at
at
at
7a74-4591-897a-963cbd18b620-StreamThread-3", id=445, state=RUNNABLE,
- locked sun.nio.ch.Util$3@56f6c0c
- locked java.util.Collections$UnmodifiableSet@34398dd9
- locked sun.nio.ch.EPollSelectorImpl@2618da55
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@563b2a58
- locked sun.nio.ch.Util$3@6dd83f0d
- locked java.util.Collections$UnmodifiableSet@13a931b6
- locked sun.nio.ch.EPollSelectorImpl@6b06215f
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@3dcf7be4
- locked sun.nio.ch.Util$3@bac8418
- locked java.util.Collections$UnmodifiableSet@2720e237
- locked sun.nio.ch.EPollSelectorImpl@7d5e43e4
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@49af9d37
DAPI_CLZ_COM_RETRIEVE-TEST_123", id=471, state=TIMED_WAITING, blockCnt=76537,
waitCnt=4569236]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@3ff3392d,
at

waitCnt=4822646]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@35ce6e2,
at

HUB_CLZ_COM_RETRIEVE-TEST_123", id=469, state=TIMED_WAITING, blockCnt=80607,
waitCnt=4823957]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@77b1f886,
at

Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@b5bd0a,
at

Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@6f5f9359,
at

waitCnt=4823123]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@b5003a6,
at
Thread [name="test-mtn-clz-com-v0.0.88-RTH_CLZ_COM_POST_PROCESS-TEST_123-a0216b91-
9d30-4899-8fd5-f538476d7b74-CleanupThread", id=464, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@73ddb
f82, ownerName=null, ownerId=-1]
at
at
at
at
at
at
9d30-4899-8fd5-f538476d7b74-StreamThread-3", id=444, state=RUNNABLE,
- locked sun.nio.ch.Util$3@278899c9
- locked java.util.Collections$UnmodifiableSet@48eb4564
- locked sun.nio.ch.EPollSelectorImpl@68fa2714
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@4b9471f4
- locked sun.nio.ch.Util$3@6734088c
- locked sun.nio.ch.EPollSelectorImpl@164326e1
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6d71b55c
- locked sun.nio.ch.Util$3@2f3861ba
- locked java.util.Collections$UnmodifiableSet@680f6cb9
- locked sun.nio.ch.EPollSelectorImpl@600de8d9
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@673dfdc5
waitCnt=4822085]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@25c063ed,
at

waitCnt=4825347]
Lock
at

HUB_CLZ_COM_POST_PROCESS-TEST_123", id=460, state=TIMED_WAITING, blockCnt=76559,
waitCnt=4568645]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@3d509110,
at

waitCnt=4817018]
Lock
at

waitCnt=4824957]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@596c81c,
at

MTN_CLZ_COM_RETRIEVE-TEST_123", id=456, state=TIMED_WAITING, blockCnt=80183,
waitCnt=4813712]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@4cbd5b8,
at
Thread [name="test-mtn-clz-com-v0.0.88-DAPI_CLZ_COM_RETRIEVE-TEST_123-47b6dd5f-
e68e-42cb-a64c-275f43c3bdaa-CleanupThread", id=455, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3daa1
at
at
at
at
at
at
e68e-42cb-a64c-275f43c3bdaa-StreamThread-3", id=439, state=RUNNABLE,
- locked sun.nio.ch.Util$3@6f76be17
- locked java.util.Collections$UnmodifiableSet@32f37f2
- locked sun.nio.ch.EPollSelectorImpl@10d6f98a
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@4da3e782
- locked java.util.Collections$UnmodifiableSet@45ce138a
- locked sun.nio.ch.EPollSelectorImpl@3f242c66
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@4faba190
- locked sun.nio.ch.EPollSelectorImpl@26e4d1ba
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@42a9480e
Thread [name="test-mtn-clz-com-v0.0.88-HUB_CLZ_COM_RETRIEVE-TEST_123-7e3e9003-4ff6-
434b-81a0-bd7487790271-CleanupThread", id=453, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
0a2, ownerName=null, ownerId=-1]
at
at
at
at
at
at
434b-81a0-bd7487790271-StreamThread-3", id=440, state=RUNNABLE, blockCnt=1670365,
waitCnt=14741]
- locked sun.nio.ch.Util$3@38bd8e1c
- locked java.util.Collections$UnmodifiableSet@6cb5c53a
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@42d703e0
waitCnt=12904]
- locked sun.nio.ch.Util$3@6142e5eb
- locked java.util.Collections$UnmodifiableSet@355b5ab1
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@d80a19c
waitCnt=12936]
- locked java.util.Collections$UnmodifiableSet@3c038b8c
- locked sun.nio.ch.EPollSelectorImpl@5362d06f
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
waitCnt=4814842]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@553fe25d,
at
Thread [name="test-mtn-clz-com-v0.0.88-DAPI_CLZ_COM_BCKGRND_PROCESS-TEST_123-
23d6704e-4428-452e-a8ba-2f2545a2eb9d-CleanupThread", id=450, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@f039b
at
at
at
at
at
at
23d6704e-4428-452e-a8ba-2f2545a2eb9d-StreamThread-3", id=438, state=RUNNABLE,
- locked sun.nio.ch.Util$3@a179b4b
- locked java.util.Collections$UnmodifiableSet@2930c8a
- locked sun.nio.ch.EPollSelectorImpl@22eca8bf
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@11d88960
- locked sun.nio.ch.EPollSelectorImpl@4bde56a4
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@7e39261f
- locked sun.nio.ch.Util$3@cb0a630
- locked java.util.Collections$UnmodifiableSet@2cf10b5e
- locked sun.nio.ch.EPollSelectorImpl@62b6c0c3
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@49c3e11d
waitCnt=4808884]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@62a5cd8b,
at
Thread [name="test-mtn-clz-com-v0.0.88-HUB_CLZ_COM_POST_PROCESS-TEST_123-f33e6e23-
07a4-4150-9137-95191cea7918-CleanupThread", id=447, state=TIMED_WAITING,
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@54cc9
e6d, ownerName=null, ownerId=-1]
at
at
at
at
at
at
07a4-4150-9137-95191cea7918-StreamThread-3", id=437, state=RUNNABLE,
- locked sun.nio.ch.Util$3@5b64d147
- locked java.util.Collections$UnmodifiableSet@52ec6d3d
- locked sun.nio.ch.EPollSelectorImpl@766327c5
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@129f3403
- locked java.util.Collections$UnmodifiableSet@5f1452fc
- locked sun.nio.ch.EPollSelectorImpl@6e2049a6
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@677d2d64
- locked sun.nio.ch.EPollSelectorImpl@64fb6a27
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@958d4a6
Thread [name="test-mtn-clz-com-v0.0.88-MTN_CLZ_COM_RETRIEVE-TEST_123-ca18f767-af1e-
406b-a14d-1561d4f9903e-CleanupThread", id=443, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e4ba
3a3, ownerName=null, ownerId=-1]
at
at
at
at
at
at
406b-a14d-1561d4f9903e-StreamThread-3", id=435, state=RUNNABLE, blockCnt=1727579,
waitCnt=28529]
at org.apache.phoenix.schema.ValueSchema.getField(ValueSchema.java:334)
at org.apache.phoenix.schema.KeyValueSchema.next(KeyValueSchema.java:208)
at
org.apache.phoenix.schema.KeyValueSchema.iterator(KeyValueSchema.java:166)
at
at
at
org.apache.phoenix.expression.ProjectedColumnExpression.evaluate(ProjectedColumnExp
ression.java:119)
at
org.apache.phoenix.compile.ExpressionProjector.getValue(ExpressionProjector.java:69
)
at
org.apache.phoenix.jdbc.PhoenixResultSet.getObject(PhoenixResultSet.java:585)
at
com.clouds.phoenix.utility.PhoenixUtility.makeObjectsList(PhoenixUtility.java:185)
at
com.clouds.phoenix.retrieve.PhoenixFetch.fetchListData(PhoenixFetch.java:268)
at
3)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.ja
va:344)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(Reflec
tiveMethodInvocation.java:198)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMeth
odInvocation.java:163)
at
org.springframework.aop.framework.adapter.MethodBeforeAdviceInterceptor.invoke(Meth
odBeforeAdviceInterceptor.java:56)
at
at
org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodI
nvocationProceedingJoinPoint.java:88)
at
com.clouds.database.parent.logger.TrackOperation.dbadvice(TrackOperation.java:71)
at
3)
at
org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenAr
gs(AbstractAspectJAdvice.java:644)
at
org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAs
pectJAdvice.java:633)
at
org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java
:70)
at
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvoca
tionInterceptor.java:93)
at
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java
:212)
at com.sun.proxy.$Proxy152.fetchListData(Unknown Source)
at
com.clouds.database.generic.retrieve.DatabaseFetch.fetchListData(DatabaseFetch.java
:176)
at
com.clouds.maintenance.activity.ondemand.FetchHubReleaseVersions.getVersions(FetchH
ubReleaseVersions.java:276)
at
com.clouds.maintenance.activity.ondemand.FetchHubReleaseVersions.fetchData(FetchHub
ReleaseVersions.java:86)
at
com.clouds.retrieval.core.base.CloudRetreivalService.fetchData(CloudRetreivalServic
e.java:33)
at
com.clouds.retrieval.core.processor.UnfilteredDataRetrievalStreamProcessor.processS
ingleSourceRequest(UnfilteredDataRetrievalStreamProcessor.java:117)
at
ubRequest(UnfilteredDataRetrievalStreamProcessor.java:89)
at
com.clouds.retrieval.core.processor.UnfilteredDataRetrievalStreamProcessor.processR
equest(UnfilteredDataRetrievalStreamProcessor.java:61)
at
com.clouds.core.stream.processor.UFLDRRequestProcessor.process(UFLDRRequestProcesso
r.java:10)
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)
Thread [name="kafka-producer-network-thread | test-mtn-clz-com-v0.0.88-

HUB_CLZ_COM_BCKGRND_PROCESS-TEST_123-142d3669-5783-4079-9ccb-f5a76a7c9218-
StreamThread-3-producer", id=442, state=RUNNABLE, blockCnt=20, waitCnt=0]
- locked sun.nio.ch.Util$3@8fa6267
- locked java.util.Collections$UnmodifiableSet@19cc0286
- locked sun.nio.ch.EPollSelectorImpl@2a3856f6
at
org.apache.kafka.clients.producer.internals.Sender.runOnce(Sender.java:328)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:243)
waitCnt=29500]
at
at
at
at
ression.java:119)
at
)
at
at
at
at
3)
at
va:344)
at
at
at
at
at
at
at
3)
at
at
at
:70)
at
at
at
at
:212)
at
:176)
at
at
at
e.java:33)
at
at
at
at
r.java:10)
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)
waitCnt=52135]
at
at
at
at
ression.java:119)
at
)
at
at
at
at
3)
at
va:344)
at
at
at
at
at
at
at
3)
at
at
at
:70)
at
at
at
at
:212)
at
:176)
at
at
at
e.java:33)
at
at
at
at
r.java:10)
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
at
at
ter.java:71)
at
va:157)
at
at
at
at
at
sk.java:793)
at
at
93)
at
)
at
java:100)
at
:81)
at
182)
at
:770)
at
:588)
at
)

MTN_CLZ_COM_DELETE-TEST_123-da8a730f-6145-437e-9171-e7d733239469-StreamThread-3-
producer", id=436, state=RUNNABLE, blockCnt=8, waitCnt=0]
- locked sun.nio.ch.Util$3@15adabc4
- locked java.util.Collections$UnmodifiableSet@4e1f1a4c
- locked sun.nio.ch.EPollSelectorImpl@373ef93b
at

RTH_CLZ_COM_RETRIEVE-TEST_123", id=434, state=TIMED_WAITING, blockCnt=76551,
waitCnt=4567703]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@218407b0,
at

waitCnt=4819791]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@110bda1b,
at
Thread [name="kafka-producer-network-thread | test-mtn-clz-com-v0.0.88-RTH_CLZ_COM-

TEST_123-a865a3b0-d26a-4428-b19a-11b5f1e4111f-StreamThread-3-producer", id=432,
- locked sun.nio.ch.Util$3@5f5986cc
- locked java.util.Collections$UnmodifiableSet@27d2f568
- locked sun.nio.ch.EPollSelectorImpl@11a946a4
at

waitCnt=4819357]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@66e38532,
at

RTH_CLZ_COM_DELETE-TEST_123-f99ad426-bccb-4e1b-9907-4af104390725-StreamThread-3-
- locked sun.nio.ch.Util$3@511e828b
- locked sun.nio.ch.EPollSelectorImpl@722bd503
at

DAPI_CLZ_COM-TEST_123", id=429, state=TIMED_WAITING, blockCnt=80752,
waitCnt=4823079]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@5528a4d5,
at

waitCnt=4822755]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@54109047,
at

waitCnt=4817855]
Lock
[object=org.apache.kafka.clients.consumer.internals.ConsumerCoordinator@1b8453dc,
at

DAPI_CLZ_COM_DELETE-TEST_123-1f43f117-62f8-49c6-be42-73aa89020063-StreamThread-3-
- locked java.util.Collections$UnmodifiableSet@c43e4f8
- locked sun.nio.ch.EPollSelectorImpl@5e7f8af3
at
Thread [name="test-mtn-clz-com-v0.0.88-RTH_CLZ_COM_RETRIEVE-TEST_123-b0410c4e-5a81-
409b-9e67-bb37ab9d1a67-CleanupThread", id=425, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5890b
2ab, ownerName=null, ownerId=-1]
at
at
at
at
at
at
409b-9e67-bb37ab9d1a67-StreamThread-3", id=417, state=RUNNABLE, blockCnt=1748066,
waitCnt=30726]
- locked sun.nio.ch.Util$3@7250cff2
- locked java.util.Collections$UnmodifiableSet@3af8202d
- locked sun.nio.ch.EPollSelectorImpl@298f77ae
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
waitCnt=30955]
- locked sun.nio.ch.Util$3@b05587e
- locked java.util.Collections$UnmodifiableSet@7f5e29b3
- locked sun.nio.ch.EPollSelectorImpl@4dc973d
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6f1111de
waitCnt=50974]
- locked sun.nio.ch.Util$3@563ca20f
- locked java.util.Collections$UnmodifiableSet@77dbd61e
- locked sun.nio.ch.EPollSelectorImpl@79466ae7
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@106de045
RTH_CLZ_COM_BCKGRND_PROCESS-TEST_123-9b3ec258-e059-4979-a978-796e345adeb5-
- locked sun.nio.ch.Util$3@19c37f8d
- locked java.util.Collections$UnmodifiableSet@2adc9a60
- locked sun.nio.ch.EPollSelectorImpl@39ec203
at
Thread [name="test-mtn-clz-com-v0.0.88-DAPI_CLZ_COM-TEST_123-24eee553-bd2b-4c15-
99d2-aa4f0bba17e9-CleanupThread", id=422, state=TIMED_WAITING, blockCnt=0,
waitCnt=450]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@488eb
b60, ownerName=null, ownerId=-1]
at
at
at
at
at
at
99d2-aa4f0bba17e9-StreamThread-3", id=411, state=RUNNABLE, blockCnt=1687748,
waitCnt=13683]
- locked sun.nio.ch.Util$3@49c591c5
- locked java.util.Collections$UnmodifiableSet@4b78a067
- locked sun.nio.ch.EPollSelectorImpl@5393cf2
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@563ba9f7
waitCnt=14478]
- locked sun.nio.ch.Util$3@70c2688c
- locked java.util.Collections$UnmodifiableSet@156c94e8
- locked sun.nio.ch.EPollSelectorImpl@1b9965ba
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@703c3546
waitCnt=13528]
- locked sun.nio.ch.Util$3@578b1497
- locked java.util.Collections$UnmodifiableSet@346e80e2
at
orkClient.java:280)
at
orkClient.java:251)
at
311)
at
at
at
.java:971)
at
va:919)
at
:737)
at
:588)
at
)
java.util.concurrent.locks.ReentrantLock$FairSync@6d11b490
Thread [name="kafka-producer-network-thread | test-mtn-clz-com-v0.0.88-MTN_CLZ_COM-
TEST_123-68e561cb-c3c0-4fa8-a191-b39fdbe68776-StreamThread-3-producer", id=420,
- locked java.util.Collections$UnmodifiableSet@1a90116b
- locked sun.nio.ch.EPollSelectorImpl@5a39eb4b
at

MTN_CLZ_COM_RETRIEVE-TEST_123-ca18f767-af1e-406b-a14d-1561d4f9903e-StreamThread-3-
- locked sun.nio.ch.Util$3@3bc6ff4e
- locked java.util.Collections$UnmodifiableSet@2ae0fff0
- locked sun.nio.ch.EPollSelectorImpl@3061a88e
at
Thread [name="kafka-producer-network-thread | test-mtn-clz-com-v0.0.88-HUB_CLZ_COM-

TEST_123-c3c77b25-6c36-4d2a-9782-07837f90f8f4-StreamThread-3-producer", id=418,
- locked sun.nio.ch.Util$3@25176e7a
- locked sun.nio.ch.EPollSelectorImpl@415bcb2
at

DAPI_CLZ_COM_POST_PROCESS-TEST_123-e9aa56bf-eb0b-4e7c-a4c1-a118f8c5058a-
- locked sun.nio.ch.Util$3@398d5166
- locked java.util.Collections$UnmodifiableSet@27f78c93
- locked sun.nio.ch.EPollSelectorImpl@300fe675
at

DAPI_CLZ_COM_RETRIEVE-TEST_123-47b6dd5f-e68e-42cb-a64c-275f43c3bdaa-StreamThread-3-
- locked sun.nio.ch.Util$3@19d62cea
- locked java.util.Collections$UnmodifiableSet@4a8358d7
- locked sun.nio.ch.EPollSelectorImpl@518bc600
at

DAPI_CLZ_COM_BCKGRND_PROCESS-TEST_123-23d6704e-4428-452e-a8ba-2f2545a2eb9d-
- locked sun.nio.ch.Util$3@1e78be0a
- locked sun.nio.ch.EPollSelectorImpl@6674d3b8
at

RTH_CLZ_COM_POST_PROCESS-TEST_123-a0216b91-9d30-4899-8fd5-f538476d7b74-
- locked sun.nio.ch.Util$3@69c0a2b0
- locked java.util.Collections$UnmodifiableSet@3213c96c
- locked sun.nio.ch.EPollSelectorImpl@f4dd485
at

MTN_CLZ_COM_BCKGRND_PROCESS-TEST_123-e01a8bfe-68c6-45e7-8ac6-4457e81b2396-
- locked sun.nio.ch.Util$3@4f643fc7
- locked java.util.Collections$UnmodifiableSet@23ddc1b9
- locked sun.nio.ch.EPollSelectorImpl@f293407
at

MTN_CLZ_COM_POST_PROCESS-TEST_123-f06ae088-7a74-4591-897a-963cbd18b620-
- locked sun.nio.ch.Util$3@6e21309b
- locked java.util.Collections$UnmodifiableSet@d5a09f
- locked sun.nio.ch.EPollSelectorImpl@4dc87935
at

HUB_CLZ_COM_POST_PROCESS-TEST_123-f33e6e23-07a4-4150-9137-95191cea7918-
- locked sun.nio.ch.Util$3@b87e587
- locked java.util.Collections$UnmodifiableSet@6c45a984
- locked sun.nio.ch.EPollSelectorImpl@41392dcb
at

HUB_CLZ_COM_DELETE-TEST_123-ed0d4009-2adb-4b5c-9a7c-c99fcf9a598b-StreamThread-3-
- locked sun.nio.ch.Util$3@4fb6e508
- locked java.util.Collections$UnmodifiableSet@7b299aa4
- locked sun.nio.ch.EPollSelectorImpl@3a6115cc
at

HUB_CLZ_COM_RETRIEVE-TEST_123-7e3e9003-4ff6-434b-81a0-bd7487790271-StreamThread-3-
- locked sun.nio.ch.Util$3@323e303e
- locked java.util.Collections$UnmodifiableSet@4a580dd4
at

DAPI_CLZ_COM-TEST_123-24eee553-bd2b-4c15-99d2-aa4f0bba17e9-StreamThread-3-
- locked sun.nio.ch.Util$3@4f6ef78a
- locked java.util.Collections$UnmodifiableSet@c129199
- locked sun.nio.ch.EPollSelectorImpl@62e0b62
at

RTH_CLZ_COM_RETRIEVE-TEST_123-b0410c4e-5a81-409b-9e67-bb37ab9d1a67-StreamThread-3-
- locked sun.nio.ch.Util$3@614e567c
- locked sun.nio.ch.EPollSelectorImpl@4122f39c
at
- locked java.util.Collections$UnmodifiableSet@24cfd8ef
- locked sun.nio.ch.EPollSelectorImpl@5c686ce1
at

- locked sun.nio.ch.Util$3@a80dae3
- locked java.util.Collections$UnmodifiableSet@3134d9ef
at

- locked sun.nio.ch.Util$3@74f6116f
- locked java.util.Collections$UnmodifiableSet@782d8ad0
- locked sun.nio.ch.EPollSelectorImpl@6fc1cf9b
at

- locked sun.nio.ch.Util$3@3743dfb2
- locked java.util.Collections$UnmodifiableSet@73facd5c
- locked sun.nio.ch.EPollSelectorImpl@6715a2f9
at

- locked java.util.Collections$UnmodifiableSet@2fdee5a3
- locked sun.nio.ch.EPollSelectorImpl@a6b4d29
at

- locked sun.nio.ch.Util$3@40a334f7
- locked java.util.Collections$UnmodifiableSet@e00edc8
- locked sun.nio.ch.EPollSelectorImpl@56abf256
at

- locked sun.nio.ch.Util$3@679efaf
- locked java.util.Collections$UnmodifiableSet@5f24671b
- locked sun.nio.ch.EPollSelectorImpl@56c3af9a
at

- locked sun.nio.ch.Util$3@5f73ce56
- locked java.util.Collections$UnmodifiableSet@16be5680
- locked sun.nio.ch.EPollSelectorImpl@6adee7f3
at

- locked sun.nio.ch.Util$3@21aa3a4d
- locked java.util.Collections$UnmodifiableSet@606a4150
- locked sun.nio.ch.EPollSelectorImpl@404018b
at

- locked sun.nio.ch.Util$3@19f88f87
- locked java.util.Collections$UnmodifiableSet@69f0b6a7
- locked sun.nio.ch.EPollSelectorImpl@5b220a08
at

- locked sun.nio.ch.Util$3@7f8e50be
- locked java.util.Collections$UnmodifiableSet@22577c7f
- locked sun.nio.ch.EPollSelectorImpl@4e7ea31c
at

- locked sun.nio.ch.EPollSelectorImpl@116c08c4
at

- locked sun.nio.ch.Util$3@2a312fb6
- locked java.util.Collections$UnmodifiableSet@13db736b
at

- locked java.util.Collections$UnmodifiableSet@4066b7d8
- locked sun.nio.ch.EPollSelectorImpl@86ed494
at

- locked sun.nio.ch.Util$3@9fc9e77
- locked java.util.Collections$UnmodifiableSet@2119aab0
- locked sun.nio.ch.EPollSelectorImpl@54b4113c
at

- locked sun.nio.ch.Util$3@2a00b54a
- locked java.util.Collections$UnmodifiableSet@47e2895b
- locked sun.nio.ch.EPollSelectorImpl@2bc2e73d
at

- locked sun.nio.ch.Util$3@5f8b2e88
- locked java.util.Collections$UnmodifiableSet@708e6ad6
- locked sun.nio.ch.EPollSelectorImpl@23758dcd
at

- locked sun.nio.ch.Util$3@1fb26d91
- locked java.util.Collections$UnmodifiableSet@194bef81
- locked sun.nio.ch.EPollSelectorImpl@1a10fea4
at

- locked sun.nio.ch.Util$3@150403ce
- locked java.util.Collections$UnmodifiableSet@19d63a1a
- locked sun.nio.ch.EPollSelectorImpl@5f27e946
at

- locked sun.nio.ch.Util$3@425fc9d8
- locked java.util.Collections$UnmodifiableSet@7770a82e
- locked sun.nio.ch.EPollSelectorImpl@5fde4a30
at

- locked sun.nio.ch.Util$3@3f53e554
- locked java.util.Collections$UnmodifiableSet@3fefa6f
- locked sun.nio.ch.EPollSelectorImpl@713d03e6
at

- locked sun.nio.ch.Util$3@5c9f483b
- locked java.util.Collections$UnmodifiableSet@7c467fb9
- locked sun.nio.ch.EPollSelectorImpl@81fcfdc
at

- locked sun.nio.ch.Util$3@11a4cf0a
- locked java.util.Collections$UnmodifiableSet@37dc8395
- locked sun.nio.ch.EPollSelectorImpl@78a89960
at

- locked sun.nio.ch.Util$3@ebfd886
- locked java.util.Collections$UnmodifiableSet@39549ac4
- locked sun.nio.ch.EPollSelectorImpl@6ac77650
at

- locked java.util.Collections$UnmodifiableSet@2613184c
- locked sun.nio.ch.EPollSelectorImpl@3b0c34f0
at

- locked sun.nio.ch.Util$3@6a28b037
- locked java.util.Collections$UnmodifiableSet@782d238a
- locked sun.nio.ch.EPollSelectorImpl@46c295bb
at

- locked java.util.Collections$UnmodifiableSet@33b5209e
at

- locked sun.nio.ch.Util$3@70a852a5
- locked java.util.Collections$UnmodifiableSet@700a443d
- locked sun.nio.ch.EPollSelectorImpl@6aa13b72
at
- locked sun.nio.ch.Util$3@639d6fcb
- locked sun.nio.ch.EPollSelectorImpl@739bb9d4
at

- locked sun.nio.ch.Util$3@1002667e
- locked sun.nio.ch.EPollSelectorImpl@58188ef4
at

- locked sun.nio.ch.Util$3@76e3c93a
- locked java.util.Collections$UnmodifiableSet@7551700e
at

- locked sun.nio.ch.Util$3@3c9ae641
- locked java.util.Collections$UnmodifiableSet@54d842f1
- locked sun.nio.ch.EPollSelectorImpl@2af79964
at

- locked sun.nio.ch.Util$3@47efeabf
- locked java.util.Collections$UnmodifiableSet@2c2cb4ed
- locked sun.nio.ch.EPollSelectorImpl@3a75a453
at

- locked sun.nio.ch.Util$3@2c5f26c0
- locked java.util.Collections$UnmodifiableSet@145d8be6
- locked sun.nio.ch.EPollSelectorImpl@5d1fce5c
at

- locked sun.nio.ch.Util$3@4adfae90
- locked java.util.Collections$UnmodifiableSet@1f63f82d
- locked sun.nio.ch.EPollSelectorImpl@6719429c
at

- locked sun.nio.ch.Util$3@44d70d0d
- locked java.util.Collections$UnmodifiableSet@734b9029
- locked sun.nio.ch.EPollSelectorImpl@73e9bcfd
at

- locked sun.nio.ch.Util$3@5e490dbe
- locked java.util.Collections$UnmodifiableSet@228adc1f
- locked sun.nio.ch.EPollSelectorImpl@defcabf
at

- locked sun.nio.ch.Util$3@335f81ab
- locked java.util.Collections$UnmodifiableSet@256d5e83
at

- locked sun.nio.ch.Util$3@223a96b6
- locked java.util.Collections$UnmodifiableSet@6da70983
- locked sun.nio.ch.EPollSelectorImpl@2271dbf2
at

- locked sun.nio.ch.Util$3@6b6bb015
- locked java.util.Collections$UnmodifiableSet@1d9204e8
- locked sun.nio.ch.EPollSelectorImpl@482cedc8
at
Thread [name="GelfSenderThread-323", id=323, state=TIMED_WAITING, blockCnt=46223,

waitCnt=34725079]
at java.lang.Thread.sleep(Thread.java:342)
at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:386)
at
org.graylog2.gelfclient.util.Uninterruptibles.sleepUninterruptibly(Uninterruptibles
.java:42)
at
org.graylog2.gelfclient.transport.GelfSenderThread$1.run(GelfSenderThread.java:102)
java.util.concurrent.locks.ReentrantLock$NonfairSync@642e1174
Thread [name="gelfUdpTransport-5-1", id=324, state=RUNNABLE, blockCnt=65103,
waitCnt=0]
at java.util.zip.Deflater.deflateBytes(Native Method)
at java.util.zip.Deflater.deflate(Deflater.java:444)
- locked java.util.zip.ZStreamRef@5273eefc
at java.util.zip.Deflater.deflate(Deflater.java:366)
at java.util.zip.GZIPOutputStream.finish(GZIPOutputStream.java:159)
at
org.graylog2.gelfclient.encoder.GelfCompressionGzipEncoder.encode(GelfCompressionGz
ipEncoder.java:38)
at
org.graylog2.gelfclient.encoder.GelfCompressionGzipEncoder.encode(GelfCompressionGz
ipEncoder.java:31)
at
io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:8
8)
at
io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerC
ontext.java:738)
at
io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerCo
ntext.java:730)
at
io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.
java:816)
at
io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.
java:723)
at
io.netty.handler.codec.MessageToMessageEncoder.write(MessageToMessageEncoder.java:1
11)
at
io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerC
ontext.java:738)
at
io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerCo
ntext.java:730)
at
io.netty.channel.AbstractChannelHandlerContext.access$1900(AbstractChannelHandlerCo
ntext.java:38)
at
io.netty.channel.AbstractChannelHandlerContext$AbstractWriteTask.write(AbstractChan
nelHandlerContext.java:1081)
at
io.netty.channel.AbstractChannelHandlerContext$WriteAndFlushTask.write(AbstractChan
nelHandlerContext.java:1128)
at
io.netty.channel.AbstractChannelHandlerContext$AbstractWriteTask.run(AbstractChanne
lHandlerContext.java:1070)
at
io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.ja
va:163)
at
io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExe
cutor.java:403)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
at
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.
java:858)
at
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultT
hreadFactory.java:138)
Thread [name="kafka-admin-client-thread | test-mtn-clz-com-v0.0.88-

MTN_CLZ_COM_DELETE-TEST_123-da8a730f-6145-437e-9171-e7d733239469-admin", id=312,
- locked sun.nio.ch.Util$3@5fcb051e
- locked sun.nio.ch.EPollSelectorImpl@6497eb43
at
org.apache.kafka.clients.admin.KafkaAdminClient$AdminClientRunnable.processRequests
(KafkaAdminClient.java:1415)
at
org.apache.kafka.clients.admin.KafkaAdminClient$AdminClientRunnable.run(KafkaAdminC
lient.java:1346)

HUB_CLZ_COM_POST_PROCESS-TEST_123-f33e6e23-07a4-4150-9137-95191cea7918-admin",
id=313, state=RUNNABLE, blockCnt=65, waitCnt=53]
- locked sun.nio.ch.Util$3@29deed23
- locked java.util.Collections$UnmodifiableSet@2ebafef3
at
at
lient.java:1346)

MTN_CLZ_COM_RETRIEVE-TEST_123-ca18f767-af1e-406b-a14d-1561d4f9903e-admin", id=314,
- locked sun.nio.ch.Util$3@324183dd
- locked java.util.Collections$UnmodifiableSet@6c28c743
- locked sun.nio.ch.EPollSelectorImpl@151cf153
at
at
lient.java:1346)

DAPI_CLZ_COM_DELETE-TEST_123-1f43f117-62f8-49c6-be42-73aa89020063-admin", id=315,
- locked sun.nio.ch.Util$3@411d2e2f
- locked java.util.Collections$UnmodifiableSet@7f7c02ba
- locked sun.nio.ch.EPollSelectorImpl@55c4d0c9
at
at
lient.java:1346)
Thread [name="kafka-admin-client-thread | test-mtn-clz-com-v0.0.88-MTN_CLZ_COM-

TEST_123-68e561cb-c3c0-4fa8-a191-b39fdbe68776-admin", id=316, state=RUNNABLE,
- locked sun.nio.ch.Util$3@2720254b
- locked java.util.Collections$UnmodifiableSet@f641bd6
at
at
lient.java:1346)

HUB_CLZ_COM_BCKGRND_PROCESS-TEST_123-142d3669-5783-4079-9ccb-f5a76a7c9218-admin",
- locked sun.nio.ch.EPollSelectorImpl@6a9a1b8f
at
at
lient.java:1346)

DAPI_CLZ_COM_POST_PROCESS-TEST_123-e9aa56bf-eb0b-4e7c-a4c1-a118f8c5058a-admin",
- locked sun.nio.ch.Util$3@6f073fbf
- locked java.util.Collections$UnmodifiableSet@1794b82f
- locked sun.nio.ch.EPollSelectorImpl@2ab42d14
at
at
lient.java:1346)

HUB_CLZ_COM_DELETE-TEST_123-ed0d4009-2adb-4b5c-9a7c-c99fcf9a598b-admin", id=319,
- locked sun.nio.ch.Util$3@10481dae
- locked java.util.Collections$UnmodifiableSet@466422df
- locked sun.nio.ch.EPollSelectorImpl@7403affc
at
at
lient.java:1346)

RTH_CLZ_COM_RETRIEVE-TEST_123-b0410c4e-5a81-409b-9e67-bb37ab9d1a67-admin", id=320,
- locked sun.nio.ch.Util$3@7fd4e0e5
- locked java.util.Collections$UnmodifiableSet@2f2137dc
- locked sun.nio.ch.EPollSelectorImpl@48b48a3d
at
at
lient.java:1346)
Thread [name="kafka-admin-client-thread | test-mtn-clz-com-v0.0.88-DAPI_CLZ_COM-

TEST_123-24eee553-bd2b-4c15-99d2-aa4f0bba17e9-admin", id=321, state=RUNNABLE,
- locked java.util.Collections$UnmodifiableSet@79aabf91
- locked sun.nio.ch.EPollSelectorImpl@5f93e8af
at
at
lient.java:1346)

RTH_CLZ_COM_DELETE-TEST_123-f99ad426-bccb-4e1b-9907-4af104390725-admin", id=322,
- locked java.util.Collections$UnmodifiableSet@1f93a799
- locked sun.nio.ch.EPollSelectorImpl@3369c5cd
at
at
lient.java:1346)
Thread [name="kafka-admin-client-thread | test-mtn-clz-com-v0.0.88-RTH_CLZ_COM-

TEST_123-a865a3b0-d26a-4428-b19a-11b5f1e4111f-admin", id=307, state=RUNNABLE,
- locked sun.nio.ch.Util$3@737f959f
at
at
lient.java:1346)

RTH_CLZ_COM_POST_PROCESS-TEST_123-a0216b91-9d30-4899-8fd5-f538476d7b74-admin",
- locked java.util.Collections$UnmodifiableSet@636edc21
- locked sun.nio.ch.EPollSelectorImpl@5a5eacaf
at
at
lient.java:1346)

HUB_CLZ_COM_RETRIEVE-TEST_123-7e3e9003-4ff6-434b-81a0-bd7487790271-admin", id=309,
- locked sun.nio.ch.Util$3@fbb3057
- locked java.util.Collections$UnmodifiableSet@700cbe32
- locked sun.nio.ch.EPollSelectorImpl@6516fbc6
at
at
lient.java:1346)

DAPI_CLZ_COM_RETRIEVE-TEST_123-47b6dd5f-e68e-42cb-a64c-275f43c3bdaa-admin", id=310,
- locked sun.nio.ch.Util$3@4a02181f
- locked java.util.Collections$UnmodifiableSet@1ceda080
at
at
lient.java:1346)

RTH_CLZ_COM_BCKGRND_PROCESS-TEST_123-9b3ec258-e059-4979-a978-796e345adeb5-admin",
- locked sun.nio.ch.Util$3@5b9f623c
- locked java.util.Collections$UnmodifiableSet@1075bb0
- locked sun.nio.ch.EPollSelectorImpl@701f02b0
at
at
lient.java:1346)

MTN_CLZ_COM_BCKGRND_PROCESS-TEST_123-e01a8bfe-68c6-45e7-8ac6-4457e81b2396-admin",
- locked sun.nio.ch.Util$3@52b8e1da
- locked sun.nio.ch.EPollSelectorImpl@4977e81a
at
at
lient.java:1346)
Thread [name="kafka-admin-client-thread | test-mtn-clz-com-v0.0.88-HUB_CLZ_COM-

TEST_123-c3c77b25-6c36-4d2a-9782-07837f90f8f4-admin", id=305, state=RUNNABLE,
- locked java.util.Collections$UnmodifiableSet@54d6f6d3
- locked sun.nio.ch.EPollSelectorImpl@113cbc05
at
at
lient.java:1346)

DAPI_CLZ_COM_BCKGRND_PROCESS-TEST_123-23d6704e-4428-452e-a8ba-2f2545a2eb9d-admin",
- locked sun.nio.ch.Util$3@677db100
- locked java.util.Collections$UnmodifiableSet@daac9a0
- locked sun.nio.ch.EPollSelectorImpl@6ea9d6a8
at
at
lient.java:1346)

MTN_CLZ_COM_POST_PROCESS-TEST_123-f06ae088-7a74-4591-897a-963cbd18b620-admin",
- locked sun.nio.ch.Util$3@1a8183ec
- locked java.util.Collections$UnmodifiableSet@10fec40d
- locked sun.nio.ch.EPollSelectorImpl@4bc2f87b
at
at
lient.java:1346)

Lock [object=java.util.concurrent.CountDownLatch$Sync@184c889e, ownerName=null,
ownerId=-1]
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(Abstrac
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(
AbstractQueuedSynchronizer.java:997)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(Ab
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at
com.clouds.stream.core.abstraction.AbstractKafkaStream.startKafkaStream(AbstractKaf
kaStream.java:117)
at
com.clouds.stream.core.abstraction.AbstractKafkaStream.startStream(AbstractKafkaStr
eam.java:67)
at
com.clouds.stream.core.processor.kafka.KafkaIgniteComputeStreamInvoker.lambda$invok
e$2(KafkaIgniteComputeStreamInvoker.java:85)
at com.clouds.stream.core.processor.kafka.KafkaIgniteComputeStreamInvoker$
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@58eebe6b
Lock [object=java.util.concurrent.CountDownLatch$Sync@21ef22f8, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@6e074366
Lock [object=java.util.concurrent.CountDownLatch$Sync@7c37d984, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@42b9bccb
Lock [object=java.util.concurrent.CountDownLatch$Sync@415f38d9, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@3d8084fc
Lock [object=java.util.concurrent.CountDownLatch$Sync@3d2b4014, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@33a0aff2
Lock [object=java.util.concurrent.CountDownLatch$Sync@650b5dab, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@5cfe0d8f
Lock [object=java.util.concurrent.CountDownLatch$Sync@27324377, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@178d097e
Lock [object=java.util.concurrent.CountDownLatch$Sync@1f132eb8, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@2075cb79
Lock [object=java.util.concurrent.CountDownLatch$Sync@785ff6ae, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@5d7658ea
Lock [object=java.util.concurrent.CountDownLatch$Sync@47060a0b, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@1c95e2e
Lock [object=java.util.concurrent.CountDownLatch$Sync@729ad067, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@b079910
Lock [object=java.util.concurrent.CountDownLatch$Sync@74fa4662, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@180cbe5b
Lock [object=java.util.concurrent.CountDownLatch$Sync@731734d5, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@5cb2518
Lock [object=java.util.concurrent.CountDownLatch$Sync@4362eee, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@3dd6e25c
Lock [object=java.util.concurrent.CountDownLatch$Sync@200bcfe7, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@6f7f9ce6
Lock [object=java.util.concurrent.CountDownLatch$Sync@6e444d2, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@593193b
Lock [object=java.util.concurrent.CountDownLatch$Sync@4c1f7947, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@71c92d8b
Lock [object=java.util.concurrent.CountDownLatch$Sync@796b7f74, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@1d79e05d
Lock [object=java.util.concurrent.CountDownLatch$Sync@7e1e349, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@19d636c
Lock [object=java.util.concurrent.CountDownLatch$Sync@2387f5b3, ownerName=null,
ownerId=-1]
at
at
at
at
kaStream.java:117)
at
eam.java:67)
at
at
at
java.util.concurrent.ThreadPoolExecutor$Worker@3ceec0d
Lock
f5f, ownerName=null, ownerId=-1]
at
at
at
at
at
Thread [name="Timer-1", id=276, state=TIMED_WAITING, blockCnt=7515, waitCnt=8119]

Lock [object=java.util.TaskQueue@146575c5, ownerName=null, ownerId=-1]
Thread [name="Timer-0", id=275, state=TIMED_WAITING, blockCnt=7529, waitCnt=8120]

Lock [object=java.util.TaskQueue@365728e2, ownerName=null, ownerId=-1]
Thread [name="grid-nio-worker-thin-client-channel-0-#67%thinClient%", id=271,

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@6b7e7b3d
- locked java.util.Collections$UnmodifiableSet@7df61b5d
- locked sun.nio.ch.EPollSelectorImpl@1b4e7642
at
o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.j
ava:2264)
at
o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910
)
Thread [name="kafka-admin-client-thread | JavaClient1692339910922", id=269,

- locked java.util.Collections$UnmodifiableSet@5fac683c
- locked sun.nio.ch.EPollSelectorImpl@12cdac67
at
at
lient.java:1346)
Thread [name="technology-connection-metrics-thread", id=268, state=TIMED_WAITING,

Lock [object=java.util.TaskQueue@f8fe9d8, ownerName=null, ownerId=-1]

Lock [object=java.util.TaskQueue@4470b320, ownerName=null, ownerId=-1]

Lock [object=java.util.TaskQueue@4184eae8, ownerName=null, ownerId=-1]
Thread [name="kafka-admin-client-thread | adminclient-1", id=263, state=RUNNABLE,

- locked sun.nio.ch.Util$3@7ad9241a
- locked java.util.Collections$UnmodifiableSet@27b4dff5
- locked sun.nio.ch.EPollSelectorImpl@1be5beb7
at
at
lient.java:1346)

Lock [object=java.util.TaskQueue@16d1d271, ownerName=null, ownerId=-1]
Thread [name="kafka-producer-network-thread | javaproducer11693317516383", id=256,

- locked java.util.Collections$UnmodifiableSet@57fe9640
- locked sun.nio.ch.EPollSelectorImpl@5634b03d
at
Thread [name="flusher-0-#66", id=253, state=TIMED_WAITING, blockCnt=0,

waitCnt=53739]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44b70
bdc, ownerName=null, ownerId=-1]
at
at
o.a.i.i.processors.cache.store.GridCacheWriteBehindStore$Flusher.awaitOperationsAva
ilableCoalescing(GridCacheWriteBehindStore.java:1110)
at
o.a.i.i.processors.cache.store.GridCacheWriteBehindStore$Flusher.body(GridCacheWrit
eBehindStore.java:1084)

waitCnt=10733566]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ff3f
c2f, ownerName=null, ownerId=-1]
at
at
at

waitCnt=10731255]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ff3f
c2f, ownerName=null, ownerId=-1]
at
at
at

waitCnt=10733457]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3df
at
at
at

waitCnt=10737073]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3df
at
at
at

waitCnt=26583289]
Lock
a0f, ownerName=null, ownerId=-1]
at
at
at

waitCnt=26570265]
Lock
a0f, ownerName=null, ownerId=-1]
at
at
at
Thread [name="prometheus-http-1-5", id=208, state=WAITING, blockCnt=5678,

waitCnt=28083]
Lock
3e3, ownerName=null, ownerId=-1]
at
at
at
at
at

waitCnt=28057]
Lock
at
at
at
at
at

waitCnt=28076]
Lock
at
at
at
at
at

waitCnt=28079]
Lock
at
at
at
at
at
Thread [name="RPCClient-NioEventLoopGroup-1-2", id=119, state=RUNNABLE,

- locked
org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet@6092ac4a
- locked java.util.Collections$UnmodifiableSet@205fea37
- locked sun.nio.ch.EPollSelectorImpl@33034ea
at
org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.se
lect(SelectedSelectionKeySetSelector.java:62)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.j
ava:807)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java
:457)
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.ru
n(SingleThreadEventExecutor.java:989)
at
org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadEx
ecutorMap.java:74)
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(Fa
stThreadLocalRunnable.java:30)
Thread [name="RpcClient-timer-pool1-t1", id=111, state=TIMED_WAITING, blockCnt=0,

waitCnt=26658264]
at
org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(H
ashedWheelTimer.java:569)
at
org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTi
mer.java:465)
Thread [name="RPCClient-NioEventLoopGroup-1-1", id=117, state=RUNNABLE,

- locked
org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet@7076d9e9
- locked java.util.Collections$UnmodifiableSet@5f3cb51
- locked sun.nio.ch.EPollSelectorImpl@5a22878a
at
org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.se
lect(SelectedSelectionKeySetSelector.java:62)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.j
ava:807)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java
:457)
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.ru
n(SingleThreadEventExecutor.java:989)
at
org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadEx
ecutorMap.java:74)
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(Fa
stThreadLocalRunnable.java:30)

waitCnt=28074]
Lock
at
at
at
at
at

waitCnt=53748]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59eb8
at
at
at
Thread [name="Idle-Rpc-Conn-Sweeper-pool2-t1", id=112, state=TIMED_WAITING,

Lock
97a, ownerName=null, ownerId=-1]
at
at
at
at
at
at
Thread [name="ReadOnlyZKClient-
192.168.24.21:2181,192.168.24.22:2181,192.168.24.23:2181,192.168.24.25:2181,192.168
.24.65:2181@0x7a6b33f1", id=108, state=TIMED_WAITING, blockCnt=4, waitCnt=4497]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44c8e
at
at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
at
org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:326)
at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$
Thread [name="flusher-1-#57", id=107, state=WAITING, blockCnt=21406,

waitCnt=10739106]
Lock [object=org.apache.hadoop.hbase.ipc.BlockingRpcCallback@58377f40,
at
org.apache.hadoop.hbase.ipc.BlockingRpcCallback.get(BlockingRpcCallback.java:62)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.
java:326)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:89)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.call
BlockingMethod(AbstractRpcClient.java:575)
at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Blocki
ngStub.mutate(ClientProtos.java:45378)
at
org.apache.hadoop.hbase.client.ClientServiceCallable.doMutate(ClientServiceCallable
.java:55)
at org.apache.hadoop.hbase.client.HTable$3.rpcCall(HTable.java:535)
at org.apache.hadoop.hbase.client.HTable$3.rpcCall(HTable.java:530)
at
org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:
127)
at
org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCal
lerImpl.java:107)
at org.apache.hadoop.hbase.client.HTable.put(HTable.java:539)
at
com.clouds.ignite.hbase.IgniteHbaseUtility.insertInHBase(IgniteHbaseUtility.java:15
0)
at
com.cloud.ignite.cache.store.CalMailCacheStoreAdapter.write(CalMailCacheStoreAdapte
r.java:62)
at o.a.i.cache.store.CacheStoreAdapter.writeAll(CacheStoreAdapter.java:76)
at
o.a.i.i.processors.cache.store.GridCacheWriteBehindStore.updateStore(GridCacheWrite
BehindStore.java:883)
at
o.a.i.i.processors.cache.store.GridCacheWriteBehindStore.applyBatch(GridCacheWriteB
ehindStore.java:777)
at
o.a.i.i.processors.cache.store.GridCacheWriteBehindStore.access$2700(GridCacheWrite
BehindStore.java:77)
at
o.a.i.i.processors.cache.store.GridCacheWriteBehindStore$Flusher.flushCacheCoalesci
ng(GridCacheWriteBehindStore.java:1220)
at

waitCnt=10731575]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@675b7
d40, ownerName=null, ownerId=-1]
at
at
at

waitCnt=53749]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7ee60
at
at
at
Thread [name="ttl-cleanup-worker-#52", id=102, state=TIMED_WAITING, blockCnt=549,
waitCnt=536652]
at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:8270)
at
o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCa
cheSharedTtlCleanupManager.java:216)
Thread [name="session-timeout-worker-#29", id=53, state=TIMED_WAITING, blockCnt=0,

waitCnt=268306]
at
o.a.i.i.processors.rest.GridRestProcessor$4.body(GridRestProcessor.java:512)
Thread [name="services-deployment-worker-#51", id=100, state=WAITING, blockCnt=0,

waitCnt=5]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3693d
b0, ownerName=null, ownerId=-1]
at
at
at
o.a.i.i.processors.service.ServiceDeploymentManager$ServicesDeploymentWorker.body(S
erviceDeploymentManager.java:466)
Thread [name="grid-nio-worker-client-listener-3-#28", id=52, state=RUNNABLE,

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@1ec84c3e
- locked java.util.Collections$UnmodifiableSet@57ab5f5
- locked sun.nio.ch.EPollSelectorImpl@4b2a6c07
at
ava:2264)
at
)

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@2884c75d
- locked java.util.Collections$UnmodifiableSet@74f1c7cf
at
ava:2264)
at
)

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@3ca61b64
- locked java.util.Collections$UnmodifiableSet@32969d82
- locked sun.nio.ch.EPollSelectorImpl@48a3eba0
at
ava:2264)
at
)

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@4c5e42c4
- locked java.util.Collections$UnmodifiableSet@40af0d10
- locked sun.nio.ch.EPollSelectorImpl@4653e20a
at
ava:2264)
at
)
Thread [name="nio-acceptor-client-listener-#50", id=99, state=RUNNABLE, blockCnt=0,

waitCnt=0]
- locked sun.nio.ch.Util$3@117aff8b
- locked java.util.Collections$UnmodifiableSet@2f8cc702
- locked sun.nio.ch.EPollSelectorImpl@17f3886e
at
o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3074)
at
o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3022)
Thread [name="exchange-worker-#39", id=88, state=TIMED_WAITING, blockCnt=2,

waitCnt=53816]
Lock
at
at
java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522)
at
java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(Gri
dCachePartitionExchangeManager.java:3252)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(Grid
CachePartitionExchangeManager.java:3181)
Thread [name="ignite-update-notifier-timer", id=87, state=TIMED_WAITING,

Lock [object=java.util.TaskQueue@30eb44ea, ownerName=null, ownerId=-1]
Thread [name="upd-ver-checker", id=86, state=TIMED_WAITING, blockCnt=0,

waitCnt=53716]
at
o.a.i.i.processors.cluster.GridUpdateNotifier$1.run(GridUpdateNotifier.java:115)
Thread [name="disco-event-worker-#37", id=84, state=WAITING, blockCnt=10,

waitCnt=134685]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@38ed2
f4c, ownerName=null, ownerId=-1]
at
at
at
o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscovery
Manager.java:3105)
at
o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryM
anager.java:3076)
Thread [name="tcp-disco-ip-finder-cleaner-#4-#36", id=83, state=TIMED_WAITING,

at
o.a.i.spi.discovery.tcp.ServerImpl$IpFinderCleaner.body(ServerImpl.java:2241)
Thread [name="tcp-disco-srvr-[:47500]-#3-#35", id=80, state=RUNNABLE, blockCnt=0,

waitCnt=0]
at java.net.PlainSocketImpl.socketAccept(Native Method)
at
java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
at java.net.ServerSocket.implAccept(ServerSocket.java:560)
at java.net.ServerSocket.accept(ServerSocket.java:528)
at o.a.i.i.util.IgniteUtils.acceptServerSocket(IgniteUtils.java:12343)
at o.a.i.spi.discovery.tcp.ServerImpl$TcpServer.body(ServerImpl.java:6605)
at
o.a.i.spi.discovery.tcp.ServerImpl$TcpServerThread.body(ServerImpl.java:6528)
Thread [name="tcp-disco-msg-worker-[e327c48f 10.233.88.1:47500 crd]-#2-#34", id=79,

Lock
4d9, ownerName=null, ownerId=-1]
at
at
java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522)
at
java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684)
at
o.a.i.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:8053)
at
o.a.i.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:3089)
at
o.a.i.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7989)
Thread [name="disco-notifier-worker-#33", id=78, state=WAITING, blockCnt=8,

waitCnt=134877]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ad38
52a, ownerName=null, ownerId=-1]
at
at
at
o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body
0(GridDiscoveryManager.java:2846)
at
o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body
(GridDiscoveryManager.java:2891)
Thread [name="grid-data-loader-flusher-#32", id=77, state=WAITING, blockCnt=0,

waitCnt=1]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5acb5
at
at java.util.concurrent.DelayQueue.take(DelayQueue.java:211)
at
o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java
:109)
Thread [name="qtp1740687627-76", id=76, state=TIMED_WAITING, blockCnt=0,

waitCnt=7064]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@511c0
35d, ownerName=null, ownerId=-1]
at
at
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.
java:875)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925
)

waitCnt=6971]
Lock
at
at
at
java:875)
at
)

waitCnt=6973]
Lock
at
at
at
java:875)
at
)

waitCnt=6842]
Lock
at
at
at
java:875)
at
)

waitCnt=7011]
Lock
at
at
at
java:875)
at
)

waitCnt=6991]
Lock
at
at
at
java:875)
at
)

waitCnt=7008]
Lock
at
at
at
java:875)
at
)

waitCnt=7043]
Lock
at
at
at
java:875)
at
)

waitCnt=6948]
Lock
at
at
at
java:875)
at
)

waitCnt=6768]
Lock
at
at
at
java:875)
at
)

waitCnt=7021]
Lock
at
at
at
java:875)
at
)

waitCnt=7030]
Lock
at
at
at
java:875)
at
)

waitCnt=6934]
Lock
at
at
at
java:875)
at
)

waitCnt=7041]
Lock
at
at
at
java:875)
at
)

waitCnt=6949]
Lock
at
at
at
java:875)
at
)

waitCnt=7172]
Lock
at
at
at
java:875)
at
)
waitCnt=7045]
Lock
at
at
at
java:875)
at
)

waitCnt=6740]
Lock
at
at
at
java:875)
at
)
Thread [name="qtp1740687627-58-acceptor-0@1c87699d-ServerConnector@71262020{HTTP/
1.1, (http/1.1)}{0.0.0.0:8080}", id=58, state=RUNNABLE, blockCnt=0, waitCnt=1]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
at
- locked java.lang.Object@4e696b28
at
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at
)
Thread [name="qtp1740687627-57", id=57, state=RUNNABLE, blockCnt=0, waitCnt=1]
- locked sun.nio.ch.Util$3@8a943c9
- locked sun.nio.ch.EPollSelectorImpl@4aef61b5
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:6
06)
at
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:
543)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.ja
va:360)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java
:184)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.jav
a:171)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:1
35)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$537/1468559919.run(Unknown
Source)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at
)
Thread [name="grid-nio-worker-tcp-rest-0-#30", id=54, state=RUNNABLE, blockCnt=0,

waitCnt=0]
- locked o.a.i.i.util.nio.SelectedSelectionKeySet@4fda73eb
- locked java.util.Collections$UnmodifiableSet@d2fc6d2
- locked sun.nio.ch.EPollSelectorImpl@6d93f217
at
ava:2264)
at
)
Thread [name="nio-acceptor-tcp-rest-#31", id=55, state=RUNNABLE, blockCnt=0,

waitCnt=0]
- locked sun.nio.ch.Util$3@5c40c4d9
- locked java.util.Collections$UnmodifiableSet@4099b191
- locked sun.nio.ch.EPollSelectorImpl@33f3f606
at
at
Thread [name="long-qry-#24", id=48, state=TIMED_WAITING, blockCnt=0,

waitCnt=268310]
at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:8270)
at
o.a.i.i.processors.query.h2.LongRunningQueryManager$1.body(LongRunningQueryManager.
java:93)
Thread [name="page-lock-tracker-timeout", id=44, state=TIMED_WAITING, blockCnt=0,

waitCnt=4491]
at o.a.i.i.util.worker.CycleThread.run(CycleThread.java:47)
Thread [name="tcp-comm-worker-#1-#21", id=41, state=TIMED_WAITING, blockCnt=0,

waitCnt=450]
Lock
3bc, ownerName=null, ownerId=-1]
at
at
at
o.a.i.spi.communication.tcp.internal.CommunicationWorker.body(CommunicationWorker.j
ava:167)
at
o.a.i.spi.communication.tcp.TcpCommunicationSpi$3.body(TcpCommunicationSpi.java:848
)
Thread [name="grid-nio-worker-tcp-comm-3-#19%TcpCommunicationSpi%", id=39,

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@4182b925
- locked java.util.Collections$UnmodifiableSet@1b3edbc0
- locked sun.nio.ch.EPollSelectorImpl@16a6e40f
at
ava:2264)
at
)

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@3a49631b
- locked java.util.Collections$UnmodifiableSet@6e56aee6
- locked sun.nio.ch.EPollSelectorImpl@3db8837
at
ava:2264)
at
)

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@227c82df
- locked java.util.Collections$UnmodifiableSet@53dc7c30
- locked sun.nio.ch.EPollSelectorImpl@8a6c933
at
ava:2264)
at
)

- locked o.a.i.i.util.nio.SelectedSelectionKeySet@57636220
- locked sun.nio.ch.EPollSelectorImpl@21c2bb8c
at
ava:2264)
at
)
Thread [name="nio-acceptor-tcp-comm-#20%TcpCommunicationSpi%", id=40,

- locked sun.nio.ch.Util$3@1f1311d6
- locked java.util.Collections$UnmodifiableSet@532d1276
- locked sun.nio.ch.EPollSelectorImpl@39cbb44a
at
at
Thread [name="grid-timeout-worker-#15", id=35, state=TIMED_WAITING, blockCnt=231,

waitCnt=968714]
Lock [object=java.lang.Object@7b1e0523, ownerName=null, ownerId=-1]
at
o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProce
ssor.java:269)
Thread [name="data-streamer-stripe-0-#9", id=29, state=WAITING, blockCnt=0,

waitCnt=1]
at
o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819)
at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623)
Thread [name="sys-stripe-7-#8", id=28, state=WAITING, blockCnt=25, waitCnt=211801]

at
Thread [name="sys-stripe-6-#7", id=27, state=RUNNABLE, blockCnt=32, waitCnt=212050]

at sun.management.ThreadImpl.dumpThreads0(Native Method)
at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:496)
at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:484)
at o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1518)
at
o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:205)
- locked o.a.i.i.processors.failure.FailureProcessor@d616624
at
o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156)
at o.a.i.i.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1735)
at o.a.i.i.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1725)
at o.a.i.i.worker.WorkersRegistry.onIdle(WorkersRegistry.java:232)
at o.a.i.i.util.worker.GridWorker.onIdle(GridWorker.java:299)

at

at

at

at

at
Thread [name="sys-stripe-0-#1", id=21, state=RUNNABLE, blockCnt=38, waitCnt=217078]
at
o.a.i.i.MarshallerContextImpl.getClassName(MarshallerContextImpl.java:392)
at o.a.i.i.MarshallerContextImpl.getClass(MarshallerContextImpl.java:379)
at o.a.i.i.binary.BinaryContext.descriptorForTypeId(BinaryContext.java:695)
at
o.a.i.i.binary.BinaryReaderExImpl.deserialize0(BinaryReaderExImpl.java:1772)
at
o.a.i.i.binary.BinaryReaderExImpl.deserialize(BinaryReaderExImpl.java:1731)
at
o.a.i.i.binary.BinaryObjectImpl.deserializeValue(BinaryObjectImpl.java:822)
at o.a.i.i.binary.BinaryObjectImpl.value(BinaryObjectImpl.java:151)
at
o.a.i.i.processors.cache.CacheObjectUtils.unwrapBinary(CacheObjectUtils.java:199)
at
o.a.i.i.processors.cache.CacheObjectUtils.unwrapBinaryIfNeeded(CacheObjectUtils.jav
a:78)
at
o.a.i.i.processors.cache.CacheObjectContext.unwrapBinaryIfNeeded(CacheObjectContext
.java:138)
at
o.a.i.i.processors.cache.GridCacheEventManager.addEvent(GridCacheEventManager.java:
306)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerSet(GridCacheMapEntry.java:1586)
at
o.a.i.i.processors.cache.distributed.GridDistributedTxRemoteAdapter.commitIfLocked(
GridDistributedTxRemoteAdapter.java:648)
at
o.a.i.i.processors.cache.distributed.GridDistributedTxRemoteAdapter.commitRemoteTx(
GridDistributedTxRemoteAdapter.java:860)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.finish(IgniteTxHandler.java:1
468)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishRequest(Ign
iteTxHandler.java:1390)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler.access$600(IgniteTxHandler.ja
va:135)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$7.apply(IgniteTxHandler.java:
259)
at
o.a.i.i.processors.cache.transactions.IgniteTxHandler$7.apply(IgniteTxHandler.java:
257)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:
1151)
at
o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:592)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:3
93)
at
o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:3
19)
at
o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:110)
at
o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:309
)
at
o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1907
)
at
o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.j
ava:1528)
at
o.a.i.i.managers.communication.GridIoManager.access$5300(GridIoManager.java:243)
at
o.a.i.i.managers.communication.GridIoManager$9.execute(GridIoManager.java:1421)
at o.a.i.i.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
java.util.concurrent.locks.ReentrantLock$NonfairSync@afb5bff
Thread [name="ignite-clock", id=20, state=TIMED_WAITING, blockCnt=0,
waitCnt=26358798]
at o.a.i.i.util.IgniteUtils$25.run(IgniteUtils.java:3563)
Thread [name="jvm-pause-detector-worker", id=19, state=TIMED_WAITING, blockCnt=0,

waitCnt=5328174]
at o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:124)

waitCnt=2671790]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@33ad0
at
at
at
java.util.concurrent.locks.ReentrantLock$NonfairSync@31fe4674
Thread [name="gelfUdpTransport-4-1", id=18, state=RUNNABLE, blockCnt=2, waitCnt=0]
- locked io.netty.channel.nio.SelectedSelectionKeySet@5573dde5
- locked sun.nio.ch.EPollSelectorImpl@3d14c11
at
io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySet
Selector.java:62)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:753)
at
java:858)
at
Thread [name="threadDeathWatcher-3-1", id=16, state=TIMED_WAITING, blockCnt=0,

waitCnt=268326]
at
io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
at

waitCnt=2671301]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5d96d
ccd, ownerName=null, ownerId=-1]
at
at
at
java.util.concurrent.locks.ReentrantLock$NonfairSync@405f77f9
Thread [name="gelfUdpTransport-1-1", id=15, state=RUNNABLE, blockCnt=7, waitCnt=0]
- locked io.netty.channel.nio.SelectedSelectionKeySet@b0806e2
- locked sun.nio.ch.EPollSelectorImpl@7fe1d93d
at
io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySet
Selector.java:62)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:753)
at
java:858)
at
Thread [name="Thread-3", id=10, state=RUNNABLE, blockCnt=24, waitCnt=0]

- locked java.util.Collections$UnmodifiableSet@f4ea8f4
- locked sun.nio.ch.EPollSelectorImpl@34bd5b84
at sun.net.httpserver.ServerImpl$Dispatcher.run(ServerImpl.java:352)
Thread [name="server-timer1", id=8, state=TIMED_WAITING, blockCnt=1,

waitCnt=268323]
Lock [object=java.util.TaskQueue@562afd41, ownerName=null, ownerId=-1]
Thread [name="server-timer", id=7, state=TIMED_WAITING, blockCnt=0, waitCnt=26904]

Lock [object=java.util.TaskQueue@53392625, ownerName=null, ownerId=-1]
Thread [name="Signal Dispatcher", id=5, state=RUNNABLE, blockCnt=0, waitCnt=0]
Thread [name="Finalizer", id=3, state=WAITING, blockCnt=1963769, waitCnt=18482]

Lock [object=java.lang.ref.ReferenceQueue$Lock@3fb6a447, ownerName=null,
ownerId=-1]
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216)
Thread [name="Reference Handler", id=2, state=WAITING, blockCnt=36513,

waitCnt=18880]
Lock [object=java.lang.ref.Reference$Lock@79b4d0f, ownerName=null, ownerId=-1]
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

Thread Dump MTN

Uploaded by

Copyright:

Available Formats

You might also like

Thread Dump MTN

Uploaded by

Document Information

Original Description:

Copyright

Available Formats

Share this document

Share or Embed Document

Sharing Options

Did you find this document useful?

Is this content inappropriate?

Copyright:

Available Formats

Thread Dump MTN

Uploaded by

Copyright:

Available Formats

org.apache.ignite.internal.processors.failure.

FailureProcessor - Thread dump at

Thread [name="hconnection-0x4fef5792-shared-pool3-t84055", id=128685,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84054", id=128684,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84053", id=128683,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84052", id=128682,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84051", id=128680,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84050", id=128678,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84049", id=128677,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84048", id=128676,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84047", id=128675,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84046", id=128674,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84045", id=128673,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84044", id=128672,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84043", id=128671,

Thread [name="hconnection-0x5a455ca3-shared-pool6-t6677", id=128670,

Thread [name="phoenix-1-thread-3231", id=128669, state=TIMED_WAITING, blockCnt=1,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84042", id=128668,

Thread [name="hconnection-0x5a455ca3-shared-pool6-t6676", id=128667,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84041", id=128666,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84040", id=128665,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84039", id=128664,

Thread [name="hconnection-0x5a455ca3-shared-pool6-t6675", id=128663,

Thread [name="phoenix-1-thread-3230", id=128662, state=TIMED_WAITING, blockCnt=1,

Thread [name="hconnection-0x5a455ca3-shared-pool6-t6674", id=128661,

Thread [name="hconnection-0x4fef5792-shared-pool3-t84037", id=128659,

Thread [name="pub-#26541", id=128605, state=WAITING, blockCnt=47, waitCnt=56]

Thread [name="pub-#26538", id=128597, state=TIMED_WAITING, blockCnt=54, waitCnt=60]

Thread [name="pub-#26536", id=128586, state=TIMED_WAITING, blockCnt=134,

Thread [name="pub-#26535", id=128576, state=TIMED_WAITING, blockCnt=41, waitCnt=54]

Thread [name="pub-#26534", id=128575, state=WAITING, blockCnt=50, waitCnt=52]

Thread [name="sys-#26447", id=127651, state=TIMED_WAITING, blockCnt=97,

Thread [name="sys-#26446", id=127650, state=TIMED_WAITING, blockCnt=105,

Thread [name="sys-#26445", id=127649, state=TIMED_WAITING, blockCnt=113,

Thread [name="sys-#26444", id=127648, state=TIMED_WAITING, blockCnt=110,

Thread [name="sys-#26442", id=127646, state=TIMED_WAITING, blockCnt=96,

Thread [name="sys-#26412", id=127263, state=TIMED_WAITING, blockCnt=143,

Thread [name="sys-#26386", id=127024, state=TIMED_WAITING, blockCnt=156,

Thread [name="pool-8-thread-4", id=5859, state=WAITING, blockCnt=50, waitCnt=75]

Thread [name="pool-8-thread-3", id=4363, state=WAITING, blockCnt=50, waitCnt=75]

Thread [name="pool-8-thread-2", id=3462, state=WAITING, blockCnt=50, waitCnt=75]

Thread [name="pool-8-thread-1", id=2381, state=WAITING, blockCnt=50, waitCnt=75]

Thread [name="org.apache.hadoop.hdfs.PeerCache@35d31883", id=1626,

Thread [name="MQTT Ping: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1048,

Thread [name="MQTT Call: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1047,

Thread [name="MQTT Snd: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1046,

Thread [name="MQTT Rec: BACKEND-TEST-MTN_CLZ_COM-PUBLISHER-v1.0.92", id=1045,

Thread [name="hive-consumer-health-checkup-thread", id=812, state=TIMED_WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-9", id=624, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-10", id=623, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-7", id=622, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-8", id=621, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-6", id=620, state=TIMED_WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-5", id=619, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-4", id=618, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-3", id=617, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-2", id=616, state=WAITING,

Thread [name="PHOENIX-SCANNER-RENEW-LEASE-thread-1", id=615, state=WAITING,