Hi Nikolai,
Below is the output of uname (Linux with Kernel Version) -
[vraman@xxxx ~]$ uname -a
Linux xxxx 2.6.32-573.7.1.el6.x86_64 #1 SMP Thu Sep 10 13:42:16 EDT 2015
x86_64 x86_64 x86_64 GNU/Linux
/usr/bin/java -version
java version "1.7.0_67"
Java(TM) SE Runtime Environment (build 1.7.0_67-b01)
Java HotSpot(TM) 64-Bit Server VM (build 24.65-b04, mixed mode)
Ignite Version - 2.0.0
Following is my configuration -
-----------------------------------------------
val cacheCfg = new CacheConfiguration()
cacheCfg.setName(name)
cacheCfg.setCacheMode(CacheMode.PARTITIONED)
cacheCfg.setBackups(2)
cacheCfg.setAtomicityMode(CacheAtomicityMode.ATOMIC)
cacheCfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new
Duration(TimeUnit.SECONDS, cacheExpiryTime)))
cacheCfg.setOnheapCacheEnabled(true)
cacheCfg.setEvictionPolicy(new
FifoEvictionPolicy(cacheSize).setMaxMemorySize(maxMemorySize))
val configuration = new IgniteConfiguration()
val tcpDisco = new TcpDiscoverySpi
val zkIpFinder = new TcpDiscoveryZookeeperIpFinder()
zkIpFinder.setZkConnectionString(zkUrl)
zkIpFinder.setAllowDuplicateRegistrations(true)
tcpDisco.setIpFinder(zkIpFinder)
configuration.setDiscoverySpi(tcpDisco)
configuration.setCacheConfiguration(cacheCfg)
configuration.setMarshaller(new OptimizedMarshaller().setPoolSize(10))
-------------------------------
After the the stack trace i mentioned, all the cache.put() calls were
blocked. Also noticed bunch of warnings
after the initial stack trace.
WARNING: >>> Possible starvation in striped pool.
Thread name: sys-stripe-0-#1%null%
Queue: [Message closure [msg=GridIoMessage [plc=2,
topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0,
skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest
[key=KeyCacheObjectImpl [part=730, val=null, hasValBytes=true],
val=CacheObjectImpl [val=null, hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false,
nearNodeId=null, nearFutId=0, flags=]]]], Message closure
[msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8,
ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl
[part=730, val=null, hasValBytes=true], val=CacheObjectImpl [val=null,
hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false,
nearNodeId=null, nearFutId=0, flags=]]]],
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout@61cc0eca,
Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE,
topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl
[part=860, val=null, hasValBytes=true], val=CacheObjectImpl [val=null,
hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false,
nearNodeId=null, nearFutId=0, flags=]]]], Message closure
[msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8,
ordered=false, timeout=0, skipOnTimeout=false,
msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl
[part=860, val=null, hasValBytes=true], val=CacheObjectImpl [val=null,
hasValBytes=true], prevVal=null,
super=GridDhtAtomicAbstractUpdateRequest [onRes=false,
nearNodeId=null, nearFutId=0, flags=]]]]]
Deadlock: false
Completed: 714
Thread [name="sys-stripe-0-#1%null%", id=110, state=WAITING,
blockCnt=0, waitCnt=459]
Lock
[object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@562ce034,
ownerName=null, ownerId=-1]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at
o.a.i.i.util.OffheapReadWriteLock.waitAcquireWriteLock(OffheapReadWriteLock.java:481)
at
o.a.i.i.util.OffheapReadWriteLock.writeLock(OffheapReadWriteLock.java:226)
at
o.a.i.i.pagemem.impl.PageMemoryNoStoreImpl.writeLock(PageMemoryNoStoreImpl.java:471)
at
o.a.i.i.processors.cache.database.tree.util.PageHandler.writeLock(PageHandler.java:400)
at
o.a.i.i.processors.cache.database.DataStructure.writeLock(DataStructure.java:155)
at
o.a.i.i.processors.cache.database.freelist.PagesList.writeLockPage(PagesList.java:956)
at
o.a.i.i.processors.cache.database.freelist.PagesList.takeEmptyPage(PagesList.java:989)
at
o.a.i.i.processors.cache.database.freelist.FreeListImpl.insertDataRow(FreeListImpl.java:475)
at o.a.i.i.processors.cache.database.RowStore.addRow(RowStore.java:72)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.createRow(IgniteCacheOffheapManagerImpl.java:978)
at
o.a.i.i.processors.cache.GridCacheMapEntry$AtomicCacheUpdateClosure.update(GridCacheMapEntry.java:4428)
at
o.a.i.i.processors.cache.GridCacheMapEntry$AtomicCacheUpdateClosure.call(GridCacheMapEntry.java:4226)
at
o.a.i.i.processors.cache.GridCacheMapEntry$AtomicCacheUpdateClosure.call(GridCacheMapEntry.java:3966)
at
o.a.i.i.processors.cache.database.tree.BPlusTree$Invoke.invokeClosure(BPlusTree.java:2966)
at
o.a.i.i.processors.cache.database.tree.BPlusTree$Invoke.access$6200(BPlusTree.java:2860)
at
o.a.i.i.processors.cache.database.tree.BPlusTree.invokeDown(BPlusTree.java:1696)
at
o.a.i.i.processors.cache.database.tree.BPlusTree.invoke(BPlusTree.java:1585)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.invoke(IgniteCacheOffheapManagerImpl.java:925)
at
o.a.i.i.processors.cache.IgniteCacheOffheapManagerImpl.invoke(IgniteCacheOffheapManagerImpl.java:326)
at
o.a.i.i.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:1693)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3120)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:127)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$8.apply(GridDhtAtomicCache.java:319)
at
o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$8.apply(GridDhtAtomicCache.java:314)
at
o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:863)
Please let me know if you need more details.
Regards,
Venkat
On Tue, Jul 4, 2017 at 10:36 AM, Nikolai Tikhonov <ntikho...@apache.org>
wrote:
> Hello,
>
> Could you share more information about your setup? Which version OS and
> java did you use?
>
> On Tue, Jul 4, 2017 at 10:00 AM, Venkat Raman <vra...@gmail.com> wrote:
>
>> Hi All,
>>
>> I am using Ignite cache on two node cluster with Zookeeper for node
>> discovery. I see the following error while trying to update Cache entry
>> using a key. I am using Ignite as an embedded cache inside an Tomcat based
>> web server. Below error does not happen immediately after the tomcat/java
>> process has startedm but starts happening after a while.
>>
>> I would really appreciate any pointers on how to debug this issue.
>>
>> class org.apache.ignite.IgniteException: Runtime failure on search row:
>> org.apache.ignite.internal.processors.cache.IgniteCacheOffhe
>> apManagerImpl$SearchRow@1c2ad2e8
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> BPlusTree.invoke(BPlusTree.java:1615)
>> at org.apache.ignite.internal.processors.cache.IgniteCacheOffhe
>> apManagerImpl$CacheDataStoreImpl.invoke(IgniteCacheOffheapMa
>> nagerImpl.java:925)
>> at org.apache.ignite.internal.processors.cache.IgniteCacheOffhe
>> apManagerImpl.invoke(IgniteCacheOffheapManagerImpl.java:326)
>> at org.apache.ignite.internal.processors.cache.GridCacheMapEntr
>> y.innerUpdate(GridCacheMapEntry.java:1693)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridDhtAtomicCache.updateSingle(GridDhtAtomicCache.java:2386)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAto
>> micCache.java:1792)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtom
>> icCache.java:1630)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridNearAtomicAbstractUpdateFuture.sendSingleRequest(
>> GridNearAtomicAbstractUpdateFuture.java:299)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridNearAtomicSingleUpdateFuture.map(GridNearAtomicSi
>> ngleUpdateFuture.java:480)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridNearAtomicSingleUpdateFuture.mapOnTopology(GridNe
>> arAtomicSingleUpdateFuture.java:440)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridNearAtomicAbstractUpdateFuture.map(GridNearAtomic
>> AbstractUpdateFuture.java:248)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridDhtAtomicCache.update0(GridDhtAtomicCache.java:1162)
>> at org.apache.ignite.internal.processors.cache.distributed.dht.
>> atomic.GridDhtAtomicCache.put0(GridDhtAtomicCache.java:651)
>> at org.apache.ignite.internal.processors.cache.GridCacheAdapter
>> .put(GridCacheAdapter.java:2345)
>> at org.apache.ignite.internal.processors.cache.GridCacheAdapter
>> .put(GridCacheAdapter.java:2322)
>> at org.apache.ignite.internal.processors.cache.IgniteCacheProxy
>> .put(IgniteCacheProxy.java:1519)
>>
>>
>> *Caused by: java.lang.IllegalMonitorStateException: Attempted to release
>> write lock while not holding it [lock=00007f399844d470,
>> state=0001033800000883*
>> at org.apache.ignite.internal.util.OffheapReadWriteLock.writeUn
>> lock(OffheapReadWriteLock.java:259)
>> at org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImp
>> l.writeUnlock(PageMemoryNoStoreImpl.java:495)
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> util.PageHandler.writeUnlock(PageHandler.java:379)
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> util.PageHandler.writePage(PageHandler.java:288)
>> at org.apache.ignite.internal.processors.cache.database.DataStr
>> ucture.write(DataStructure.java:241)
>> at org.apache.ignite.internal.processors.cache.database.freelis
>> t.FreeListImpl.updateDataRow(FreeListImpl.java:506)
>> at org.apache.ignite.internal.processors.cache.database.RowStor
>> e.updateRow(RowStore.java:82)
>> at org.apache.ignite.internal.processors.cache.IgniteCacheOffhe
>> apManagerImpl$CacheDataStoreImpl.createRow(IgniteCacheOffhea
>> pManagerImpl.java:970)
>> at org.apache.ignite.internal.processors.cache.GridCacheMapEntr
>> y$AtomicCacheUpdateClosure.update(GridCacheMapEntry.java:4428)
>> at org.apache.ignite.internal.processors.cache.GridCacheMapEntr
>> y$AtomicCacheUpdateClosure.call(GridCacheMapEntry.java:4226)
>> at org.apache.ignite.internal.processors.cache.GridCacheMapEntr
>> y$AtomicCacheUpdateClosure.call(GridCacheMapEntry.java:3966)
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> BPlusTree$Invoke.invokeClosure(BPlusTree.java:2966)
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> BPlusTree$Invoke.access$6200(BPlusTree.java:2860)
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> BPlusTree.invokeDown(BPlusTree.java:1702)
>> at org.apache.ignite.internal.processors.cache.database.tree.
>> BPlusTree.invoke(BPlusTree.java:1585)
>> ... 64 more
>>
>> Regards,
>> Venkat
>>
>
>
