[
https://issues.apache.org/jira/browse/IGNITE-8476?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dmitry Sherstobitov updated IGNITE-8476:
----------------------------------------
Description:
Run 6 nodes, start loading (8 threads, 1000 2x cache.get() and 2x cache.put()
in each thread per second)
Kill 2 nodes and try to remove one node from baseline using
control.sh --baseline remove node1
control.sh --baseline version TOPOLOGY_VERSION
Utility hangs or connected client may hangs, this assertion appears in log
For transactional cache:
{code:java}
[16:32:58,960][SEVERE][sys-stripe-14-#15][G] Failed to execute runnable.
java.lang.AssertionError: localNode = be945692-c750-4d72-b9a1-9ac4170ff125,
dhtNodes = [ZookeeperClusterNode [id=810226e6-656a-460d-8069-ca7d2dd294ef,
addrs=[172.17.0.1, 0:0:0:0:0:0:0:1%lo, 172.25.1.28, 127.0.0.1], order=1,
loc=false, client=false], ZookeeperClusterNode
[id=be945692-c750-4d72-b9a1-9ac4170ff125, addrs=[172.17.0.1, 172.25.1.28,
0:0:0:0:0:0:0:1%lo, 127.0.0.1], order=3, loc=true, client=false],
ZookeeperClusterNode [id=db4503f6-9185-4673-b38c-8890dfa69511,
addrs=[172.17.0.1, 172.25.1.37, 0:0:0:0:0:0:0:1%lo, 127.0.0.1], order=5,
loc=false, client=false], ZookeeperClusterNode
[id=3b8d8d4f-3513-4d39-a1fd-7ec5b15fc653, addrs=[172.17.0.1, 172.25.1.37,
127.0.0.1, 0:0:0:0:0:0:0:1%lo], order=4, loc=false, client=false],
ZookeeperClusterNode [id=2bfc8c2e-2f47-4126-9cc4-6f017ce7efde,
addrs=[172.17.0.1, 172.25.1.37, 0:0:0:0:0:0:0:1%lo, 127.0.0.1], order=6,
loc=false, client=false]]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.map(GridDhtTxPrepareFuture.java:1520)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.prepare0(GridDhtTxPrepareFuture.java:1239)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.mapIfLocked(GridDhtTxPrepareFuture.java:671)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.prepare(GridDhtTxPrepareFuture.java:1048)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.prepareAsync(GridDhtTxLocal.java:397)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:516)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest0(IgniteTxHandler.java:150)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest(IgniteTxHandler.java:135)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$000(IgniteTxHandler.java:97)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:177)
at
org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:175)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748){code}
For atomic cache:
{code:java}
[18:40:12,858][SEVERE][sys-stripe-10-#11][GridCacheIoManager] Failed to process
message [senderId=9fde40b1-3b21-49de-b1ad-cdd0d9d902e5, messageType=class
o.a.i.i.processors.cache.distributed.near.GridNearSingleGetRequest]
java.lang.AssertionError: Wrong ready topology version for invalid partitions
response [topVer=AffinityTopologyVersion [topVer=11, minorTopVer=0],
req=GridNearSingleGetRequest [futId=1526053201329, key=KeyCacheObjectImpl
[part=42, val=1514, hasValBytes=true], flags=1, topVer=AffinityTopologyVersion
[topVer=11, minorTopVer=0], subjId=9fde40b1-3b21-49de-b1ad-cdd0d9d902e5,
taskNameHash=0, createTtl=-1, accessTtl=-1]]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter$6.apply(GridDhtCacheAdapter.java:943)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter$6.apply(GridDhtCacheAdapter.java:906)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:353)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.processNearSingleGetRequest(GridDhtCacheAdapter.java:906)
at
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$300(GridDhtAtomicCache.java:130)
at
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4.apply(GridDhtAtomicCache.java:252)
at
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4.apply(GridDhtAtomicCache.java:247)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748){code}
was:
Run 6 nodes, start loading (8 threads, 1000 cache.put() in each thread per
second)
Kill 2 nodes and try to remove one node from baseline using
control.sh --baseline remove node1
control.sh --baseline version TOPOLOGY_VERSION
Utility hangs or connected client may hangs, this assertion appears in log
{code:java}
[18:40:12,858][SEVERE][sys-stripe-10-#11][GridCacheIoManager] Failed to process
message [senderId=9fde40b1-3b21-49de-b1ad-cdd0d9d902e5, messageType=class
o.a.i.i.processors.cache.distributed.near.GridNearSingleGetRequest]
java.lang.AssertionError: Wrong ready topology version for invalid partitions
response [topVer=AffinityTopologyVersion [topVer=11, minorTopVer=0],
req=GridNearSingleGetRequest [futId=1526053201329, key=KeyCacheObjectImpl
[part=42, val=1514, hasValBytes=true], flags=1, topVer=AffinityTopologyVersion
[topVer=11, minorTopVer=0], subjId=9fde40b1-3b21-49de-b1ad-cdd0d9d902e5,
taskNameHash=0, createTtl=-1, accessTtl=-1]]
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter$6.apply(GridDhtCacheAdapter.java:943)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter$6.apply(GridDhtCacheAdapter.java:906)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
at
org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:353)
at
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.processNearSingleGetRequest(GridDhtCacheAdapter.java:906)
at
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$300(GridDhtAtomicCache.java:130)
at
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4.apply(GridDhtAtomicCache.java:252)
at
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4.apply(GridDhtAtomicCache.java:247)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
at java.lang.Thread.run(Thread.java:748){code}
> AssertionError exception occurs when trying to remove node from baseline
> under loading
> --------------------------------------------------------------------------------------
>
> Key: IGNITE-8476
> URL: https://issues.apache.org/jira/browse/IGNITE-8476
> Project: Ignite
> Issue Type: Bug
> Affects Versions: 2.5
> Reporter: Dmitry Sherstobitov
> Priority: Blocker
>
> Run 6 nodes, start loading (8 threads, 1000 2x cache.get() and 2x cache.put()
> in each thread per second)
> Kill 2 nodes and try to remove one node from baseline using
> control.sh --baseline remove node1
> control.sh --baseline version TOPOLOGY_VERSION
>
> Utility hangs or connected client may hangs, this assertion appears in log
> For transactional cache:
> {code:java}
> [16:32:58,960][SEVERE][sys-stripe-14-#15][G] Failed to execute runnable.
> java.lang.AssertionError: localNode = be945692-c750-4d72-b9a1-9ac4170ff125,
> dhtNodes = [ZookeeperClusterNode [id=810226e6-656a-460d-8069-ca7d2dd294ef,
> addrs=[172.17.0.1, 0:0:0:0:0:0:0:1%lo, 172.25.1.28, 127.0.0.1], order=1,
> loc=false, client=false], ZookeeperClusterNode
> [id=be945692-c750-4d72-b9a1-9ac4170ff125, addrs=[172.17.0.1, 172.25.1.28,
> 0:0:0:0:0:0:0:1%lo, 127.0.0.1], order=3, loc=true, client=false],
> ZookeeperClusterNode [id=db4503f6-9185-4673-b38c-8890dfa69511,
> addrs=[172.17.0.1, 172.25.1.37, 0:0:0:0:0:0:0:1%lo, 127.0.0.1], order=5,
> loc=false, client=false], ZookeeperClusterNode
> [id=3b8d8d4f-3513-4d39-a1fd-7ec5b15fc653, addrs=[172.17.0.1, 172.25.1.37,
> 127.0.0.1, 0:0:0:0:0:0:0:1%lo], order=4, loc=false, client=false],
> ZookeeperClusterNode [id=2bfc8c2e-2f47-4126-9cc4-6f017ce7efde,
> addrs=[172.17.0.1, 172.25.1.37, 0:0:0:0:0:0:0:1%lo, 127.0.0.1], order=6,
> loc=false, client=false]]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.map(GridDhtTxPrepareFuture.java:1520)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.prepare0(GridDhtTxPrepareFuture.java:1239)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.mapIfLocked(GridDhtTxPrepareFuture.java:671)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareFuture.prepare(GridDhtTxPrepareFuture.java:1048)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.prepareAsync(GridDhtTxLocal.java:397)
> at
> org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:516)
> at
> org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest0(IgniteTxHandler.java:150)
> at
> org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest(IgniteTxHandler.java:135)
> at
> org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$000(IgniteTxHandler.java:97)
> at
> org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:177)
> at
> org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:175)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
> at
> org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
> at java.lang.Thread.run(Thread.java:748){code}
> For atomic cache:
> {code:java}
> [18:40:12,858][SEVERE][sys-stripe-10-#11][GridCacheIoManager] Failed to
> process message [senderId=9fde40b1-3b21-49de-b1ad-cdd0d9d902e5,
> messageType=class
> o.a.i.i.processors.cache.distributed.near.GridNearSingleGetRequest]
> java.lang.AssertionError: Wrong ready topology version for invalid partitions
> response [topVer=AffinityTopologyVersion [topVer=11, minorTopVer=0],
> req=GridNearSingleGetRequest [futId=1526053201329, key=KeyCacheObjectImpl
> [part=42, val=1514, hasValBytes=true], flags=1,
> topVer=AffinityTopologyVersion [topVer=11, minorTopVer=0],
> subjId=9fde40b1-3b21-49de-b1ad-cdd0d9d902e5, taskNameHash=0, createTtl=-1,
> accessTtl=-1]]
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter$6.apply(GridDhtCacheAdapter.java:943)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter$6.apply(GridDhtCacheAdapter.java:906)
> at
> org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:383)
> at
> org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:353)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.processNearSingleGetRequest(GridDhtCacheAdapter.java:906)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$300(GridDhtAtomicCache.java:130)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4.apply(GridDhtAtomicCache.java:252)
> at
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$4.apply(GridDhtAtomicCache.java:247)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1054)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99)
> at
> org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1556)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1184)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:125)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1091)
> at
> org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:511)
> at java.lang.Thread.run(Thread.java:748){code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
