[
https://issues.apache.org/jira/browse/IGNITE-25217?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Vladimir Dmitrienko updated IGNITE-25217:
-----------------------------------------
Description:
*Steps to reproduce:*
# Start 3 nodes.
# Create 50 tables with 5 columns in 10 threads.
# Assert 50 tables are present in system view.
# Insert 1 row into each.
# Assert rows content is correct in 10 threads.
# Repeat steps 2-5 while amount of tables is 1000.
*Expected result:*
1000 tables created.
*Actual result:*
SQLException: Send with retry timed out on step 2 (tables 700 - 749):
{code:java}
java.util.concurrent.CompletionException:
java.util.concurrent.TimeoutException: Send with retry timed out [retryCount =
1, groupId = metastorage_group, traceId = ab44ca06-4818-4844-8d11-2f19c0040c09,
request = org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand
= null].
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
[retryCount = 1, groupId = metastorage_group, traceId =
ab44ca06-4818-4844-8d11-2f19c0040c09, request =
org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
... 8 more
2025-04-22 03:18:15:525 +0000
[ERROR][%TablesAmountCapacityMultiNodeTest_cluster_0%Raft-Group-Client-10][ReplicaManager]
Couldn't fetch pending assignments for rebalance failover
[groupId=1269_part_22, term=2].
java.util.concurrent.CompletionException:
java.util.concurrent.TimeoutException: Send with retry timed out [retryCount =
1, groupId = metastorage_group, traceId = 96370b85-c2d8-4ac8-b48e-add269bc0c58,
request = org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand
= null].
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
[retryCount = 1, groupId = metastorage_group, traceId =
96370b85-c2d8-4ac8-b48e-add269bc0c58, request =
org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
... 8 more
2025-04-22 03:18:15:525 +0000
[ERROR][%TablesAmountCapacityMultiNodeTest_cluster_0%Raft-Group-Client-1][ReplicaManager]
Couldn't fetch pending assignments for rebalance failover
[groupId=1331_part_11, term=2].
java.util.concurrent.CompletionException:
java.util.concurrent.TimeoutException: Send with retry timed out [retryCount =
1, groupId = metastorage_group, traceId = ca3421fb-b262-4c4a-b384-dcebb0187a5e,
request = org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand
= null].
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
[retryCount = 1, groupId = metastorage_group, traceId =
ca3421fb-b262-4c4a-b384-dcebb0187a5e, request =
org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
... 8 more{code}
was:
*Steps to reproduce:*
# Start 3 nodes.
# Create 50 tables with 5 columns in 10 threads.
# Assert 50 tables are present in system view.
# Insert 1 row into each.
# Assert rows content is correct in 10 threads.
# Repeat steps 2-5 while amount of tables is 1000.
*Expected result:*
1000 tables created.
*Actual result:*
SQLException: Send with retry timed out on step 2:
{code:java}
java.util.concurrent.CompletionException:
java.util.concurrent.TimeoutException: Send with retry timed out [retryCount =
1, groupId = metastorage_group, traceId = ab44ca06-4818-4844-8d11-2f19c0040c09,
request = org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand
= null].
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
[retryCount = 1, groupId = metastorage_group, traceId =
ab44ca06-4818-4844-8d11-2f19c0040c09, request =
org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
... 8 more
2025-04-22 03:18:15:525 +0000
[ERROR][%TablesAmountCapacityMultiNodeTest_cluster_0%Raft-Group-Client-10][ReplicaManager]
Couldn't fetch pending assignments for rebalance failover
[groupId=1269_part_22, term=2].
java.util.concurrent.CompletionException:
java.util.concurrent.TimeoutException: Send with retry timed out [retryCount =
1, groupId = metastorage_group, traceId = 96370b85-c2d8-4ac8-b48e-add269bc0c58,
request = org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand
= null].
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
[retryCount = 1, groupId = metastorage_group, traceId =
96370b85-c2d8-4ac8-b48e-add269bc0c58, request =
org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
... 8 more
2025-04-22 03:18:15:525 +0000
[ERROR][%TablesAmountCapacityMultiNodeTest_cluster_0%Raft-Group-Client-1][ReplicaManager]
Couldn't fetch pending assignments for rebalance failover
[groupId=1331_part_11, term=2].
java.util.concurrent.CompletionException:
java.util.concurrent.TimeoutException: Send with retry timed out [retryCount =
1, groupId = metastorage_group, traceId = ca3421fb-b262-4c4a-b384-dcebb0187a5e,
request = org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand
= null].
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
at
java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
at
org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
[retryCount = 1, groupId = metastorage_group, traceId =
ca3421fb-b262-4c4a-b384-dcebb0187a5e, request =
org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
... 8 more{code}
> [Flaky] Tables creation in 10 threads throws "Send with retry timed out"
> ------------------------------------------------------------------------
>
> Key: IGNITE-25217
> URL: https://issues.apache.org/jira/browse/IGNITE-25217
> Project: Ignite
> Issue Type: Bug
> Components: persistence
> Environment: 3 nodes (each node is CMG, each node
> {color:#067d17}"-Xms4096m"{color}, {color:#067d17}"-Xmx4096m"{color}), each
> on separate host. Each host vCPU: 4, Memory: 16GB.
> Reporter: Vladimir Dmitrienko
> Priority: Major
>
> *Steps to reproduce:*
> # Start 3 nodes.
> # Create 50 tables with 5 columns in 10 threads.
> # Assert 50 tables are present in system view.
> # Insert 1 row into each.
> # Assert rows content is correct in 10 threads.
> # Repeat steps 2-5 while amount of tables is 1000.
> *Expected result:*
> 1000 tables created.
> *Actual result:*
> SQLException: Send with retry timed out on step 2 (tables 700 - 749):
> {code:java}
> java.util.concurrent.CompletionException:
> java.util.concurrent.TimeoutException: Send with retry timed out [retryCount
> = 1, groupId = metastorage_group, traceId =
> ab44ca06-4818-4844-8d11-2f19c0040c09, request =
> org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
> at
> java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
> at
> java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
> at
> java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
> at
> java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
> at
> org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
> at
> org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
> at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
> at
> java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
> [retryCount = 1, groupId = metastorage_group, traceId =
> ab44ca06-4818-4844-8d11-2f19c0040c09, request =
> org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
> ... 8 more
> 2025-04-22 03:18:15:525 +0000
> [ERROR][%TablesAmountCapacityMultiNodeTest_cluster_0%Raft-Group-Client-10][ReplicaManager]
> Couldn't fetch pending assignments for rebalance failover
> [groupId=1269_part_22, term=2].
> java.util.concurrent.CompletionException:
> java.util.concurrent.TimeoutException: Send with retry timed out [retryCount
> = 1, groupId = metastorage_group, traceId =
> 96370b85-c2d8-4ac8-b48e-add269bc0c58, request =
> org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
> at
> java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
> at
> java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
> at
> java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
> at
> java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
> at
> org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
> at
> org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
> at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
> at
> java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
> [retryCount = 1, groupId = metastorage_group, traceId =
> 96370b85-c2d8-4ac8-b48e-add269bc0c58, request =
> org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
> ... 8 more
> 2025-04-22 03:18:15:525 +0000
> [ERROR][%TablesAmountCapacityMultiNodeTest_cluster_0%Raft-Group-Client-1][ReplicaManager]
> Couldn't fetch pending assignments for rebalance failover
> [groupId=1331_part_11, term=2].
> java.util.concurrent.CompletionException:
> java.util.concurrent.TimeoutException: Send with retry timed out [retryCount
> = 1, groupId = metastorage_group, traceId =
> ca3421fb-b262-4c4a-b384-dcebb0187a5e, request =
> org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
> at
> java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332)
> at
> java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347)
> at
> java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:636)
> at
> java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2194)
> at
> org.apache.ignite.internal.raft.RaftGroupServiceImpl.sendWithRetry(RaftGroupServiceImpl.java:618)
> at
> org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$scheduleRetry$50(RaftGroupServiceImpl.java:815)
> at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
> at
> java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> Caused by: java.util.concurrent.TimeoutException: Send with retry timed out
> [retryCount = 1, groupId = metastorage_group, traceId =
> ca3421fb-b262-4c4a-b384-dcebb0187a5e, request =
> org.apache.ignite.raft.jraft.rpc.ReadActionRequestImpl, originCommand = null].
> ... 8 more{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
