[ https://issues.apache.org/jira/browse/CASSANDRA-20596?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17947177#comment-17947177 ]
Jon Haddad commented on CASSANDRA-20596: ---------------------------------------- I'm not sure if this is because of what I did before, but it looks like the journal is now corrupt: {noformat} ERROR [AccordExecutor[3,3]] 2025-04-24 17:56:39,671 AccordAgent.java:149 - Uncaught accord exception java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:272) at accord.local.Cleanup.cleanupWithFullRoute(Cleanup.java:181) at accord.local.Cleanup.shouldCleanupInternal(Cleanup.java:170) at accord.local.Cleanup.shouldCleanup(Cleanup.java:128) at accord.impl.CommandChange$Builder.shouldCleanup(CommandChange.java:321) at accord.impl.CommandChange$Builder.maybeCleanup(CommandChange.java:329) at org.apache.cassandra.service.accord.AccordJournal.loadCommand(AccordJournal.java:232) at org.apache.cassandra.service.accord.AccordCommandStore.loadCommand(AccordCommandStore.java:475) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1178) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1169) at org.apache.cassandra.service.accord.AccordCacheEntry.lambda$load$0(AccordCacheEntry.java:333) at org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:1074) at org.apache.cassandra.service.accord.AccordExecutorAbstractLockLoop.runWithoutLock(AccordExecutorAbstractLockLoop.java:257) at org.apache.cassandra.service.accord.AccordExecutorLoops.lambda$wrap$0(AccordExecutorLoops.java:63) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) ERROR [AccordExecutor[3,3]] 2025-04-24 17:56:39,671 JVMStabilityInspector.java:71 - Exception in thread Thread[AccordExecutor[3,3],5,system] java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:272) at accord.local.Cleanup.cleanupWithFullRoute(Cleanup.java:181) at accord.local.Cleanup.shouldCleanupInternal(Cleanup.java:170) at accord.local.Cleanup.shouldCleanup(Cleanup.java:128) at accord.impl.CommandChange$Builder.shouldCleanup(CommandChange.java:321) at accord.impl.CommandChange$Builder.maybeCleanup(CommandChange.java:329) at org.apache.cassandra.service.accord.AccordJournal.loadCommand(AccordJournal.java:232) at org.apache.cassandra.service.accord.AccordCommandStore.loadCommand(AccordCommandStore.java:475) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1178) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1169) at org.apache.cassandra.service.accord.AccordCacheEntry.lambda$load$0(AccordCacheEntry.java:333) at org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:1074) at org.apache.cassandra.service.accord.AccordExecutorAbstractLockLoop.runWithoutLock(AccordExecutorAbstractLockLoop.java:257) at org.apache.cassandra.service.accord.AccordExecutorLoops.lambda$wrap$0(AccordExecutorLoops.java:63) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) ERROR [AccordExecutor[3,3]] 2025-04-24 17:56:39,672 AccordAgent.java:149 - Uncaught accord exception java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:272) at accord.local.Cleanup.cleanupWithFullRoute(Cleanup.java:181) at accord.local.Cleanup.shouldCleanupInternal(Cleanup.java:170) at accord.local.Cleanup.shouldCleanup(Cleanup.java:128) at accord.impl.CommandChange$Builder.shouldCleanup(CommandChange.java:321) at accord.impl.CommandChange$Builder.maybeCleanup(CommandChange.java:329) at org.apache.cassandra.service.accord.AccordJournal.loadCommand(AccordJournal.java:232) at org.apache.cassandra.service.accord.AccordCommandStore.loadCommand(AccordCommandStore.java:475) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1178) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1169) at org.apache.cassandra.service.accord.AccordCacheEntry.lambda$load$0(AccordCacheEntry.java:333) at org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:1074) at org.apache.cassandra.service.accord.AccordExecutorAbstractLockLoop.runWithoutLock(AccordExecutorAbstractLockLoop.java:257) at org.apache.cassandra.service.accord.AccordExecutorLoops.lambda$wrap$0(AccordExecutorLoops.java:63) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) ERROR [AccordExecutor[3,3]] 2025-04-24 17:56:39,672 JVMStabilityInspector.java:71 - Exception in thread Thread[AccordExecutor[3,3],5,system] java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:272) at accord.local.Cleanup.cleanupWithFullRoute(Cleanup.java:181) at accord.local.Cleanup.shouldCleanupInternal(Cleanup.java:170) at accord.local.Cleanup.shouldCleanup(Cleanup.java:128) at accord.impl.CommandChange$Builder.shouldCleanup(CommandChange.java:321) at accord.impl.CommandChange$Builder.maybeCleanup(CommandChange.java:329) at org.apache.cassandra.service.accord.AccordJournal.loadCommand(AccordJournal.java:232) at org.apache.cassandra.service.accord.AccordCommandStore.loadCommand(AccordCommandStore.java:475) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1178) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1169) at org.apache.cassandra.service.accord.AccordCacheEntry.lambda$load$0(AccordCacheEntry.java:333) at org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:1074) at org.apache.cassandra.service.accord.AccordExecutorAbstractLockLoop.runWithoutLock(AccordExecutorAbstractLockLoop.java:257) at org.apache.cassandra.service.accord.AccordExecutorLoops.lambda$wrap$0(AccordExecutorLoops.java:63) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) Exception (java.lang.RuntimeException) encountered during startup: Can not replay journal. java.lang.RuntimeException: Can not replay journal. at org.apache.cassandra.service.accord.AccordJournal.replay(AccordJournal.java:471) at org.apache.cassandra.service.accord.AccordService.replayJournal(AccordService.java:246) at org.apache.cassandra.service.accord.AccordService.startup(AccordService.java:235) at org.apache.cassandra.tcm.Startup.startup(Startup.java:427) at org.apache.cassandra.tcm.Startup.startup(Startup.java:399) at org.apache.cassandra.service.StorageService.joinRing(StorageService.java:954) at org.apache.cassandra.service.StorageService.initServer(StorageService.java:849) at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:373) at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:735) at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:889) Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.async.AsyncChains.getUnchecked(AsyncChains.java:865) at org.apache.cassandra.service.accord.AccordJournal.replay(AccordJournal.java:454) ... 9 more Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.async.AsyncChains.getBlocking(AsyncChains.java:818) at accord.utils.async.AsyncChains.getUninterruptibly(AsyncChains.java:842) at accord.utils.async.AsyncChains.getUninterruptibly(AsyncChains.java:825) at accord.utils.async.AsyncChains.getUnchecked(AsyncChains.java:861) ... 10 more Caused by: java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:272) at accord.local.Cleanup.cleanupWithFullRoute(Cleanup.java:181) at accord.local.Cleanup.shouldCleanupInternal(Cleanup.java:170) at accord.local.Cleanup.shouldCleanup(Cleanup.java:128) at accord.impl.CommandChange$Builder.shouldCleanup(CommandChange.java:321) at accord.impl.CommandChange$Builder.maybeCleanup(CommandChange.java:329) at org.apache.cassandra.service.accord.AccordJournal.loadCommand(AccordJournal.java:232) at org.apache.cassandra.service.accord.AccordCommandStore.loadCommand(AccordCommandStore.java:475) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1178) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1169) at org.apache.cassandra.service.accord.AccordCacheEntry.lambda$load$0(AccordCacheEntry.java:333) at org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:1074) at org.apache.cassandra.service.accord.AccordExecutorAbstractLockLoop.runWithoutLock(AccordExecutorAbstractLockLoop.java:257) at org.apache.cassandra.service.accord.AccordExecutorLoops.lambda$wrap$0(AccordExecutorLoops.java:63) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) ERROR [main] 2025-04-24 17:56:39,672 CassandraDaemon.java:916 - Exception encountered during startup java.lang.RuntimeException: Can not replay journal. at org.apache.cassandra.service.accord.AccordJournal.replay(AccordJournal.java:471) at org.apache.cassandra.service.accord.AccordService.replayJournal(AccordService.java:246) at org.apache.cassandra.service.accord.AccordService.startup(AccordService.java:235) at org.apache.cassandra.tcm.Startup.startup(Startup.java:427) at org.apache.cassandra.tcm.Startup.startup(Startup.java:399) at org.apache.cassandra.service.StorageService.joinRing(StorageService.java:954) at org.apache.cassandra.service.StorageService.initServer(StorageService.java:849) at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:373) at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:735) at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:889) Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.async.AsyncChains.getUnchecked(AsyncChains.java:865) at org.apache.cassandra.service.accord.AccordJournal.replay(AccordJournal.java:454) ... 9 common frames omitted Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.async.AsyncChains.getBlocking(AsyncChains.java:818) at accord.utils.async.AsyncChains.getUninterruptibly(AsyncChains.java:842) at accord.utils.async.AsyncChains.getUninterruptibly(AsyncChains.java:825) at accord.utils.async.AsyncChains.getUnchecked(AsyncChains.java:861) ... 10 common frames omitted Caused by: java.lang.IllegalStateException: Command [23,1745536687761000,146(KW),1] that is being loaded is not owned by this shard on route {homeKey:1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,[1b255f4d-ef25-40a6-0000-000000000015:-4837685732957183785,1b255f4d-ef25-40a6-0000-000000000016:6237401078196462507]} at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:272) at accord.local.Cleanup.cleanupWithFullRoute(Cleanup.java:181) at accord.local.Cleanup.shouldCleanupInternal(Cleanup.java:170) at accord.local.Cleanup.shouldCleanup(Cleanup.java:128) at accord.impl.CommandChange$Builder.shouldCleanup(CommandChange.java:321) at accord.impl.CommandChange$Builder.maybeCleanup(CommandChange.java:329) at org.apache.cassandra.service.accord.AccordJournal.loadCommand(AccordJournal.java:232) at org.apache.cassandra.service.accord.AccordCommandStore.loadCommand(AccordCommandStore.java:475) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1178) at org.apache.cassandra.service.accord.AccordCache$CommandAdapter.load(AccordCache.java:1169) at org.apache.cassandra.service.accord.AccordCacheEntry.lambda$load$0(AccordCacheEntry.java:333) at org.apache.cassandra.service.accord.AccordExecutor$PlainRunnable.run(AccordExecutor.java:1074) at org.apache.cassandra.service.accord.AccordExecutorAbstractLockLoop.runWithoutLock(AccordExecutorAbstractLockLoop.java:257) at org.apache.cassandra.service.accord.AccordExecutorLoops.lambda$wrap$0(AccordExecutorLoops.java:63) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) INFO [StorageServiceShutdownHook] 2025-04-24 17:56:39,675 HintsService.java:268 - Paused hints dispatch INFO [StorageServiceShutdownHook] 2025-04-24 17:56:39,675 Gossiper.java:1827 - Announcing shutdown INFO [StorageServiceShutdownHook] 2025-04-24 17:56:39,675 StorageService.java:2190 - Node localhost/127.0.0.1:7000 state jump to shutdown ERROR [StorageServiceShutdownHook] 2025-04-24 17:56:41,682 StorageService.java:3933 - Caught an exception while draining java.lang.IllegalStateException: Caught interrupt while shutting down org.apache.cassandra.service.accord.AccordJournal@63f1c136 at org.apache.cassandra.utils.ExecutorUtils.shutdownSequentiallyAndWait(ExecutorUtils.java:108) at org.apache.cassandra.service.accord.AccordService.shutdownAndWait(AccordService.java:708) at org.apache.cassandra.service.StorageService.drain(StorageService.java:3811) at org.apache.cassandra.service.StorageService$1.runMayThrow(StorageService.java:736) at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:26) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.lang.IllegalStateException: STARTING at accord.utils.Invariants.createIllegalState(Invariants.java:77) at accord.utils.Invariants.illegalState(Invariants.java:82) at accord.utils.Invariants.require(Invariants.java:260) at org.apache.cassandra.service.accord.AccordJournal.shutdown(AccordJournal.java:201) at org.apache.cassandra.utils.ExecutorUtils.shutdownSequentiallyAndWait(ExecutorUtils.java:99) ... 6 common frames omitted INFO [StorageServiceShutdownHook] 2025-04-24 17:56:41,683 StorageService.java:740 - Cassandra shutdown complete {noformat} > NPE when executing IF block in transactions with invalid comparison > ------------------------------------------------------------------- > > Key: CASSANDRA-20596 > URL: https://issues.apache.org/jira/browse/CASSANDRA-20596 > Project: Apache Cassandra > Issue Type: Bug > Components: Accord > Reporter: Jon Haddad > Priority: Normal > > I managed to catch a NPE when trying to execute a transaction. This came > about from an error in my syntax, but I think we should handle this more > gracefully. > Here's the setup: > > > {noformat} > create KEYSPACE games WITH replication = {'class': 'SimpleStrategy', > 'replication_factor': 1}; > USE games; > CREATE TABLE games.high_scores_by_user ( user text, game text, score int, > primary key(user, game)) > WITH compaction = {'class': 'UnifiedCompactionStrategy', > 'scaling_parameters': 'L10'} > AND transactional_mode = 'full'; > CREATE TABLE games.leaderboard (game text, score int, user text, primary key > (game, score, user)) > WITH compaction = {'class': 'UnifiedCompactionStrategy', > 'scaling_parameters': 'L10'} > AND clustering order by (score DESC, user ASC) > AND transactional_mode = 'full'; > {noformat} > > The query: > > {noformat} > BEGIN TRANSACTION > LET high_score = (SELECT * from games.high_scores_by_user WHERE user = > 'jon' and game = 'pictionary'); > SELECT high_score.score; > IF high_score = 100 THEN > INSERT INTO games.high_scores_by_user (game, score, user) VALUES > ('pictionary', 110, 'jon'); > INSERT INTO games.leaderboard (game, score, user) VALUES ('pictionary', > 110, 'jon'); > DELETE from games.leaderboard WHERE user = 'jon' AND game = 'pictionary' > AND score = 100; > END IF > COMMIT TRANSACTION;{noformat} > In the IF block I forgot to put the field I was comparing, it should have > been high_score.score, and got the following: > > {noformat} > NoHostAvailable: ('Unable to complete the operation against any hosts', > {<Host: 127.0.0.1:9042 datacenter1>: <Error from server: code=0000 [Server > error] message="java.lang.NullPointerException: Cannot read field "type" > because "receiver" is null">}){noformat} > On the server side, I see this: > > {noformat} > ERROR [Native-Transport-Requests-1] 2025-04-24 16:00:15,711 > QueryMessage.java:130 - Unexpected error during query > java.lang.NullPointerException: Cannot read field "type" because "receiver" > is null > at > org.apache.cassandra.cql3.terms.Constants$Literal.testAssignment(Constants.java:326) > at > org.apache.cassandra.cql3.terms.Constants$Literal.prepare(Constants.java:295) > at > org.apache.cassandra.cql3.terms.Constants$Literal.prepare(Constants.java:243) > at > org.apache.cassandra.cql3.transactions.ConditionStatement$Raw.prepare(ConditionStatement.java:105) > at > org.apache.cassandra.cql3.statements.TransactionStatement$Parsed.prepare(TransactionStatement.java:734) > at > org.apache.cassandra.cql3.QueryProcessor.getStatement(QueryProcessor.java:911) > at org.apache.cassandra.cql3.QueryProcessor.parse(QueryProcessor.java:360) > at > org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:116) > at > org.apache.cassandra.transport.Message$Request.execute(Message.java:259) > at > org.apache.cassandra.transport.Dispatcher.processRequest(Dispatcher.java:423) > at > org.apache.cassandra.transport.Dispatcher.processRequest(Dispatcher.java:442) > at > org.apache.cassandra.transport.Dispatcher.processRequest(Dispatcher.java:469) > at > org.apache.cassandra.transport.Dispatcher$RequestProcessor.run(Dispatcher.java:314) > at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:99) > at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61) > at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71) > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:145) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.base/java.lang.Thread.run(Thread.java:840){noformat} > -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org