One of my Cassandra nodes is giving me a number of errors then effectively
dying. I think it was somehow caused by interrupting a nodetool clean
operation. Running a recent 0.7 build out of svn.
ERROR [MutationStage:26] 2010-12-04 16:23:04,395 RowMutationVerbHandler.java
(line 83) Error in row mutation
java.io.EOFException
at java.io.DataInputStream.readFully(DataInputStream.java:180)
at
org.apache.cassandra.utils.FBUtilities.readByteArray(FBUtilities.java:264)
at
org.apache.cassandra.db.ColumnSerializer.deserialize(ColumnSerializer.java:76)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:363)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:312)
at
org.apache.cassandra.db.ColumnFamilySerializer.deserializeColumns(ColumnFamilySerializer.java:129)
at
org.apache.cassandra.db.ColumnFamilySerializer.deserialize(ColumnFamilySerializer.java:120)
at
org.apache.cassandra.db.RowMutationSerializer.defreezeTheMaps(RowMutation.java:383)
at
org.apache.cassandra.db.RowMutationSerializer.deserialize(RowMutation.java:393)
at
org.apache.cassandra.db.RowMutationSerializer.deserialize(RowMutation.java:351)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:52)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
ERROR [MutationStage:13] 2010-12-04 16:25:04,061 RowMutationVerbHandler.java
(line 83) Error in row mutation
org.apache.cassandra.db.UnserializableColumnFamilyException: Couldn't find
cfId=524288
at
org.apache.cassandra.db.ColumnFamilySerializer.deserialize(ColumnFamilySerializer.java:117)
at
org.apache.cassandra.db.RowMutationSerializer.defreezeTheMaps(RowMutation.java:383)
at
org.apache.cassandra.db.RowMutationSerializer.deserialize(RowMutation.java:393)
at
org.apache.cassandra.db.RowMutationSerializer.deserialize(RowMutation.java:351)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:52)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
ERROR [MutationStage:20] 2010-12-04 16:25:25,216
DebuggableThreadPoolExecutor.java (line 103) Error in ThreadPoolExecutor
java.lang.NullPointerException
at org.apache.cassandra.db.Table.apply(Table.java:398)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:73)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
ERROR [MutationStage:20] 2010-12-04 16:25:25,216
AbstractCassandraDaemon.java (line 90) Fatal exception in thread
Thread[MutationStage:20,5,main]
java.lang.NullPointerException
at org.apache.cassandra.db.Table.apply(Table.java:398)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:73)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
ERROR [COMMIT-LOG-WRITER] 2010-12-04 16:25:25,216
AbstractCassandraDaemon.java (line 90) Fatal exception in thread
Thread[COMMIT-LOG-WRITER,5,main]
java.lang.RuntimeException: java.lang.NullPointerException
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:34)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.NullPointerException
at
org.apache.cassandra.db.commitlog.CommitLogSegment.write(CommitLogSegment.java:92)
at
org.apache.cassandra.db.commitlog.CommitLog$LogRecordAdder.run(CommitLog.java:509)
at
org.apache.cassandra.db.commitlog.PeriodicCommitLogExecutorService$1.runMayThrow(PeriodicCommitLogExecutorService.java:52)
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30)
... 1 more
