Hi
I have a single node in a 6 node Cassandra 1.0.11 cluster that seems to
have a single column family in a weird state.
Repairs, upgradesstables, anything that touches this CF crashes.
I've drained the node, removed every file for this CF from said node,
removed the commit log, started it up and as soon as data is written to
this CF on this node I'm in the same situation again. Anyone have any
suggestions for how to fix this?
I'm tempted to remove the node and re-add it but I was hoping for something
a little less disruptive.
$ nodetool -h localhost upgradesstables Player PlayerCounters
Error occured while upgrading the sstables for keyspace Player
java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError:
Could not initialize class org.apache.cassandra.utils.NodeId$LocalIds
at
java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)
at java.util.concurrent.FutureTask.get(FutureTask.java:83)
at
org.apache.cassandra.db.compaction.CompactionManager.performAllSSTableOperation(CompactionManager.java:219)
at
org.apache.cassandra.db.compaction.CompactionManager.performSSTableRewrite(CompactionManager.java:235)
at
org.apache.cassandra.db.ColumnFamilyStore.sstablesRewrite(ColumnFamilyStore.java:999)
at
org.apache.cassandra.service.StorageService.upgradeSSTables(StorageService.java:1652)
at sun.reflect.GeneratedMethodAccessor59.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
at
com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
at
com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
at
com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
at
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
at
com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
at
javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)
at
javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
at
javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)
at
javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)
at
javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
at sun.reflect.GeneratedMethodAccessor58.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:303)
at sun.rmi.transport.Transport$1.run(Transport.java:159)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.NoClassDefFoundError: Could not initialize class
org.apache.cassandra.utils.NodeId$LocalIds
at org.apache.cassandra.utils.NodeId.localIds(NodeId.java:49)
at
org.apache.cassandra.utils.NodeId.getOldLocalNodeIds(NodeId.java:79)
at
org.apache.cassandra.db.CounterColumn.computeOldShardMerger(CounterColumn.java:251)
at
org.apache.cassandra.db.CounterColumn.mergeAndRemoveOldShards(CounterColumn.java:297)
at
org.apache.cassandra.db.CounterColumn.mergeAndRemoveOldShards(CounterColumn.java:271)
at
org.apache.cassandra.db.compaction.PrecompactedRow.removeDeletedAndOldShards(PrecompactedRow.java:81)
at
org.apache.cassandra.db.compaction.PrecompactedRow.<init>(PrecompactedRow.java:97)
at
org.apache.cassandra.db.compaction.CompactionController.getCompactedRow(CompactionController.java:137)
at
org.apache.cassandra.db.compaction.CompactionIterable$Reducer.getReduced(CompactionIterable.java:97)
at
org.apache.cassandra.db.compaction.CompactionIterable$Reducer.getReduced(CompactionIterable.java:82)
at
org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext(MergeIterator.java:207)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:140)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:135)
at
com.google.common.collect.Iterators$7.computeNext(Iterators.java:614)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:140)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:135)
at
org.apache.cassandra.db.compaction.CompactionTask.execute(CompactionTask.java:141)
at
org.apache.cassandra.db.compaction.CompactionManager$4.perform(CompactionManager.java:247)
at
org.apache.cassandra.db.compaction.CompactionManager$2.call(CompactionManager.java:198)
at
java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
... 3 more
$ tail /var/log/cassandra/system.log
INFO [CompactionExecutor:852] 2012-09-04 09:45:37,021 CompactionTask.java
(line 112) Compacting
[SSTableReader(path='/var/lib/cassandra/data/Player/PlayerCounters-hd-1-Data.db')]
ERROR [CompactionExecutor:852] 2012-09-04 09:45:37,030
AbstractCassandraDaemon.java (line 139) Fatal exception in thread
Thread[CompactionExecutor:852,1,RMI Runtime]
java.lang.NoClassDefFoundError: Could not initialize class
org.apache.cassandra.utils.NodeId$LocalIds
at org.apache.cassandra.utils.NodeId.localIds(NodeId.java:49)
at
org.apache.cassandra.utils.NodeId.getOldLocalNodeIds(NodeId.java:79)
at
org.apache.cassandra.db.CounterColumn.computeOldShardMerger(CounterColumn.java:251)
at
org.apache.cassandra.db.CounterColumn.mergeAndRemoveOldShards(CounterColumn.java:297)
at
org.apache.cassandra.db.CounterColumn.mergeAndRemoveOldShards(CounterColumn.java:271)
at
org.apache.cassandra.db.compaction.PrecompactedRow.removeDeletedAndOldShards(PrecompactedRow.java:81)
at
org.apache.cassandra.db.compaction.PrecompactedRow.<init>(PrecompactedRow.java:97)
at
org.apache.cassandra.db.compaction.CompactionController.getCompactedRow(CompactionController.java:137)
at
org.apache.cassandra.db.compaction.CompactionIterable$Reducer.getReduced(CompactionIterable.java:97)
at
org.apache.cassandra.db.compaction.CompactionIterable$Reducer.getReduced(CompactionIterable.java:82)
at
org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext(MergeIterator.java:207)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:140)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:135)
at
com.google.common.collect.Iterators$7.computeNext(Iterators.java:614)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:140)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:135)
at
org.apache.cassandra.db.compaction.CompactionTask.execute(CompactionTask.java:141)
at
org.apache.cassandra.db.compaction.CompactionManager$4.perform(CompactionManager.java:247)
at
org.apache.cassandra.db.compaction.CompactionManager$2.call(CompactionManager.java:198)
at
java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
