[
https://issues.apache.org/jira/browse/CASSANDRA-7245?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14011299#comment-14011299
]
Jason Brown commented on CASSANDRA-7245:
----------------------------------------
Hmm, unfortunately, this patch makes things worse for me :(. I still am seeing
the decorated key exception on compaction (including here since we haven'y so
far in the ticket):
{code}ERROR [CompactionExecutor:4] 2014-05-28 16:42:36,909
CassandraDaemon.java:166 - Exception in thread
Thread[CompactionExecutor:4,1,main]
java.lang.RuntimeException: Last written key DecoratedKey(-2628246975040818544,
30303030373145304638) >= current key DecoratedKey(-7730803788015025148,
30303030364642324138) writing into
/u/sdd/cassandra-jasobrown/data/Keyspace1/Standard1-48037880e68611e3865a919228363515/Keyspace1-Standard1-tmp-ka-22-Data.db
at
org.apache.cassandra.io.sstable.SSTableWriter.beforeAppend(SSTableWriter.java:172)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.io.sstable.SSTableWriter.append(SSTableWriter.java:196)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.io.sstable.SSTableRewriter.append(SSTableRewriter.java:109)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.db.compaction.CompactionTask.runWith(CompactionTask.java:177)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.io.util.DiskAwareRunnable.runMayThrow(DiskAwareRunnable.java:48)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:74)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:64)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.db.compaction.CompactionManager$BackgroundCompactionTask.run(CompactionManager.java:235)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
~[na:1.7.0_13]
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
~[na:1.7.0_13]
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
~[na:1.7.0_13]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_13]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_13]
at java.lang.Thread.run(Thread.java:722) [na:1.7.0_13]
{code}
But now I also ref counting exceptions:
{code}ERROR [Native-Transport-Requests:176] 2014-05-28 16:37:54,838
ErrorMessage.java:218 - Unexpected exception during request
io.netty.util.IllegalReferenceCountException: refCnt: 0, increment: 1
at
io.netty.buffer.AbstractReferenceCountedByteBuf.retain(AbstractReferenceCountedByteBuf.java:64)
~[netty-all-4.0.17.Final.jar:4.0.17.Final]
at org.apache.cassandra.transport.Frame.retain(Frame.java:60)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at org.apache.cassandra.db.Mutation.retain(Mutation.java:114)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.net.MessagingService.addCallback(MessagingService.java:584)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.net.MessagingService.sendRR(MessagingService.java:650)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.service.StorageProxy.sendToHintedEndpoints(StorageProxy.java:812)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.service.StorageProxy$2.apply(StorageProxy.java:122)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.service.StorageProxy.performWrite(StorageProxy.java:703)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:467)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.service.StorageProxy.mutateWithTriggers(StorageProxy.java:537)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.cql3.statements.ModificationStatement.executeWithoutCondition(ModificationStatement.java:503)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.cql3.statements.ModificationStatement.execute(ModificationStatement.java:487)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:187)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:413)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.transport.messages.ExecuteMessage.execute(ExecuteMessage.java:133)
~[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:414)
[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:309)
[apache-cassandra-2.1.0-beta2-SNAPSHOT.jar:2.1.0-beta2-SNAPSHOT]
at
io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:103)
[netty-all-4.0.17.Final.jar:4.0.17.Final]
at
io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:340)
[netty-all-4.0.17.Final.jar:4.0.17.Final]
at
io.netty.channel.DefaultChannelHandlerContext.access$700(DefaultChannelHandlerContext.java:29)
[netty-all-4.0.17.Final.jar:4.0.17.Final]
at
io.netty.channel.DefaultChannelHandlerContext$8.run(DefaultChannelHandlerContext.java:331)
[netty-all-4.0.17.Final.jar:4.0.17.Final]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[na:1.7.0_13]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_13]
at java.lang.Thread.run(Thread.java:722) [na:1.7.0_13]
{code}
fwiw, it looks like the ref counting exception occurs when a memtable flush is
starting:
{code}INFO [SlabPoolCleaner] 2014-05-28 16:37:48,597
ColumnFamilyStore.java:857 - Enqueuing flush of Standard1: 836991055 (40%)
on-heap, 0 (0%) off-heap
INFO [MemtableFlushWriter:3] 2014-05-28 16:37:48,633 Memtable.java:331 -
Writing Memtable-Standard1@1182801803(255151849 serialized bytes, 7100814 ops,
40%/0% of on/off-heap limit)
INFO [MemtableFlushWriter:4] 2014-05-28 16:37:51,545 Memtable.java:365 -
Completed flushing
/u/sdd/cassandra-jasobrown/flush/Keyspace1/Standard1-48037880e68611e3865a919228363515/Keyspace1-Standard1-ka-2-Data.db
(313657615 bytes) for commitlog position
ReplayPosition(segmentId=1401294928141, position=20412051)
ERROR [Native-Transport-Requests:176] 2014-05-28 16:37:54,838
ErrorMessage.java:218 - Unexpected exception during request
io.netty.util.IllegalReferenceCountException: refCnt: 0, increment: 1
{code}
I could be wrong about this, but it anecdotally looks related.
> Out-of-Order keys with stress + CQL3
> ------------------------------------
>
> Key: CASSANDRA-7245
> URL: https://issues.apache.org/jira/browse/CASSANDRA-7245
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Reporter: Pavel Yaskevich
> Assignee: T Jake Luciani
> Fix For: 2.1 rc1
>
> Attachments: 7245.txt
>
>
> We have been generating data (stress with CQL3 prepared) for CASSANDRA-4718
> and found following problem almost in every SSTable generated (~200 GB of
> data and 821 SSTables).
> We set up they keys to be 10 bytes in size (default) and population between 1
> and 600000000.
> Once I ran 'sstablekeys' on the generated SSTable files I got following
> exceptions:
> _There is a problem with sorting of normal looking keys:_
> 30303039443538353645
> 30303039443745364242
> java.io.IOException: Key out of order! DecoratedKey(-217680888487824985,
> *30303039443745364242*) > DecoratedKey(-1767746583617597213,
> *30303039443437454333*)
> 00000a30303033343933
> 37344400001388343933
> java.io.IOException: Key out of order! DecoratedKey(5440473860101999581,
> *37344400001388343933*) > DecoratedKey(-7565486415339257200,
> *30303033344639443137*)
> 30303033354244363031
> 30303033354133423742
> java.io.IOException: Key out of order! DecoratedKey(2687072396429900180,
> *30303033354133423742*) > DecoratedKey(-7838239767410066684,
> *30303033354145344534*)
> 30303034313442354137
> 30343136353633340000
> java.io.IOException: Key out of order! DecoratedKey(1516003874415400462,
> *30343136353633340000*) > DecoratedKey(-9106177395653818217,
> *30303034313333444238*)
> 30303035373044373435
> 30303035373044334631
> java.io.IOException: Key out of order! DecoratedKey(-3645715702154616540,
> *30303035373044334631*) > DecoratedKey(-4296696226469000945,
> *30303035373132364138*)
> _And completely different ones:_
> 30303041333745373543
> 7cd045c59a90d7587d8d
> java.io.IOException: Key out of order! DecoratedKey(-3595402345023230196,
> *7cd045c59a90d7587d8d*) > DecoratedKey(-5146766422778260690,
> *30303041333943303232*)
> 30303033323144444144
> 30303033323346343932
> java.io.IOException: Key out of order! DecoratedKey(7071845511166615635,
> *30303033323346343932*) > DecoratedKey(5233296131921119414,
> *53d83e00000012287e03*)
> 30303034314531374431
> 3806734b256c27e41ec2
> java.io.IOException: Key out of order! DecoratedKey(-7720474642702543193,
> *3806734b256c27e41ec2*) > DecoratedKey(-8072288379146044663,
> *30303034314136413343*)
> _And sometimes there is no problem at all:_
> 30303033353144463637
> 0000002a31b3b31a1c2f
> 5d616dd38211ebb5d6ec
> 44423645000013880000
> 00001388138844463744
> 30303033353143394343
> It's worth to mention that we have got 22 timeout exceptions but number of
> out-of-order keys is much larger than that.
--
This message was sent by Atlassian JIRA
(v6.2#6252)
