[
https://issues.apache.org/jira/browse/CASSANDRA-21529?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Brad Schoening updated CASSANDRA-21529:
---------------------------------------
Description:
Here is our Cassandra cluster details:
Cassandra version 5.0.6
Cluster has 45 nodes, 3DCs each has 3 Racks (5 nodes per rack)
-XX:MaxDirectMemorySize=15872M
-Xms24G
-Xmx24G
-XX:+UseG1GC
-XX:ConcGCThreads=4
-XX:G1HeapRegionSize=8m
chunk-cache pool: 8GiB
networking pool: 4GiB
bufferpooluseheapif_exhausted=false
memtable_allocation_type = offheap_objects
concurrent_compactors = 4
vmMemoryGb = 96
vmNumCores=12
14 tables use compression =
\{'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'}
3 tables use 16KB chunk length
This is happening frequently during heavy write operations, Please suggest any
fix for this type of issue. Let me know if you need any additional information.
Error Message from cassandra.log
2026-07-14 09:36:59,357 [ERROR] [CompactionExecutor:14919] cluster_id=16**
ip_address={*}.{*}.{*}.{*} JVMStabilityInspector.java:70 - Exception in thread
Thread[CompactionExecutor:14919,5,CompactionExecutor]
java.lang.OutOfMemoryError: Cannot reserve 65536 bytes of direct buffer memory
(allocated: 16642989867, limit: 16642998272)
at java.base/java.nio.Bits.reserveMemory(Bits.java:178)
at java.base/java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:121)
at java.base/java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:332)
at org.apache.cassandra.utils.memory.BufferPool.allocate(BufferPool.java:238)
at
org.apache.cassandra.utils.memory.BufferPool$LocalPool.get(BufferPool.java:923)
at
org.apache.cassandra.utils.memory.BufferPool$LocalPool.get(BufferPool.java:896)
at org.apache.cassandra.utils.memory.BufferPool.get(BufferPool.java:211)
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:163)
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:46)
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$3(LocalLoadingCache.java:183)
at
com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$2(LocalCache.java:167)
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2688)
at
java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916)
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2686)
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2669)
at
com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:112)
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:58)
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:241)
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:220)
at
org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:67)
at
org.apache.cassandra.io.util.RandomAccessReader.reBuffer(RandomAccessReader.java:61)
at
org.apache.cassandra.io.util.RebufferingInputStream.read(RebufferingInputStream.java:90)
at
org.apache.cassandra.io.util.RebufferingInputStream.readFully(RebufferingInputStream.java:68)
at
org.apache.cassandra.io.util.RebufferingInputStream.readFully(RebufferingInputStream.java:62)
at
org.apache.cassandra.db.marshal.ByteArrayAccessor.read(ByteArrayAccessor.java:103)
at
org.apache.cassandra.db.marshal.ByteArrayAccessor.read(ByteArrayAccessor.java:40)
at org.apache.cassandra.db.marshal.AbstractType.read(AbstractType.java:599)
at org.apache.cassandra.db.marshal.AbstractType.readArray(AbstractType.java:579)
at
org.apache.cassandra.db.ClusteringPrefix$Serializer.deserializeValuesWithoutSize(ClusteringPrefix.java:517)
at
org.apache.cassandra.db.Clustering$Serializer.deserialize(Clustering.java:177)
at
org.apache.cassandra.db.rows.UnfilteredSerializer.deserializeOne(UnfilteredSerializer.java:481)
at
org.apache.cassandra.db.rows.UnfilteredSerializer.deserialize(UnfilteredSerializer.java:438)
at
org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:92)
at
org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:70)
at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.hasNext(SSTableIdentityIterator.java:175)
at
org.apache.cassandra.utils.MergeIterator$TrivialOneToOne.computeNext(MergeIterator.java:486)
at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
at
org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:525)
at
org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:398)
at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:133)
at
org.apache.cassandra.db.transform.UnfilteredRows.isEmpty(UnfilteredRows.java:74)
at
org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:85)
at
org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:26)
at
org.apache.cassandra.db.transform.BasePartitions.hasNext(BasePartitions.java:94)
at
org.apache.cassandra.db.compaction.CompactionIterator.hasNext(CompactionIterator.java:304)
at
org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:216)
at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:26)
at
org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:94)
at
org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:100)
at
org.apache.cassandra.db.compaction.CompactionManager$BackgroundCompactionCandidate.run(CompactionManager.java:374)
at org.apache.cassandra.concurrent.FutureTask$3.call(FutureTask.java:141)
at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:842)
was:
Here is our Cassandra cluster details:
Cassandra version 5.0.6
Cluster has 45 nodes, 3DCs each has 3 Racks (5 nodes per rack)
-XX:MaxDirectMemorySize=15872M
-Xms24G
-Xmx24G
-XX:+UseG1GC
-XX:ConcGCThreads=4
-XX:G1HeapRegionSize=8m
chunk-cache pool: 8GiB
networking pool: 4GiB
bufferpooluseheapif_exhausted=false
memtable_allocation_type = offheap_objects
concurrent_compactors = 4
vmMemoryGb = 96
vmNumCores=12
Most tables use compression =
\{'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'}
This is happening frequently during heavy write operations, Please suggest any
fix for this type of issue. Let me know if you need any additional information.
Error Message from cassandra.log
2026-07-14 09:36:59,357 [ERROR] [CompactionExecutor:14919] cluster_id=16**
ip_address={*}.{*}.{*}.{*} JVMStabilityInspector.java:70 - Exception in thread
Thread[CompactionExecutor:14919,5,CompactionExecutor]
java.lang.OutOfMemoryError: Cannot reserve 65536 bytes of direct buffer memory
(allocated: 16642989867, limit: 16642998272)
at java.base/java.nio.Bits.reserveMemory(Bits.java:178)
at java.base/java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:121)
at java.base/java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:332)
at org.apache.cassandra.utils.memory.BufferPool.allocate(BufferPool.java:238)
at
org.apache.cassandra.utils.memory.BufferPool$LocalPool.get(BufferPool.java:923)
at
org.apache.cassandra.utils.memory.BufferPool$LocalPool.get(BufferPool.java:896)
at org.apache.cassandra.utils.memory.BufferPool.get(BufferPool.java:211)
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:163)
at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:46)
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$3(LocalLoadingCache.java:183)
at
com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$2(LocalCache.java:167)
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2688)
at
java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916)
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2686)
at
com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2669)
at
com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:112)
at
com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:58)
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:241)
at
org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:220)
at
org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:67)
at
org.apache.cassandra.io.util.RandomAccessReader.reBuffer(RandomAccessReader.java:61)
at
org.apache.cassandra.io.util.RebufferingInputStream.read(RebufferingInputStream.java:90)
at
org.apache.cassandra.io.util.RebufferingInputStream.readFully(RebufferingInputStream.java:68)
at
org.apache.cassandra.io.util.RebufferingInputStream.readFully(RebufferingInputStream.java:62)
at
org.apache.cassandra.db.marshal.ByteArrayAccessor.read(ByteArrayAccessor.java:103)
at
org.apache.cassandra.db.marshal.ByteArrayAccessor.read(ByteArrayAccessor.java:40)
at org.apache.cassandra.db.marshal.AbstractType.read(AbstractType.java:599)
at org.apache.cassandra.db.marshal.AbstractType.readArray(AbstractType.java:579)
at
org.apache.cassandra.db.ClusteringPrefix$Serializer.deserializeValuesWithoutSize(ClusteringPrefix.java:517)
at
org.apache.cassandra.db.Clustering$Serializer.deserialize(Clustering.java:177)
at
org.apache.cassandra.db.rows.UnfilteredSerializer.deserializeOne(UnfilteredSerializer.java:481)
at
org.apache.cassandra.db.rows.UnfilteredSerializer.deserialize(UnfilteredSerializer.java:438)
at
org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:92)
at
org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:70)
at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.hasNext(SSTableIdentityIterator.java:175)
at
org.apache.cassandra.utils.MergeIterator$TrivialOneToOne.computeNext(MergeIterator.java:486)
at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
at
org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:525)
at
org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:398)
at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:133)
at
org.apache.cassandra.db.transform.UnfilteredRows.isEmpty(UnfilteredRows.java:74)
at
org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:85)
at
org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:26)
at
org.apache.cassandra.db.transform.BasePartitions.hasNext(BasePartitions.java:94)
at
org.apache.cassandra.db.compaction.CompactionIterator.hasNext(CompactionIterator.java:304)
at
org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:216)
at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:26)
at
org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:94)
at
org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:100)
at
org.apache.cassandra.db.compaction.CompactionManager$BackgroundCompactionCandidate.run(CompactionManager.java:374)
at org.apache.cassandra.concurrent.FutureTask$3.call(FutureTask.java:141)
at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:842)
> Cassandra process is crashing due to OOM errors on direct buffer memory.
> ------------------------------------------------------------------------
>
> Key: CASSANDRA-21529
> URL: https://issues.apache.org/jira/browse/CASSANDRA-21529
> Project: Apache Cassandra
> Issue Type: Bug
> Components: Local/Caching
> Reporter: Krishna S
> Priority: Normal
>
> Here is our Cassandra cluster details:
> Cassandra version 5.0.6
> Cluster has 45 nodes, 3DCs each has 3 Racks (5 nodes per rack)
> -XX:MaxDirectMemorySize=15872M
> -Xms24G
> -Xmx24G
> -XX:+UseG1GC
> -XX:ConcGCThreads=4
> -XX:G1HeapRegionSize=8m
> chunk-cache pool: 8GiB
> networking pool: 4GiB
> bufferpooluseheapif_exhausted=false
> memtable_allocation_type = offheap_objects
> concurrent_compactors = 4
> vmMemoryGb = 96
> vmNumCores=12
>
> 14 tables use compression =
> \{'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'}
> 3 tables use 16KB chunk length
> This is happening frequently during heavy write operations, Please suggest
> any fix for this type of issue. Let me know if you need any additional
> information.
>
> Error Message from cassandra.log
> 2026-07-14 09:36:59,357 [ERROR] [CompactionExecutor:14919] cluster_id=16**
> ip_address={*}.{*}.{*}.{*} JVMStabilityInspector.java:70 - Exception in
> thread Thread[CompactionExecutor:14919,5,CompactionExecutor]
> java.lang.OutOfMemoryError: Cannot reserve 65536 bytes of direct buffer
> memory (allocated: 16642989867, limit: 16642998272)
> at java.base/java.nio.Bits.reserveMemory(Bits.java:178)
> at java.base/java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:121)
> at java.base/java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:332)
> at org.apache.cassandra.utils.memory.BufferPool.allocate(BufferPool.java:238)
> at
> org.apache.cassandra.utils.memory.BufferPool$LocalPool.get(BufferPool.java:923)
> at
> org.apache.cassandra.utils.memory.BufferPool$LocalPool.get(BufferPool.java:896)
> at org.apache.cassandra.utils.memory.BufferPool.get(BufferPool.java:211)
> at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:163)
> at org.apache.cassandra.cache.ChunkCache.load(ChunkCache.java:46)
> at
> com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$3(LocalLoadingCache.java:183)
> at
> com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$2(LocalCache.java:167)
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2688)
> at
> java.base/java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916)
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2686)
> at
> com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2669)
> at
> com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:112)
> at
> com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:58)
> at
> org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:241)
> at
> org.apache.cassandra.cache.ChunkCache$CachingRebufferer.rebuffer(ChunkCache.java:220)
> at
> org.apache.cassandra.io.util.RandomAccessReader.reBufferAt(RandomAccessReader.java:67)
> at
> org.apache.cassandra.io.util.RandomAccessReader.reBuffer(RandomAccessReader.java:61)
> at
> org.apache.cassandra.io.util.RebufferingInputStream.read(RebufferingInputStream.java:90)
> at
> org.apache.cassandra.io.util.RebufferingInputStream.readFully(RebufferingInputStream.java:68)
> at
> org.apache.cassandra.io.util.RebufferingInputStream.readFully(RebufferingInputStream.java:62)
> at
> org.apache.cassandra.db.marshal.ByteArrayAccessor.read(ByteArrayAccessor.java:103)
> at
> org.apache.cassandra.db.marshal.ByteArrayAccessor.read(ByteArrayAccessor.java:40)
> at org.apache.cassandra.db.marshal.AbstractType.read(AbstractType.java:599)
> at
> org.apache.cassandra.db.marshal.AbstractType.readArray(AbstractType.java:579)
> at
> org.apache.cassandra.db.ClusteringPrefix$Serializer.deserializeValuesWithoutSize(ClusteringPrefix.java:517)
> at
> org.apache.cassandra.db.Clustering$Serializer.deserialize(Clustering.java:177)
> at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserializeOne(UnfilteredSerializer.java:481)
> at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserialize(UnfilteredSerializer.java:438)
> at
> org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:92)
> at
> org.apache.cassandra.io.sstable.SSTableSimpleIterator$CurrentFormatIterator.computeNext(SSTableSimpleIterator.java:70)
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> at
> org.apache.cassandra.io.sstable.SSTableIdentityIterator.hasNext(SSTableIdentityIterator.java:175)
> at
> org.apache.cassandra.utils.MergeIterator$TrivialOneToOne.computeNext(MergeIterator.java:486)
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> at
> org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:525)
> at
> org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:398)
> at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:133)
> at
> org.apache.cassandra.db.transform.UnfilteredRows.isEmpty(UnfilteredRows.java:74)
> at
> org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:85)
> at
> org.apache.cassandra.db.partitions.PurgeFunction.applyToPartition(PurgeFunction.java:26)
> at
> org.apache.cassandra.db.transform.BasePartitions.hasNext(BasePartitions.java:94)
> at
> org.apache.cassandra.db.compaction.CompactionIterator.hasNext(CompactionIterator.java:304)
> at
> org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:216)
> at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:26)
> at
> org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:94)
> at
> org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:100)
> at
> org.apache.cassandra.db.compaction.CompactionManager$BackgroundCompactionCandidate.run(CompactionManager.java:374)
> at org.apache.cassandra.concurrent.FutureTask$3.call(FutureTask.java:141)
> at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
> at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at
> io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
> at java.base/java.lang.Thread.run(Thread.java:842)
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
