Maybe a firewall killing a connection?

What does the following show?
cat /proc/sys/net/ipv4/tcp_keepalive_time
cat /proc/sys/net/ipv4/tcp_keepalive_intvl
cat /proc/sys/net/ipv4/tcp_keepalive_probes

On Thu, May 7, 2020 at 10:31 AM Surbhi Gupta <surbhi.gupt...@gmail.com>
wrote:

> Hi,
>
> We are trying to expand a datacenter and trying to add nodes but when node
> is bootstrapping , it goes half way through and then fail with below error,
> We have increased stremthroughput from 200 to 400 when we were trying for
> the 2nd time but still it failed. We are on 3.11.0 , using G1GC with 31GB
> heap.
>
> ERROR [MessagingService-Incoming-/10.X.X.X] 2020-05-07 09:42:38,933
> CassandraDaemon.java:228 - Exception in thread
> Thread[MessagingService-Incoming-/10.X.X.X,main]
>
> java.io.IOError: java.io.EOFException: Stream ended prematurely
>
>         at
> org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer$1.computeNext(UnfilteredRowIteratorSerializer.java:227)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer$1.computeNext(UnfilteredRowIteratorSerializer.java:215)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize30(PartitionUpdate.java:839)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize(PartitionUpdate.java:814)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:425)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:434)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:371)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at org.apache.cassandra.net.MessageIn.read(MessageIn.java:123)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:192)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:180)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:94)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
> Caused by: java.io.EOFException: Stream ended prematurely
>
>         at
> net.jpountz.lz4.LZ4BlockInputStream.readFully(LZ4BlockInputStream.java:218)
> ~[lz4-1.3.0.jar:na]
>
>         at
> net.jpountz.lz4.LZ4BlockInputStream.refill(LZ4BlockInputStream.java:150)
> ~[lz4-1.3.0.jar:na]
>
>         at
> net.jpountz.lz4.LZ4BlockInputStream.read(LZ4BlockInputStream.java:117)
> ~[lz4-1.3.0.jar:na]
>
>         at java.io.DataInputStream.readFully(DataInputStream.java:195)
> ~[na:1.8.0_242]
>
>         at java.io.DataInputStream.readFully(DataInputStream.java:169)
> ~[na:1.8.0_242]
>
>         at
> org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:402)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.marshal.AbstractType.readValue(AbstractType.java:437)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.Cell$Serializer.deserialize(Cell.java:245)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredSerializer.readComplexColumn(UnfilteredSerializer.java:665)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredSerializer.lambda$deserializeRowBody$1(UnfilteredSerializer.java:606)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.utils.btree.BTree.applyForwards(BTree.java:1242)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at org.apache.cassandra.utils.btree.BTree.apply(BTree.java:1197)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at org.apache.cassandra.db.Columns.apply(Columns.java:377)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserializeRowBody(UnfilteredSerializer.java:600)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserializeOne(UnfilteredSerializer.java:475)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredSerializer.deserialize(UnfilteredSerializer.java:431)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         at
> org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer$1.computeNext(UnfilteredRowIteratorSerializer.java:222)
> ~[apache-cassandra-3.11.0.jar:3.11.0]
>
>         ... 11 common frames omitted
>
> Thanks
> Surbhi
>

Reply via email to