Hi ,
I am using Kafka 0.9 broker and 0.8 Consumer. Consumer was running fine since long time but today, I am getting the below exception. [ConsumerFetcherThread--5], Error for partition [rtdp.bizlogging.iditextlogdata,93] to broker 5:class kafka.common.NotLeaderForPartitionException If I try to print out the offset using consumer-groups or the offset-checker script I get below exceptions. 1) ./bin/kafka-consumer-groups.sh --zookeeper xx.xx.xx.xxx:2181 --describe --group hdfs.iditextlogdata.horton.slca GROUP, TOPIC, PARTITION, CURRENT OFFSET, LOG END OFFSET, LAG hdfs.iditextlogdata.horton.slca, rtdp.bizlogging.iditextlogdata, 0, 118872516, 118872523, 7 hdfs.iditextlogdata.horton.slca, rtdp.bizlogging.iditextlogdata, 1, 118866187, 118866209, 22 hdfs.iditextlogdata.horton.slca, rtdp.bizlogging.iditextlogdata, 2, 118883367, 118883389, 22 Error while executing consumer group command null java.io.EOFException at org.apache.kafka.common.network.NetworkReceive.readFromReadableChannel(NetworkReceive.java:83) at kafka.network.BlockingChannel.readCompletely(BlockingChannel.scala:129) at kafka.network.BlockingChannel.receive(BlockingChannel.scala:120) at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:99) at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:83) at kafka.consumer.SimpleConsumer.getOffsetsBefore(SimpleConsumer.scala:149) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService$$anonfun$getLogEndOffset$1.apply(ConsumerGroupCommand.scala:193) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService$$anonfun$getLogEndOffset$1.apply(ConsumerGroupCommand.scala:190) at scala.Option.map(Option.scala:146) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService.getLogEndOffset(ConsumerGroupCommand.scala:190) at kafka.admin.ConsumerGroupCommand$ConsumerGroupService$class.kafka$admin$ConsumerGroupCommand$ConsumerGroupService$$describePartition(ConsumerGroupCommand.scala:121) at kafka.admin.ConsumerGroupCommand$ConsumerGroupService$$anonfun$describeTopicPartition$2.apply(ConsumerGroupCommand.scala:102) at kafka.admin.ConsumerGroupCommand$ConsumerGroupService$$anonfun$describeTopicPartition$2.apply(ConsumerGroupCommand.scala:101) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.admin.ConsumerGroupCommand$ConsumerGroupService$class.describeTopicPartition(ConsumerGroupCommand.scala:101) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService.describeTopicPartition(ConsumerGroupCommand.scala:130) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService.kafka$admin$ConsumerGroupCommand$ZkConsumerGroupService$$describeTopic(ConsumerGroupCommand.scala:177) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService$$anonfun$describeGroup$1.apply(ConsumerGroupCommand.scala:162) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService$$anonfun$describeGroup$1.apply(ConsumerGroupCommand.scala:162) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService.describeGroup(ConsumerGroupCommand.scala:162) at kafka.admin.ConsumerGroupCommand$ConsumerGroupService$class.describe(ConsumerGroupCommand.scala:84) at kafka.admin.ConsumerGroupCommand$ZkConsumerGroupService.describe(ConsumerGroupCommand.scala:130) at kafka.admin.ConsumerGroupCommand$.main(ConsumerGroupCommand.scala:63) at kafka.admin.ConsumerGroupCommand.main(ConsumerGroupCommand.scala) 2) ./bin/kafka-consumer-offset-checker.sh --zookeeper xx.xx.xx.xxx:2181 --group hdfs.iditextlogdata.horton.slca Group Topic Pid Offset logSize Lag hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 0 118873069 118873072 3 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 1 118866744 118866762 18 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 2 118883933 118883955 22 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 3 118879212 118879225 13 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 4 118874876 118874895 19 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 5 118887748 118887768 20 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 6 118897484 118897505 21 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 7 118888124 118888154 30 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 8 118903070 118903106 36 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 9 118900494 118900518 24 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 10 118879171 118879196 25 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 11 118887240 118887274 34 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 12 118877722 118877742 20 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 13 118891045 118891066 21 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 14 118906037 118906066 29 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 15 118875014 118875036 22 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 16 118896568 118896600 32 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 17 118878738 118878778 40 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 18 118881174 118881194 20 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 19 118888617 118888637 20 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 20 118896919 118896947 28 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 21 118888287 118888314 27 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 22 118881832 118881863 31 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 23 118890799 118890832 33 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 24 118879354 118879385 31 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 25 118899885 118899906 21 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 26 118875610 118875652 42 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 27 118893277 118893304 27 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 28 118898665 118898691 26 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 29 118894762 118894797 35 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 30 118896128 118896158 30 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 31 118901312 118901342 30 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 32 118898093 118898123 30 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 33 118897418 118897441 23 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 34 118874062 118874087 25 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 35 118870834 118870877 43 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 36 118878706 118878743 37 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 37 118890590 118890622 32 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 38 118887209 118887246 37 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 39 118878585 118878613 28 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 40 118900998 118901032 34 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 41 118886133 118886162 29 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 42 118892844 118892880 36 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 43 118889861 118889902 41 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 44 118902251 118902299 48 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 45 118883572 118883607 35 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 46 118881642 118881688 46 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 47 118883077 118883106 29 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 48 118902774 118902807 33 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 49 118887276 118887310 34 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 50 117109002 117109037 35 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 51 118868797 118868839 42 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 52 118884002 118884040 38 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 53 118874290 118874341 51 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 54 118892835 118892870 35 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 55 118898296 118898331 35 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 56 118902058 118902104 46 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 57 118888603 118888634 31 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 58 118887894 118887943 49 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 59 118896965 118897001 36 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 60 118896731 118896759 28 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 61 118892627 118892652 25 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 62 118885821 118885854 33 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 63 118889186 118889217 31 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 64 118875668 118875712 44 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 65 118898876 118898919 43 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 66 118891420 118891446 26 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 67 118881092 118881129 37 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 68 118898093 118898133 40 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 69 117114024 117114053 29 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 70 118886208 118886251 43 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 71 118902410 118902453 43 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 72 118879508 118879544 36 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 73 118892326 118892378 52 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 74 118874371 118874416 45 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 75 118869774 118869808 34 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 76 118880671 118880718 47 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 77 118896683 118896721 38 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 78 117113648 117113684 36 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 79 118884780 118884821 41 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 80 118883443 118883488 45 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 81 118902357 118902389 32 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 82 118889555 118889613 58 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 83 118872520 118872552 32 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 84 118895347 118895389 42 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 85 118902926 118902971 45 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 86 118878923 118878965 42 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 87 118889576 118889607 31 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 88 118905974 118906009 35 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 89 118888985 118889026 41 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 90 118871284 118871310 26 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 91 118887617 118887674 57 hdfs.iditextlogdata.horton.slca rtdp.bizlogging.iditextlogdata 92 118888770 118888809 39 Exiting due to: empty.head. As you can see that it gives "Exiting due to empty head” exception. It does not print out from partition number 93 at which I am getting exceptions while consuming. Has any one encountered this before? Any pointers will be appreciated. Thanks Tushar
