I am getting error with twitter streaming with spark 1.4 version and twitter4j 3.0.6. There is another thread which also pointed the error. The error happened after the streaming ran for more than 12 hours. Here is the error log. I will try to use 3.0.3 as per the link below and try it..
http://mail-archives.us.apache.org/mod_mbox/spark-user/201407.mbox/%3CCAMwrk0=MpFU8YbcQPWibPoiwL4oRubO3r=tjz9rzgc1hc-m...@mail.gmail.com%3E 15/09/17 08:00:02 ERROR BlockGenerator: Error in block pushing thread java.io.NotSerializableException: twitter4j.internal.json.ScopesImpl Serialization stack: - object not serializable (class: twitter4j.internal.json.ScopesImpl, value: twitter4j.internal.json.ScopesImpl@357a15bb) - field (class: twitter4j.internal.json.StatusJSONImpl, name: scopes, type: interface twitter4j.Scopes) - object (class twitter4j.internal.json.StatusJSONImpl, StatusJSONImpl{createdAt=Thu Sep 17 08:00:02 BST 2015, id=644405431868125184, text='Visiting Southampton Boat Show? Start your day off with a Full English Breakfast for just £4.95 #southamptonboatshow #casino #breakfast', source='<a href="http://sproutsocial.com"; rel="nofollow">Sprout Social</a>', isTruncated=false, inReplyToStatusId=-1, inReplyToUserId=-1, isFavorited=false, isRetweeted=false, favoriteCount=0, inReplyToScreenName='null', geoLocation=null, place=null, retweetCount=0, isPossiblySensitive=false, isoLanguageCode='null', lang='en', contributorsIDs=[], retweetedStatus=null, userMentionEntities=[], urlEntities=[], hashtagEntities=[HashtagEntityJSONImpl{text='southamptonboatshow'}, HashtagEntityJSONImpl{text='casino'}, HashtagEntityJSONImpl{text='breakfast'}], mediaEntities=[], symbolEntities=[], currentUserRetweetId=-1, user=UserJSONImpl{id=66352550, name='GrosvenorSouthampton', screenName='GCSouthampton', location='Southampton UK', description='Southampton's most exciting & welcoming venue! Boasting a fantastic restaurant, electronic & live gaming, regular poker games, late night bar & much more', isContributorsEnabled=false, profileImageUrl='http://pbs.twimg.com/profile_images/589163086365335553/V-Dm2O5A_normal.jpg', profileImageUrlHttps='https://pbs.twimg.com/profile_images/589163086365335553/V-Dm2O5A_normal.jpg', url='http://www.grosvenorcasinos.com/local-casinos/southampton', isProtected=false, followersCount=1242, status=null, profileBackgroundColor='131516', profileTextColor='333333', profileLinkColor='42D3FF', profileSidebarFillColor='EFEFEF', profileSidebarBorderColor='EEEEEE', profileUseBackgroundImage=true, showAllInlineMedia=false, friendsCount=362, createdAt=Mon Aug 17 14:23:55 BST 2009, favouritesCount=105, utcOffset=-1, timeZone='null', profileBackgroundImageUrl='http://pbs.twimg.com/profile_background_images/530851272/Twitter_Background_Southampton.jpg', profileBackgroundImageUrlHttps='https://pbs.twimg.com/profile_background_images/530851272/Twitter_Background_Southampton.jpg', profileBackgroundTiled=true, lang='en', statusesCount=6927, isGeoEnabled=false, isVerified=false, translator=false, listedCount=19, isFollowRequestSent=false}}) at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40) at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47) at org.apache.spark.serializer.SerializationStream.writeAll(Serializer.scala:153) at org.apache.spark.storage.BlockManager.dataSerializeStream(BlockManager.scala:1189) at org.apache.spark.storage.BlockManager.dataSerialize(BlockManager.scala:1198) at org.apache.spark.storage.MemoryStore.putArray(MemoryStore.scala:131) at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:168) at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:142) at org.apache.spark.storage.BlockManager.doPut(BlockManager.scala:790) at org.apache.spark.storage.BlockManager.putIterator(BlockManager.scala:637) at org.apache.spark.streaming.receiver.BlockManagerBasedBlockHandler.storeBlock(ReceivedBlockHandler.scala:77) at org.apache.spark.streaming.receiver.ReceiverSupervisorImpl.pushAndReportBlock(ReceiverSupervisorImpl.scala:141) at org.apache.spark.streaming.receiver.ReceiverSupervisorImpl.pushArrayBuffer(ReceiverSupervisorImpl.scala:112) at org.apache.spark.streaming.receiver.ReceiverSupervisorImpl$$anon$2.onPushBlock(ReceiverSupervisorImpl.scala:97) at org.apache.spark.streaming.receiver.BlockGenerator.pushBlock(BlockGenerator.scala:198) at org.apache.spark.streaming.receiver.BlockGenerator.org$apache$spark$streaming$receiver$BlockGenerator$$keepPushingBlocks(BlockGenerator.scala:171) at org.apache.spark.streaming.receiver.BlockGenerator$$anon$1.run(BlockGenerator.scala:89) 15/09/17 08:00:02 WARN ReceiverSupervisorImpl: Reported error Error in block pushing thread - java.io.NotSerializableException: twitter4j.internal.json.ScopesImpl Serialization stack: - object not serializable (class: twitter4j.internal.json.ScopesImpl, value: twitter4j.internal.json.ScopesImpl@357a15bb) - field (class: twitter4j.internal.json.StatusJSONImpl, name: scopes, type: interface twitter4j.Scopes) - object (class twitter4j.internal.json.StatusJSONImpl, StatusJSONImpl{ edited -twitter msg removed}) 15/09/17 08:00:02 WARN ReceiverTracker: Error reported by receiver for stream 0: Error in block pushing thread - java.io.NotSerializableException: twitter4j.internal.json.ScopesImpl Serialization stack: - object not serializable (class: twitter4j.internal.json.ScopesImpl, value: twitter4j.internal.json.ScopesImpl@357a15bb) - field (class: twitter4j.internal.json.StatusJSONImpl, name: scopes, type: interface twitter4j.Scopes) - object (class twitter4j.internal.json.StatusJSONImpl, StatusJSONImpl{ edited -twitter msg removed }) at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40) at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47) at org.apache.spark.serializer.SerializationStream.writeAll(Serializer.scala:153) at org.apache.spark.storage.BlockManager.dataSerializeStream(BlockManager.scala:1189) at org.apache.spark.storage.BlockManager.dataSerialize(BlockManager.scala:1198) at org.apache.spark.storage.MemoryStore.putArray(MemoryStore.scala:131) at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:168) at org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:142) at org.apache.spark.storage.BlockManager.doPut(BlockManager.scala:790) at org.apache.spark.storage.BlockManager.putIterator(BlockManager.scala:637) at org.apache.spark.streaming.receiver.BlockManagerBasedBlockHandler.storeBlock(ReceivedBlockHandler.scala:77) at org.apache.spark.streaming.receiver.ReceiverSupervisorImpl.pushAndReportBlock(ReceiverSupervisorImpl.scala:141) at org.apache.spark.streaming.receiver.ReceiverSupervisorImpl.pushArrayBuffer(ReceiverSupervisorImpl.scala:112) at org.apache.spark.streaming.receiver.ReceiverSupervisorImpl$$anon$2.onPushBlock(ReceiverSupervisorImpl.scala:97) at org.apache.spark.streaming.receiver.BlockGenerator.pushBlock(BlockGenerator.scala:198) at org.apache.spark.streaming.receiver.BlockGenerator.org$apache$spark$streaming$receiver$BlockGenerator$$keepPushingBlocks(BlockGenerator.scala:171) at org.apache.spark.streaming.receiver.BlockGenerator$$anon$1.run(BlockGenerator.scala:89) --------------------------------------------------------------------- To unsubscribe, e-mail: user-unsubscr...@spark.apache.org For additional commands, e-mail: user-h...@spark.apache.org
