Different classloader potentially inside HS2? I know there used to be some interesting cases of classloading when trying to add custom UDFs. Not sure what the current state is.
Kevin Risden On Tue, Mar 5, 2019 at 4:46 PM Shawn Weeks <swe...@weeksconsulting.us> wrote: > Didn't here anything on the dev mailing list, has anyone here seen this > scenario. Custom Hive Function starts throwing class not found errors but > only after a while, the function runs fine most of the time and only has a > single dependency. > > Thanks > Shawn > > -----Original Message----- > From: Shawn Weeks <swe...@weeksconsulting.us> > Sent: Monday, March 4, 2019 1:17 PM > To: d...@hive.apache.org > Subject: Custom UDF Loses Depenencies > > Are there any known gotchas with Hive UDFs that included shaded > dependencies in their jar. I've got an issue where I've built a an xml udtf > based on Saxon HE and after some period of time the hive server starts > raising this exception. The function is created using the permanent > function syntax referencing a jar in HDFS. The jar works initially and then > goes to failing after the instance has been up for a few days. I've > verified that the jar the function uses has these classes in there. > > I'm on Hive 1.2.1 with HDP 2.6.5 patches > > Thanks > Shawn Weeks > > 2019-03-04 19:13:31,175 WARN [HiveServer2-HttpHandler-Pool: > Thread-70490]: servlet.ServletHandler (ServletHandler.java:doHandle(571)) - > Error for /cliservice > java.lang.NoClassDefFoundError: net/sf/saxon/pattern/NodeTest$2 > at net.sf.saxon.pattern.NodeTest.getContentType(NodeTest.java:243) > at > net.sf.saxon.expr.AxisExpression.checkPlausibility(AxisExpression.java:292) > at > net.sf.saxon.expr.AxisExpression.typeCheck(AxisExpression.java:152) > at net.sf.saxon.expr.Operand.typeCheck(Operand.java:185) > at > net.sf.saxon.expr.RawSlashExpression.typeCheck(RawSlashExpression.java:178) > at > net.sf.saxon.sxpath.XPathEvaluator.createExpression(XPathEvaluator.java:141) > at > net.sf.saxon.s9api.XPathCompiler.internalCompile(XPathCompiler.java:514) > at net.sf.saxon.s9api.XPathCompiler.compile(XPathCompiler.java:489) > at > xxx.xxx.xxxGenericUDTFXPathExplode.initialize(GenericUDTFXPathExplode.java:84) > at > org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.initialize(GenericUDTF.java:56) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genUDTFPlan(SemanticAnalyzer.java:7037) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:3986) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genLateralViewPlan(SemanticAnalyzer.java:10077) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genLateralViewPlans(SemanticAnalyzer.java:10021) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9876) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9795) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genOPTree(SemanticAnalyzer.java:10471) > at > org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:342) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10485) > at > org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:219) > at > org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:238) > at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:474) > at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:330) > at > org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1233) > at > org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1227) > at > org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:146) > at > org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:228) > at > org.apache.hive.service.cli.operation.Operation.run(Operation.java:264) > at > org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:479) > at > org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:466) > at > org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:315) > at > org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:509) > at > org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1377) > at > org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1362) > at > org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) > at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) > at org.apache.thrift.server.TServlet.doPost(TServlet.java:83) > at > org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:206) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) > at > org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565) > at > org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479) > at > org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:225) > at > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406) > at > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111) > at org.eclipse.jetty.server.Server.handle(Server.java:349) > at > org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:449) > at > org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:925) > at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:857) > at > org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) > at > org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:76) > at > org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:609) > at > org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:45) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) >
