Didn't here anything on the dev mailing list, has anyone here seen this
scenario. Custom Hive Function starts throwing class not found errors but only
after a while, the function runs fine most of the time and only has a single
dependency.
Thanks
Shawn
-----Original Message-----
From: Shawn Weeks <swe...@weeksconsulting.us>
Sent: Monday, March 4, 2019 1:17 PM
To: d...@hive.apache.org
Subject: Custom UDF Loses Depenencies
Are there any known gotchas with Hive UDFs that included shaded dependencies in
their jar. I've got an issue where I've built a an xml udtf based on Saxon HE
and after some period of time the hive server starts raising this exception.
The function is created using the permanent function syntax referencing a jar
in HDFS. The jar works initially and then goes to failing after the instance
has been up for a few days. I've verified that the jar the function uses has
these classes in there.
I'm on Hive 1.2.1 with HDP 2.6.5 patches
Thanks
Shawn Weeks
2019-03-04 19:13:31,175 WARN [HiveServer2-HttpHandler-Pool: Thread-70490]:
servlet.ServletHandler (ServletHandler.java:doHandle(571)) - Error for
/cliservice
java.lang.NoClassDefFoundError: net/sf/saxon/pattern/NodeTest$2
at net.sf.saxon.pattern.NodeTest.getContentType(NodeTest.java:243)
at
net.sf.saxon.expr.AxisExpression.checkPlausibility(AxisExpression.java:292)
at net.sf.saxon.expr.AxisExpression.typeCheck(AxisExpression.java:152)
at net.sf.saxon.expr.Operand.typeCheck(Operand.java:185)
at
net.sf.saxon.expr.RawSlashExpression.typeCheck(RawSlashExpression.java:178)
at
net.sf.saxon.sxpath.XPathEvaluator.createExpression(XPathEvaluator.java:141)
at
net.sf.saxon.s9api.XPathCompiler.internalCompile(XPathCompiler.java:514)
at net.sf.saxon.s9api.XPathCompiler.compile(XPathCompiler.java:489)
at
xxx.xxx.xxxGenericUDTFXPathExplode.initialize(GenericUDTFXPathExplode.java:84)
at
org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.initialize(GenericUDTF.java:56)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genUDTFPlan(SemanticAnalyzer.java:7037)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:3986)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genLateralViewPlan(SemanticAnalyzer.java:10077)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genLateralViewPlans(SemanticAnalyzer.java:10021)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9876)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9782)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9809)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9795)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genOPTree(SemanticAnalyzer.java:10471)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:342)
at
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10485)
at
org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:219)
at
org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:238)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:474)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:330)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1233)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1227)
at
org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:146)
at
org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:228)
at
org.apache.hive.service.cli.operation.Operation.run(Operation.java:264)
at
org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:479)
at
org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:466)
at
org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:315)
at
org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:509)
at
org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1377)
at
org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1362)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.thrift.server.TServlet.doPost(TServlet.java:83)
at
org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:206)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
at
org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:225)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111)
at org.eclipse.jetty.server.Server.handle(Server.java:349)
at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:449)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:925)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:857)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at
org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:76)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:609)
at
org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:45)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
