[
https://issues.apache.org/jira/browse/HIVE-22879?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17040728#comment-17040728
]
Zoltan Haindrich commented on HIVE-22879:
-----------------------------------------
[~rajesh.balamohan] I think HIVE-22881 should make this issue go away - could
you test it ?
In case you are looking at profiler outputs: keep an eye on the total time
spent in and under the RexSimplify class; especially in case of complex
filter/join conditions - let me know if it eats up resources for no good reason
:D
It's not something which is easily fixable; but it would be good to know how
much it would worth focusing on.
> Optimise jar file loading in CalcitePlanner
> -------------------------------------------
>
> Key: HIVE-22879
> URL: https://issues.apache.org/jira/browse/HIVE-22879
> Project: Hive
> Issue Type: Improvement
> Components: CBO
> Reporter: Rajesh Balamohan
> Priority: Major
>
> {{CalcitePlanner}} internally uses {{org.codehaus.janino.UnitCompiler
> (calcite dependency)}} and this appears to load the jars in every thread.
> Need to check if this can be avoided.
> Here is an example.
> {noformat}
> at java.util.zip.ZipFile.getEntry(Native Method)
> at java.util.zip.ZipFile.getEntry(ZipFile.java:310)
> - locked <0x00000005c1af21c0> (a java.util.jar.JarFile)
> at java.util.jar.JarFile.getEntry(JarFile.java:240)
> at java.util.jar.JarFile.getJarEntry(JarFile.java:223)
> at sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:1005)
> at sun.misc.URLClassPath.getResource(URLClassPath.java:212)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:365)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
> - locked <0x00000005caa3be88> (a java.lang.Object)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:348)
> at
> org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89)
> at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:312)
> - locked <0x0000000686136868> (a
> org.codehaus.janino.ClassLoaderIClassLoader)
> at
> org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8556)
> at
> org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8478)
> at
> org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8471)
> at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8331)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6855)
> at org.codehaus.janino.UnitCompiler.access$14200(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6497)
> at
> org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6494)
> at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4224)
> at
> org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6494)
> at
> org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6490)
> at org.codehaus.janino.Java$Lvalue.accept(Java.java:4148)
> at
> org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
> at
> org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9026)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7106)
> at org.codehaus.janino.UnitCompiler.access$15800(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6517)
> at
> org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6490)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
> at
> org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
> at
> org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
> at
> org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9237)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9123)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9025)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5062)
> at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4423)
> at
> org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4396)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5622)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5592)
> at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4434)
> at
> org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4396)
> at org.codehaus.janino.Java$NewInitializedArray.accept(Java.java:5373)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2649)
> at org.codehaus.janino.UnitCompiler.access$2800(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1504)
> at
> org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1487)
> at org.codehaus.janino.Java$ReturnStatement.accept(Java.java:3563)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
> at
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
> at
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
> at
> org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
> at
> org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:465)
> at
> org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313)
> at
> org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235)
> at org.apache.calcite.rex.RexExecutable.compile(RexExecutable.java:60)
> at org.apache.calcite.rex.RexExecutable.<init>(RexExecutable.java:49)
> at
> org.apache.calcite.rex.RexExecutorImpl.reduce(RexExecutorImpl.java:127)
> at
> org.apache.calcite.rex.RexSimplify.simplifyCast(RexSimplify.java:1810)
> at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:270)
> at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
> at
> org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:321)
> at
> org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:313)
> at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:288)
> at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
> at org.apache.calcite.rex.RexSimplify.simplifyAnds(RexSimplify.java:463)
> at org.apache.calcite.rex.RexUtil.simplifyAnds(RexUtil.java:1748)
> at
> org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:759)
> at
> org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:749)
> at
> org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule.onMatch(HiveSubQueryRemoveRule.java:115)
> at
> org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(AbstractRelOptPlanner.java:319)
> at org.apache.calcite.plan.hep.HepPlanner.applyRule(HepPlanner.java:560)
> at
> org.apache.calcite.plan.hep.HepPlanner.applyRules(HepPlanner.java:419)
> at
> org.apache.calcite.plan.hep.HepPlanner.executeInstruction(HepPlanner.java:256)
> at
> org.apache.calcite.plan.hep.HepInstruction$RuleInstance.execute(HepInstruction.java:127)
> at
> org.apache.calcite.plan.hep.HepPlanner.executeProgram(HepPlanner.java:215)
> at
> org.apache.calcite.plan.hep.HepPlanner.findBestExp(HepPlanner.java:202)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2518)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2484)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.hepPlan(CalcitePlanner.java:2459)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1814)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1731)
> at
> org.apache.calcite.tools.Frameworks.lambda$withPlanner$0(Frameworks.java:130)
> at
> org.apache.calcite.tools.Frameworks$$Lambda$162/385313421.apply(Unknown
> Source)
> at
> org.apache.calcite.prepare.CalcitePrepareImpl.perform(CalcitePrepareImpl.java:915)
> at org.apache.calcite.tools.Frameworks.withPrepare(Frameworks.java:179)
> at org.apache.calcite.tools.Frameworks.withPlanner(Frameworks.java:125)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner.logicalPlan(CalcitePlanner.java:1492)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:468)
> at
> org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12488)
> at
> org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:358)
> at
> org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
> at
> org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:171)
> at
> org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
> at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:219)
> at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:103)
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:215)
> at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:828)
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:774)
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:768)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:125)
> at
> org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:203)
> at
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:325)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1688)
> at
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:345)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
