[ https://issues.apache.org/jira/browse/HIVE-18009?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16248270#comment-16248270 ]
Hive QA commented on HIVE-18009: -------------------------------- Here are the results of testing the latest attachment: https://issues.apache.org/jira/secure/attachment/12897115/HIVE-18009.3.patch {color:green}SUCCESS:{color} +1 due to 1 test(s) being added or modified. {color:red}ERROR:{color} -1 due to 11 failed/errored test(s), 11376 tests executed *Failed tests:* {noformat} org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[dbtxnmgr_showlocks] (batchId=77) org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[unionDistinct_1] (batchId=146) org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[insert_values_orig_table_use_metadata] (batchId=162) org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[sysdb] (batchId=156) org.apache.hadoop.hive.cli.TestNegativeMinimrCliDriver.testCliDriver[ct_noperm_loc] (batchId=94) org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[subquery_multi] (batchId=111) org.apache.hadoop.hive.cli.control.TestDanglingQOuts.checkDanglingQOut (batchId=206) org.apache.hadoop.hive.ql.TestTxnCommands.testMergeOnTezEdges (batchId=288) org.apache.hadoop.hive.ql.parse.TestReplicationScenarios.testConstraints (batchId=223) org.apache.hive.hcatalog.pig.TestHCatLoaderComplexSchema.testSyntheticComplexSchema[4] (batchId=187) org.apache.hive.hcatalog.pig.TestHCatLoaderComplexSchema.testTupleInBagInTupleInBag[3] (batchId=187) {noformat} Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/7768/testReport Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/7768/console Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-7768/ Messages: {noformat} Executing org.apache.hive.ptest.execution.TestCheckPhase Executing org.apache.hive.ptest.execution.PrepPhase Executing org.apache.hive.ptest.execution.ExecutionPhase Executing org.apache.hive.ptest.execution.ReportingPhase Tests exited with: TestsFailedException: 11 tests failed {noformat} This message is automatically generated. ATTACHMENT ID: 12897115 - PreCommit-HIVE-Build > Multiple lateral view query is slow on hive on spark > ---------------------------------------------------- > > Key: HIVE-18009 > URL: https://issues.apache.org/jira/browse/HIVE-18009 > Project: Hive > Issue Type: Improvement > Components: Spark > Affects Versions: 3.0.0 > Reporter: Aihua Xu > Assignee: Aihua Xu > Attachments: HIVE-18009.1.patch, HIVE-18009.2.patch, > HIVE-18009.3.patch > > > When running the query with multiple lateral view, HoS is busy with the > compilation. GenSparkUtils has an efficient implementation of > getChildOperator when we have diamond hierarchy in operator trees (lateral > view in this case) since the node may be visited multiple times. > {noformat} > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:442) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.getChildOperator(GenSparkUtils.java:438) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWork.process(GenSparkWork.java:104) > at > org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90) > at > org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:90) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.walk(GenSparkWorkWalker.java:106) > at > org.apache.hadoop.hive.ql.parse.spark.GenSparkWorkWalker.startWalking(GenSparkWorkWalker.java:72) > at > org.apache.hadoop.hive.ql.parse.spark.SparkCompiler.generateTaskTree(SparkCompiler.java:225) > at > org.apache.hadoop.hive.ql.parse.TaskCompiler.compile(TaskCompiler.java:204) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10083) > at > org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:9876) > at > org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:222) > at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:445) > at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:311) > at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1189) > at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1284) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1115) > at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1103) > at > org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:220) > at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:172) > at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:383) > at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:318) > at > org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:416) > at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:432) > at > org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:726) > at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:693) > at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:628) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at org.apache.hadoop.util.RunJar.run(RunJar.java:221) > at org.apache.hadoop.util.RunJar.main(RunJar.java:136) > {noformat} -- This message was sent by Atlassian JIRA (v6.4.14#64029)