[
https://issues.apache.org/jira/browse/HIVE-14564?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15959793#comment-15959793
]
Hive QA commented on HIVE-14564:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12862236/HIVE-14564.003.patch
{color:green}SUCCESS:{color} +1 due to 1 test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 149 failed/errored test(s), 10580 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestBeeLineDriver.testCliDriver[drop_with_concurrency]
(batchId=235)
org.apache.hadoop.hive.cli.TestBeeLineDriver.testCliDriver[escape_comments]
(batchId=235)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[autoColumnStats_8]
(batchId=13)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[columnstats_partlvl]
(batchId=33)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[columnstats_partlvl_dp]
(batchId=48)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[columnstats_tbllvl]
(batchId=8)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[complex_alias]
(batchId=16)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[constant_prop_3]
(batchId=41)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[correlationoptimizer13]
(batchId=10)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[decimal_udf] (batchId=9)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[display_colstats_tbllvl]
(batchId=3)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[distinct_windowing_no_cbo]
(batchId=61)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[druid_basic2]
(batchId=10)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[dynamic_rdd_cache]
(batchId=51)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[except_all] (batchId=43)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby9] (batchId=6)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby_join_pushdown]
(batchId=74)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[groupby_position]
(batchId=37)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[having2] (batchId=15)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[index_auto_update]
(batchId=69)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[limit_pushdown_negative]
(batchId=38)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[multi_insert_gby3]
(batchId=70)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[multigroupby_singlemr]
(batchId=65)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[nested_column_pruning]
(batchId=32)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_types_non_dictionary_encoding_vectorization]
(batchId=79)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[parquet_types_vectorization]
(batchId=13)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_gby2] (batchId=81)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_gby] (batchId=35)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ppd_windowing1]
(batchId=42)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[ptfgroupbyjoin]
(batchId=79)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[reduce_deduplicate_extended2]
(batchId=56)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_in_having]
(batchId=54)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_notexists]
(batchId=83)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_notexists_having]
(batchId=79)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_notin_having]
(batchId=47)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[subquery_unqualcolumnrefs]
(batchId=17)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[temp_table_display_colstats_tbllvl]
(batchId=73)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[udaf_binarysetfunctions]
(batchId=35)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_aggregate]
(batchId=17)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_decimal_round_2]
(batchId=22)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_groupby_3]
(batchId=61)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_groupby_reduce]
(batchId=53)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vector_orderby_5]
(batchId=39)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_13]
(batchId=47)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_15]
(batchId=60)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorization_limit]
(batchId=34)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[vectorized_parquet_types]
(batchId=62)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[windowing_gby2]
(batchId=33)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[dynamic_partition_pruning_2]
(batchId=139)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[explainuser_2]
(batchId=140)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[llap_stats]
(batchId=138)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[bucket_groupby]
(batchId=158)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[column_table_stats]
(batchId=156)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[column_table_stats_orc]
(batchId=142)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[explainuser_1]
(batchId=148)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[lateral_view]
(batchId=157)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[limit_pushdown3]
(batchId=147)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[limit_pushdown]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[multi_insert_lateral_view]
(batchId=152)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[offset_limit_ppd_optimizer]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[ptf]
(batchId=144)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[subquery_in]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[subquery_multi]
(batchId=145)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[subquery_notin]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[subquery_scalar]
(batchId=149)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[subquery_select]
(batchId=149)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_decimal_aggregate]
(batchId=144)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_decimal_round_2]
(batchId=146)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_decimal_udf]
(batchId=159)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_groupby_3]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_groupby_reduce]
(batchId=153)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_if_expr]
(batchId=143)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_interval_2]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vector_orderby_5]
(batchId=150)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorization_0]
(batchId=158)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorization_13]
(batchId=151)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorization_15]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorization_short_regress]
(batchId=151)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorized_parquet]
(batchId=152)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorized_parquet_types]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[vectorized_ptf]
(batchId=154)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[windowing_gby]
(batchId=150)
org.apache.hadoop.hive.cli.TestMiniTezCliDriver.testCliDriver[vectorization_limit]
(batchId=96)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query12]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query15]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query16]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query17]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query18]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query19]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query1] (batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query20]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query22]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query25]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query26]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query29]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query30]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query31]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query38]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query39]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query3] (batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query42]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query46]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query51]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query52]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query54]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query55]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query58]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query5] (batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query64]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query65]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query66]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query67]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query68]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query69]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query6] (batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query70]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query71]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query72]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query79]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query7] (batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query81]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query83]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query85]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query87]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query89]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query8] (batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query91]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query92]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query97]
(batchId=230)
org.apache.hadoop.hive.cli.TestPerfCliDriver.testCliDriver[query98]
(batchId=230)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[dynamic_rdd_cache]
(batchId=121)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby9]
(batchId=101)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[groupby_position]
(batchId=114)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[limit_pushdown]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_gby3]
(batchId=130)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multi_insert_lateral_view]
(batchId=120)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[multigroupby_singlemr]
(batchId=127)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[ptf] (batchId=105)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[subquery_in]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vector_decimal_aggregate]
(batchId=106)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vector_groupby_3]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vector_orderby_5]
(batchId=115)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_0]
(batchId=133)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_13]
(batchId=119)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_15]
(batchId=125)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorization_short_regress]
(batchId=118)
org.apache.hadoop.hive.cli.TestSparkCliDriver.testCliDriver[vectorized_ptf]
(batchId=125)
org.apache.hive.hcatalog.pig.TestTextFileHCatStorer.testWriteVarchar
(batchId=178)
org.apache.hive.jdbc.TestJdbcDriver2.testResultSetMetaData (batchId=221)
{noformat}
Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/4587/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/4587/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-4587/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 149 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12862236 - PreCommit-HIVE-Build
> Column Pruning generates out of order columns in SelectOperator which cause
> ArrayIndexOutOfBoundsException.
> -----------------------------------------------------------------------------------------------------------
>
> Key: HIVE-14564
> URL: https://issues.apache.org/jira/browse/HIVE-14564
> Project: Hive
> Issue Type: Bug
> Components: Query Planning
> Affects Versions: 2.1.0
> Reporter: zhihai xu
> Assignee: zhihai xu
> Priority: Critical
> Attachments: HIVE-14564.000.patch, HIVE-14564.001.patch,
> HIVE-14564.002.patch, HIVE-14564.003.patch
>
>
> Column Pruning generates out of order columns in SelectOperator which cause
> ArrayIndexOutOfBoundsException.
> {code}
> 2016-07-26 21:49:24,390 FATAL [main]
> org.apache.hadoop.hive.ql.exec.mr.ExecMapper:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:507)
> at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170)
> at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54)
> at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:453)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
> at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException:
> java.lang.ArrayIndexOutOfBoundsException
> at
> org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.processOp(ReduceSinkOperator.java:397)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:815)
> at
> org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
> at
> org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:497)
> ... 9 more
> Caused by: java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at org.apache.hadoop.io.Text.set(Text.java:225)
> at
> org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryString.init(LazyBinaryString.java:48)
> at
> org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct.uncheckedGetField(LazyBinaryStruct.java:264)
> at
> org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct.getField(LazyBinaryStruct.java:201)
> at
> org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector.getStructFieldData(LazyBinaryStructObjectInspector.java:64)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator._evaluate(ExprNodeColumnEvaluator.java:94)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:77)
> at
> org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:65)
> at
> org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.makeValueWritable(ReduceSinkOperator.java:550)
> at
> org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.processOp(ReduceSinkOperator.java:377)
> ... 13 more
> {code}
> The exception is because the serialization and deserialization doesn't match.
> The serialization by LazyBinarySerDe from previous MapReduce job used
> different order of columns. When the current MapReduce job deserialized the
> intermediate sequence file generated by previous MapReduce job, it will get
> corrupted data from the deserialization using wrong order of columns by
> LazyBinaryStruct. The unmatched columns between serialization and
> deserialization is caused by SelectOperator's Column Pruning
> {{ColumnPrunerSelectProc}}.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
