[
https://issues.apache.org/jira/browse/HIVE-21225?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16871097#comment-16871097
]
Hive QA commented on HIVE-21225:
--------------------------------
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12972698/HIVE-21225.3.patch
{color:green}SUCCESS:{color} +1 due to 2 test(s) being added or modified.
{color:red}ERROR:{color} -1 due to 64 failed/errored test(s), 16339 tests
executed
*Failed tests:*
{noformat}
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[acid_nullscan]
(batchId=73)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[acid_stats5] (batchId=23)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[autoColumnStats_4]
(batchId=13)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[mm_all] (batchId=74)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[row__id] (batchId=86)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[stats_nonpart]
(batchId=14)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[stats_part2] (batchId=22)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[stats_part] (batchId=52)
org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver[stats_sizebug]
(batchId=89)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[acid_bucket_pruning]
(batchId=155)
org.apache.hadoop.hive.cli.TestMiniLlapCliDriver.testCliDriver[mm_all]
(batchId=157)
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[mm_exim]
(batchId=186)
org.apache.hadoop.hive.ql.TestTxnCommands.testMmExim (batchId=341)
org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion1
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion2
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion3
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnCommands2.testOriginalFileReaderWhenNonAcidConvertedToAcid
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnCommands2.updateDeletePartitioned (batchId=322)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion1
(batchId=336)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion2
(batchId=336)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion3
(batchId=336)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testOriginalFileReaderWhenNonAcidConvertedToAcid
(batchId=336)
org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.updateDeletePartitioned
(batchId=336)
org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testMmExim
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnExIm.testImport (batchId=322)
org.apache.hadoop.hive.ql.TestTxnExIm.testImportNoTarget (batchId=322)
org.apache.hadoop.hive.ql.TestTxnExIm.testMM (batchId=322)
org.apache.hadoop.hive.ql.TestTxnExIm.testMMCreate (batchId=322)
org.apache.hadoop.hive.ql.TestTxnLoadData.loadData (batchId=298)
org.apache.hadoop.hive.ql.TestTxnLoadData.loadDataNonAcid2AcidConversion
(batchId=298)
org.apache.hadoop.hive.ql.TestTxnLoadData.loadDataUpdate (batchId=298)
org.apache.hadoop.hive.ql.TestTxnLoadData.testMultiStatement (batchId=298)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testCompactStatsGather (batchId=322)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testEmptyCompactionResult
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnNoBuckets.testToAcidConversionMultiBucket
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testCompactStatsGather
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testEmptyCompactionResult
(batchId=322)
org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testToAcidConversionMultiBucket
(batchId=322)
org.apache.hadoop.hive.ql.io.TestAcidUtils.testBaseDeltas (batchId=310)
org.apache.hadoop.hive.ql.io.TestAcidUtils.testBaseWithDeleteDeltas
(batchId=310)
org.apache.hadoop.hive.ql.io.TestAcidUtils.testObsoleteOriginals (batchId=310)
org.apache.hadoop.hive.ql.io.TestAcidUtils.testOriginal (batchId=310)
org.apache.hadoop.hive.ql.io.TestAcidUtils.testOriginalDeltas (batchId=310)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testACIDReaderFooterSerialize
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testACIDReaderFooterSerializeWithDeltas
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testACIDReaderNoFooterSerialize
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testACIDReaderNoFooterSerializeWithDeltas
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testEmptyFile
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testSplitGenReadOps
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testSplitGenReadOpsLocalCache
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testSplitGenReadOpsLocalCacheChangeFileLen
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.testSplitGenReadOpsLocalCacheChangeModificationTime
(batchId=313)
org.apache.hadoop.hive.ql.io.orc.TestOrcRawRecordMerger.testOriginalReaderPair
(batchId=314)
org.apache.hadoop.hive.ql.io.orc.TestOrcRawRecordMerger.testOriginalReaderPairNoMin
(batchId=314)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesWithJsonMessage.testDumpAcidTableWithPartitionDirMissing
(batchId=260)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesWithJsonMessage.testDumpAcidTableWithTableDirMissing
(batchId=260)
org.apache.hadoop.hive.ql.parse.TestReplAcidTablesWithJsonMessage.testMultiDBTxn
(batchId=260)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTables.testDumpAcidTableWithPartitionDirMissing
(batchId=262)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTables.testDumpAcidTableWithTableDirMissing
(batchId=262)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosAcidTables.testMultiDBTxn
(batchId=262)
org.apache.hadoop.hive.ql.parse.TestReplicationScenariosIncrementalLoadAcidTables.testAcidTableIncrementalReplication
(batchId=263)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testStatsAfterCompactionPartTbl
(batchId=245)
org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testStatsAfterCompactionPartTblNew
(batchId=245)
org.apache.hadoop.hive.ql.util.TestUpgradeTool.testPostUpgrade (batchId=298)
{noformat}
Test results:
https://builds.apache.org/job/PreCommit-HIVE-Build/17705/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/17705/console
Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-17705/
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.TestCheckPhase
Executing org.apache.hive.ptest.execution.PrepPhase
Executing org.apache.hive.ptest.execution.YetusPhase
Executing org.apache.hive.ptest.execution.ExecutionPhase
Executing org.apache.hive.ptest.execution.ReportingPhase
Tests exited with: TestsFailedException: 64 tests failed
{noformat}
This message is automatically generated.
ATTACHMENT ID: 12972698 - PreCommit-HIVE-Build
> ACID: getAcidState() should cache a recursive dir listing locally
> -----------------------------------------------------------------
>
> Key: HIVE-21225
> URL: https://issues.apache.org/jira/browse/HIVE-21225
> Project: Hive
> Issue Type: Improvement
> Components: Transactions
> Reporter: Gopal V
> Assignee: Vaibhav Gumashta
> Priority: Major
> Attachments: HIVE-21225.1.patch, HIVE-21225.2.patch,
> HIVE-21225.3.patch, async-pid-44-2.svg
>
>
> Currently getAcidState() makes 3 calls into the FS api which could be
> answered by making a single recursive listDir call and reusing the same data
> to check for isRawFormat() and isValidBase().
> All delta operations for a single partition can go against a single listed
> directory snapshot instead of interacting with the NameNode or ObjectStore
> within the inner loop.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
