[
https://issues.apache.org/jira/browse/HIVE-4051?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13724749#comment-13724749
]
Hive QA commented on HIVE-4051:
-------------------------------
{color:red}Overall{color}: -1 no tests executed
Here are the results of testing the latest attachment:
https://issues.apache.org/jira/secure/attachment/12595079/HIVE-4051.D11805.4.patch
Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/252/testReport
Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/252/console
Messages:
{noformat}
Executing org.apache.hive.ptest.execution.PrepPhase
Tests failed with: NonZeroExitCodeException: Command 'bash
/data/hive-ptest/working/scratch/source-prep.sh' failed with exit status 1 and
output '+ [[ -n '' ]]
+ export 'ANT_OPTS=-Xmx1g -XX:MaxPermSize=256m -Dhttp.proxyHost=localhost
-Dhttp.proxyPort=3128'
+ ANT_OPTS='-Xmx1g -XX:MaxPermSize=256m -Dhttp.proxyHost=localhost
-Dhttp.proxyPort=3128'
+ cd /data/hive-ptest/working/
+ tee /data/hive-ptest/logs/PreCommit-HIVE-Build-252/source-prep.txt
+ mkdir -p maven ivy
+ [[ svn = \s\v\n ]]
+ [[ -n '' ]]
+ [[ -d apache-svn-trunk-source ]]
+ [[ ! -d apache-svn-trunk-source/.svn ]]
+ [[ ! -d apache-svn-trunk-source ]]
+ cd apache-svn-trunk-source
+ svn revert -R .
++ awk '{print $2}'
++ egrep -v '^X|^Performing status on external'
++ svn status --no-ignore
+ rm -rf
+ svn update
U testutils/ptest2/src/test/resources/SomeTest-success.xml
U testutils/ptest2/src/test/resources/test-outputs/SomeTest-truncated.xml
U
testutils/ptest2/src/test/resources/test-outputs/skewjoin_union_remove_1.q-TEST-org.apache.hadoop.hive.cli.TestCliDriver.xml
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingUnitTest.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestCleanupPhase.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPhase.testRsyncFromLocalToRemoteInstancesWithFailureUnknown.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestCleanupPhase.testExecute.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPhase.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPrepPhase.testExecute.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPhase.testRsyncFromLocalToRemoteInstancesWithFailureOne.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingUnitTest.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingQFileTest.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockSSHCommandExecutor.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockRSyncCommandExecutor.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestReportParser.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingQFile.approved.txt
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockLocalCommandFactory.java
U
testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestExecutor.java
U testutils/ptest2/src/main/resources/batch-exec.vm
U testutils/ptest2/src/main/resources/source-prep.vm
U testutils/ptest2/src/main/resources/log4j.properties
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ReportingPhase.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
U testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java
U testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java
A
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutorBuilder.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java
U testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java
U
testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/AbstractSSHCommand.java
U eclipse-templates/.classpath
U eclipse-templates/.classpath._hbase
Fetching external item into 'hcatalog/src/test/e2e/harness'
Updated external to revision 1508709.
Updated to revision 1508708.
+ patchCommandPath=/data/hive-ptest/working/scratch/smart-apply-patch.sh
+ patchFilePath=/data/hive-ptest/working/scratch/build.patch
+ [[ -f /data/hive-ptest/working/scratch/build.patch ]]
+ chmod +x /data/hive-ptest/working/scratch/smart-apply-patch.sh
+ /data/hive-ptest/working/scratch/smart-apply-patch.sh
/data/hive-ptest/working/scratch/build.patch
The patch does not appear to apply with p0 to p2
+ exit 1
'
{noformat}
This message is automatically generated.
> Hive's metastore suffers from 1+N queries when querying partitions & is slow
> ----------------------------------------------------------------------------
>
> Key: HIVE-4051
> URL: https://issues.apache.org/jira/browse/HIVE-4051
> Project: Hive
> Issue Type: Bug
> Components: Clients, Metastore
> Environment: RHEL 6.3 / EC2 C1.XL
> Reporter: Gopal V
> Assignee: Sergey Shelukhin
> Attachments: HIVE-4051.D11805.1.patch, HIVE-4051.D11805.2.patch,
> HIVE-4051.D11805.3.patch, HIVE-4051.D11805.4.patch
>
>
> Hive's query client takes a long time to initialize & start planning queries
> because of delays in creating all the MTable/MPartition objects.
> For a hive db with 1800 partitions, the metastore took 6-7 seconds to
> initialize - firing approximately 5900 queries to the mysql database.
> Several of those queries fetch exactly one row to create a single object on
> the client.
> The following 12 queries were repeated for each partition, generating a storm
> of SQL queries
> {code}
> 4 Query SELECT
> `A0`.`SD_ID`,`B0`.`INPUT_FORMAT`,`B0`.`IS_COMPRESSED`,`B0`.`IS_STOREDASSUBDIRECTORIES`,`B0`.`LOCATION`,`B0`.`NUM_BUCKETS`,`B0`.`OUTPUT_FORMAT`,`B0`.`SD_ID`
> FROM `PARTITIONS` `A0` LEFT OUTER JOIN `SDS` `B0` ON `A0`.`SD_ID` =
> `B0`.`SD_ID` WHERE `A0`.`PART_ID` = 3945
> 4 Query SELECT `A0`.`CD_ID`,`B0`.`CD_ID` FROM `SDS` `A0` LEFT OUTER JOIN
> `CDS` `B0` ON `A0`.`CD_ID` = `B0`.`CD_ID` WHERE `A0`.`SD_ID` =4871
> 4 Query SELECT COUNT(*) FROM `COLUMNS_V2` THIS WHERE THIS.`CD_ID`=1546
> AND THIS.`INTEGER_IDX`>=0
> 4 Query SELECT
> `A0`.`COMMENT`,`A0`.`COLUMN_NAME`,`A0`.`TYPE_NAME`,`A0`.`INTEGER_IDX` AS
> NUCORDER0 FROM `COLUMNS_V2` `A0` WHERE `A0`.`CD_ID` = 1546 AND
> `A0`.`INTEGER_IDX` >= 0 ORDER BY NUCORDER0
> 4 Query SELECT `A0`.`SERDE_ID`,`B0`.`NAME`,`B0`.`SLIB`,`B0`.`SERDE_ID`
> FROM `SDS` `A0` LEFT OUTER JOIN `SERDES` `B0` ON `A0`.`SERDE_ID` =
> `B0`.`SERDE_ID` WHERE `A0`.`SD_ID` =4871
> 4 Query SELECT COUNT(*) FROM `SORT_COLS` THIS WHERE THIS.`SD_ID`=4871 AND
> THIS.`INTEGER_IDX`>=0
> 4 Query SELECT `A0`.`COLUMN_NAME`,`A0`.`ORDER`,`A0`.`INTEGER_IDX` AS
> NUCORDER0 FROM `SORT_COLS` `A0` WHERE `A0`.`SD_ID` =4871 AND
> `A0`.`INTEGER_IDX` >= 0 ORDER BY NUCORDER0
> 4 Query SELECT COUNT(*) FROM `SKEWED_VALUES` THIS WHERE
> THIS.`SD_ID_OID`=4871 AND THIS.`INTEGER_IDX`>=0
> 4 Query SELECT 'org.apache.hadoop.hive.metastore.model.MStringList' AS
> NUCLEUS_TYPE,`A1`.`STRING_LIST_ID`,`A0`.`INTEGER_IDX` AS NUCORDER0 FROM
> `SKEWED_VALUES` `A0` INNER JOIN `SKEWED_STRING_LIST` `A1` ON
> `A0`.`STRING_LIST_ID_EID` = `A1`.`STRING_LIST_ID` WHERE `A0`.`SD_ID_OID`
> =4871 AND `A0`.`INTEGER_IDX` >= 0 ORDER BY NUCORDER0
> 4 Query SELECT COUNT(*) FROM `SKEWED_COL_VALUE_LOC_MAP` WHERE `SD_ID`
> =4871 AND `STRING_LIST_ID_KID` IS NOT NULL
> 4 Query SELECT 'org.apache.hadoop.hive.metastore.model.MStringList' AS
> NUCLEUS_TYPE,`A0`.`STRING_LIST_ID` FROM `SKEWED_STRING_LIST` `A0` INNER JOIN
> `SKEWED_COL_VALUE_LOC_MAP` `B0` ON `A0`.`STRING_LIST_ID` =
> `B0`.`STRING_LIST_ID_KID` WHERE `B0`.`SD_ID` =4871
> 4 Query SELECT `A0`.`STRING_LIST_ID_KID`,`A0`.`LOCATION` FROM
> `SKEWED_COL_VALUE_LOC_MAP` `A0` WHERE `A0`.`SD_ID` =4871 AND NOT
> (`A0`.`STRING_LIST_ID_KID` IS NULL)
> {code}
> This data is not detached or cached, so this operation is performed during
> every query plan for the partitions, even in the same hive client.
> The queries are automatically generated by JDO/DataNucleus which makes it
> nearly impossible to rewrite it into a single denormalized join operation &
> process it locally.
> Attempts to optimize this with JDO fetch-groups did not bear fruit in
> improving the query count.
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
