[
https://issues.apache.org/jira/browse/HIVE-13353?focusedWorklogId=815109&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-815109
]
ASF GitHub Bot logged work on HIVE-13353:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 10/Oct/22 07:36
Start Date: 10/Oct/22 07:36
Worklog Time Spent: 10m
Work Description: veghlaci05 commented on code in PR #3608:
URL: https://github.com/apache/hive/pull/3608#discussion_r990966219
##########
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java:
##########
@@ -2117,4 +2121,168 @@ public void testIsRawFormatFile() throws Exception {
List<String> res = runStatementOnDriver("select * from file_formats");
Assert.assertEquals(3, res.size());
}
+ @Test
+ public void testShowCompactions() throws Exception {
+ d.destroy();
+ hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+ d = new Driver(hiveConf);
+ //generate some compaction history
+ runStatementOnDriver("drop database if exists mydb1 cascade");
+ runStatementOnDriver("create database mydb1");
+ runStatementOnDriver("create table mydb1.tbl0 " + "(a int, b int)
partitioned by (p string) clustered by (a) into " +
+ BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES
('transactional'='true')");
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(1,2,'p1'),(3,4,'p1'),(1,2,'p2'),(3,4,'p2'),(1,2,'p3'),(3,4,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p3')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(4,5,'p1'),(6,7,'p1'),(4,5,'p2'),(6,7,'p2'),(4,5,'p3'),(6,7,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p3')
compact 'MAJOR' pool 'pool0'");
+ TestTxnCommands2.runWorker(hiveConf);
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+
+ SessionState.get().setCurrentDatabase("mydb1");
+
+ //testing show compaction command
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<String> r = runStatementOnDriver("SHOW COMPACTIONS");
+ Assert.assertEquals(rsp.getCompacts().size()+1, r.size());//includes
Header row
+
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 STATUS 'ready for
cleaning'");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getState().equals("ready
for cleaning")).count() +1,
Review Comment:
Your are only checking here that the driver command result contains the same
amount of records with ready for cleaning state, than rsp has. You should also
compare (at least partially) the content of the driver result. What if the
returned counts are equal but the returned row is something completely
different?
##########
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java:
##########
@@ -2117,4 +2121,168 @@ public void testIsRawFormatFile() throws Exception {
List<String> res = runStatementOnDriver("select * from file_formats");
Assert.assertEquals(3, res.size());
}
+ @Test
+ public void testShowCompactions() throws Exception {
+ d.destroy();
+ hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+ d = new Driver(hiveConf);
+ //generate some compaction history
+ runStatementOnDriver("drop database if exists mydb1 cascade");
+ runStatementOnDriver("create database mydb1");
+ runStatementOnDriver("create table mydb1.tbl0 " + "(a int, b int)
partitioned by (p string) clustered by (a) into " +
+ BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES
('transactional'='true')");
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(1,2,'p1'),(3,4,'p1'),(1,2,'p2'),(3,4,'p2'),(1,2,'p3'),(3,4,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p3')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(4,5,'p1'),(6,7,'p1'),(4,5,'p2'),(6,7,'p2'),(4,5,'p3'),(6,7,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p3')
compact 'MAJOR' pool 'pool0'");
+ TestTxnCommands2.runWorker(hiveConf);
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+
+ SessionState.get().setCurrentDatabase("mydb1");
+
+ //testing show compaction command
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<String> r = runStatementOnDriver("SHOW COMPACTIONS");
+ Assert.assertEquals(rsp.getCompacts().size()+1, r.size());//includes
Header row
+
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 STATUS 'ready for
cleaning'");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getState().equals("ready
for cleaning")).count() +1,
+ r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 TYPE 'MAJOR' ");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+ filter(x->x.getType().equals(CompactionType.MAJOR)).count()+1,
r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'poolx' TYPE
'MINOR' ");
+ //includes Header row
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+
filter(x->x.getPoolName().equals("poolx")).filter(x->x.getType().equals(CompactionType.MAJOR)).count()+1,
r.size());
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'pool0' TYPE
'MAJOR'");
+ Assert.assertEquals(2, r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'pool0'");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+ filter(x->x.getPoolName().equals("pool0")).count()+1,
r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS DATABASE mydb1 POOL 'pool0'");
+ Assert.assertEquals(2, r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS tbl0 TYPE 'MAJOR' ");
Review Comment:
Ok. Altough I still think this is a bit odd, I discussed it with @kasakrisz
and he mentioned that SHOW LOCKS works the same way and you agreed to copy it.
For consistency between these similar commands I think it's fine.
Issue Time Tracking
-------------------
Worklog Id: (was: 815109)
Time Spent: 3h (was: 2h 50m)
> SHOW COMPACTIONS should support filtering options
> -------------------------------------------------
>
> Key: HIVE-13353
> URL: https://issues.apache.org/jira/browse/HIVE-13353
> Project: Hive
> Issue Type: Improvement
> Components: Transactions
> Affects Versions: 1.3.0, 2.0.0
> Reporter: Eugene Koifman
> Assignee: KIRTI RUGE
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-13353.01.patch
>
> Time Spent: 3h
> Remaining Estimate: 0h
>
> Since we now have historical information in SHOW COMPACTIONS the output can
> easily become unwieldy. (e.g. 1000 partitions with 3 lines of history each)
> this is a significant usability issue
> Need to add ability to filter by db/table/partition
> Perhaps would also be useful to filter by status
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
