[
https://issues.apache.org/jira/browse/HIVE-13353?focusedWorklogId=815587&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-815587
]
ASF GitHub Bot logged work on HIVE-13353:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 11/Oct/22 09:54
Start Date: 11/Oct/22 09:54
Worklog Time Spent: 10m
Work Description: veghlaci05 commented on code in PR #3608:
URL: https://github.com/apache/hive/pull/3608#discussion_r992100677
##########
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java:
##########
@@ -2117,4 +2121,211 @@ public void testIsRawFormatFile() throws Exception {
List<String> res = runStatementOnDriver("select * from file_formats");
Assert.assertEquals(3, res.size());
}
+ @Test
+ public void testShowCompactions() throws Exception {
+ d.destroy();
+ hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+ d = new Driver(hiveConf);
+ //generate some compaction history
+ runStatementOnDriver("drop database if exists mydb1 cascade");
+ runStatementOnDriver("create database mydb1");
+ runStatementOnDriver("create table mydb1.tbl0 " + "(a int, b int)
partitioned by (p string) clustered by (a) into " +
+ BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES
('transactional'='true')");
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(1,2,'p1'),(3,4,'p1'),(1,2,'p2'),(3,4,'p2'),(1,2,'p3'),(3,4,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p3')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(4,5,'p1'),(6,7,'p1'),(4,5,'p2'),(6,7,'p2'),(4,5,'p3'),(6,7,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p3')
compact 'MAJOR' pool 'pool0'");
+ TestTxnCommands2.runWorker(hiveConf);
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+
+ SessionState.get().setCurrentDatabase("mydb1");
+
+ //testing show compaction command
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<String> r = runStatementOnDriver("SHOW COMPACTIONS");
+ Assert.assertEquals(rsp.getCompacts().size()+1, r.size());//includes
Header row
+
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 STATUS 'ready for
cleaning'");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getState().equals("ready
for cleaning")).count() +1,
+ r.size());//includes Header row
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 TYPE 'MAJOR' ");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+ filter(x->x.getType().equals(CompactionType.MAJOR)).count()+1,
r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'poolx' TYPE
'MINOR' ");
+ //includes Header row
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+
filter(x->x.getPoolName().equals("poolx")).filter(x->x.getType().equals(CompactionType.MINOR)).count(),
+
r.stream().filter(x->x.equals("mydb1")).filter(x->x.equals("poolx")).filter(x->x.equals(CompactionType.MINOR)).count());
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'pool0' TYPE
'MAJOR'");
+ Assert.assertEquals(2, r.size());//includes Header row
+
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'pool0'");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+ filter(x->x.getPoolName().equals("pool0")).count()+1,
r.size());//includes Header row
+ r = runStatementOnDriver("SHOW COMPACTIONS DATABASE mydb1 POOL 'pool0'");
+ Assert.assertEquals(2, r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS tbl0 TYPE 'MAJOR' ");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getTablename().equals("tbl0")).
+ filter(x->x.getType().equals(CompactionType.MAJOR)).count()+1,
r.size());//includes Header row
+
+ List<String> partitions = runStatementOnDriver("SHOW PARTITIONS mydb1.tbl0
");
Review Comment:
Is this intended? Shouldn't it be rather SHOW COMPACTIONS with partition
filtering?
##########
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java:
##########
@@ -2117,4 +2121,211 @@ public void testIsRawFormatFile() throws Exception {
List<String> res = runStatementOnDriver("select * from file_formats");
Assert.assertEquals(3, res.size());
}
+ @Test
+ public void testShowCompactions() throws Exception {
+ d.destroy();
+ hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+ d = new Driver(hiveConf);
+ //generate some compaction history
+ runStatementOnDriver("drop database if exists mydb1 cascade");
+ runStatementOnDriver("create database mydb1");
+ runStatementOnDriver("create table mydb1.tbl0 " + "(a int, b int)
partitioned by (p string) clustered by (a) into " +
+ BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES
('transactional'='true')");
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(1,2,'p1'),(3,4,'p1'),(1,2,'p2'),(3,4,'p2'),(1,2,'p3'),(3,4,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION(p='p3')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("insert into mydb1.tbl0" + " PARTITION(p) " +
+ "
values(4,5,'p1'),(6,7,'p1'),(4,5,'p2'),(6,7,'p2'),(4,5,'p3'),(6,7,'p3')");
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p1')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p2')
compact 'MAJOR'");
+ TestTxnCommands2.runWorker(hiveConf);
+ runStatementOnDriver("alter table mydb1.tbl0" + " PARTITION (p='p3')
compact 'MAJOR' pool 'pool0'");
+ TestTxnCommands2.runWorker(hiveConf);
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+
+ SessionState.get().setCurrentDatabase("mydb1");
+
+ //testing show compaction command
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<String> r = runStatementOnDriver("SHOW COMPACTIONS");
+ Assert.assertEquals(rsp.getCompacts().size()+1, r.size());//includes
Header row
+
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 STATUS 'ready for
cleaning'");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getState().equals("ready
for cleaning")).count() +1,
+ r.size());//includes Header row
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 TYPE 'MAJOR' ");
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+ filter(x->x.getType().equals(CompactionType.MAJOR)).count()+1,
r.size());//includes Header row
+
+ r = runStatementOnDriver("SHOW COMPACTIONS SCHEMA mydb1 POOL 'poolx' TYPE
'MINOR' ");
+ //includes Header row
+
Assert.assertEquals(rsp.getCompacts().stream().filter(x->x.getDbname().equals("mydb1")).
+
filter(x->x.getPoolName().equals("poolx")).filter(x->x.getType().equals(CompactionType.MINOR)).count(),
+
r.stream().filter(x->x.equals("mydb1")).filter(x->x.equals("poolx")).filter(x->x.equals(CompactionType.MINOR)).count());
Review Comment:
Please use this technique for the remaining cases as well
Issue Time Tracking
-------------------
Worklog Id: (was: 815587)
Time Spent: 3h 20m (was: 3h 10m)
> SHOW COMPACTIONS should support filtering options
> -------------------------------------------------
>
> Key: HIVE-13353
> URL: https://issues.apache.org/jira/browse/HIVE-13353
> Project: Hive
> Issue Type: Improvement
> Components: Transactions
> Affects Versions: 1.3.0, 2.0.0
> Reporter: Eugene Koifman
> Assignee: KIRTI RUGE
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-13353.01.patch
>
> Time Spent: 3h 20m
> Remaining Estimate: 0h
>
> Since we now have historical information in SHOW COMPACTIONS the output can
> easily become unwieldy. (e.g. 1000 partitions with 3 lines of history each)
> this is a significant usability issue
> Need to add ability to filter by db/table/partition
> Perhaps would also be useful to filter by status
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
