This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new c38afdf50 IMPALA-11390: Describe formatted statement on materialized
view should show the view definition
c38afdf50 is described below
commit c38afdf50f652dcc454eb8f20da45224e3c9bfca
Author: pengdou1990 <[email protected]>
AuthorDate: Sat Oct 8 12:08:21 2022 +0800
IMPALA-11390: Describe formatted statement on materialized view should show
the view definition
DESCRIBE FORMATTED/EXTENDED statement on materialized view show the
view definition like hive.
Change-Id: Ie62be1ca6d74370bad4b32bb046df93027e6f651
Reviewed-on: http://gerrit.cloudera.org:8080/19109
Tested-by: Impala Public Jenkins <[email protected]>
Reviewed-by: Quanlong Huang <[email protected]>
---
.../org/apache/impala/compat/MetastoreShim.java | 12 +++++++
.../org/apache/impala/compat/MetastoreShim.java | 39 ++++++++++++++++++++++
.../impala/util/HiveMetadataFormatUtils.java | 8 ++++-
.../QueryTest/describe-materialized-view.test | 25 ++++++++++++++
tests/metadata/test_ddl.py | 4 +++
5 files changed, 87 insertions(+), 1 deletion(-)
diff --git
a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index 1c06b502e..6d5eb9370 100644
---
a/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++
b/fe/src/compat-apache-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -88,6 +88,8 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.impala.util.HiveMetadataFormatUtils.formatOutput;
+
/**
* A wrapper around some of Hive's Metastore API's to abstract away differences
* between major versions of different Hive publishers. This implements the
shimmed
@@ -902,4 +904,14 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
}
return reads;
}
+
+ public static void getMaterializedViewInfo(StringBuilder tableInfo,
+ Table tbl, boolean isOutputPadded) {
+ formatOutput("View Original Text:", tbl.getViewOriginalText(), tableInfo);
+ formatOutput("View Expanded Text:", tbl.getViewExpandedText(), tableInfo);
+ formatOutput("Rewrite Enabled:", tbl.isRewriteEnabled() ? "Yes" : "No",
tableInfo);
+ // TODO: IMPALA-11815: hive metastore doesn't privide api to judge whether
the
+ // materialized view is outdated for rewriting, so set the flag to
"Unknown"
+ formatOutput("Outdated for Rewriting:", "Unknown", tableInfo);
+ }
}
diff --git
a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index 8cc30a4c0..10985ee4a 100644
--- a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -21,6 +21,8 @@ import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCES
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READWRITE;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_WRITEONLY;
+import static org.apache.impala.util.HiveMetadataFormatUtils.LINE_DELIM;
+import static org.apache.impala.util.HiveMetadataFormatUtils.formatOutput;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -31,6 +33,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -65,6 +68,7 @@ import
org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionsRequest;
import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
+import org.apache.hadoop.hive.metastore.api.SourceTable;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
import org.apache.hadoop.hive.metastore.api.WriteNotificationLogRequest;
@@ -79,6 +83,7 @@ import
org.apache.hadoop.hive.metastore.messaging.MessageFactory;
import org.apache.hadoop.hive.metastore.messaging.MessageSerializer;
import org.apache.hadoop.hive.metastore.messaging.json.JSONDropDatabaseMessage;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.metadata.formatting.TextMetaDataTable;
import org.apache.impala.analysis.TableName;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogServiceCatalog;
@@ -821,4 +826,38 @@ public class MetastoreShim extends Hive3MetastoreShimBase {
return false;
}
}
+
+ public static void getMaterializedViewInfo(StringBuilder tableInfo, Table
tbl,
+ boolean isOutputPadded) {
+ formatOutput("View Original Text:", tbl.getViewOriginalText(), tableInfo);
+ formatOutput("View Expanded Text:", tbl.getViewExpandedText(), tableInfo);
+ formatOutput("Rewrite Enabled:", tbl.isRewriteEnabled() ? "Yes" : "No",
tableInfo);
+ // TODO: IMPALA-11815: hive metastore doesn't privide api to judge whether
the
+ // materialized view is outdated for rewriting, so set the flag to
"Unknown"
+ formatOutput("Outdated for Rewriting:", "Unknown", tableInfo);
+
+ tableInfo.append(LINE_DELIM)
+ .append("# Materialized View Source table information")
+ .append(LINE_DELIM);
+ TextMetaDataTable metaDataTable = new TextMetaDataTable();
+ metaDataTable.addRow("Table name", "I/U/D since last rebuild");
+ List<SourceTable> sourceTableList =
+ new ArrayList<>(tbl.getCreationMetadata().getSourceTables());
+ sourceTableList.sort(
+ Comparator
+ .<SourceTable, String>comparing(
+ sourceTable -> sourceTable.getTable().getDbName())
+ .thenComparing(sourceTable ->
sourceTable.getTable().getTableName()));
+ for (SourceTable sourceTable : sourceTableList) {
+ String qualifiedTableName =
org.apache.hadoop.hive.common.TableName.getQualified(
+ sourceTable.getTable().getCatName(),
+ sourceTable.getTable().getDbName(),
+ sourceTable.getTable().getTableName());
+ metaDataTable.addRow(qualifiedTableName,
+ String.format("%d/%d/%d",
+ sourceTable.getInsertedCount(), sourceTable.getUpdatedCount(),
+ sourceTable.getDeletedCount()));
+ tableInfo.append(metaDataTable.renderTable(isOutputPadded));
+ }
+ }
}
diff --git
a/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
b/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
index 0b1e6282b..61f33a2b2 100644
--- a/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
+++ b/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.common.FileSystemUtil;
+import org.apache.impala.compat.MetastoreShim;
/**
* Most of the code in this class is copied from Hive 2.1.1. This is used so
that
@@ -522,6 +523,11 @@ public class HiveMetadataFormatUtils {
if
(TableType.VIRTUAL_VIEW.equals(TableType.valueOf(table.getTableType()))) {
tableInfo.append(LINE_DELIM).append("# View
Information").append(LINE_DELIM);
getViewInfo(tableInfo, table);
+ } else if (TableType.MATERIALIZED_VIEW.equals(
+ TableType.valueOf(table.getTableType()))) {
+ tableInfo.append(LINE_DELIM).append("# Materialized View Information")
+ .append(LINE_DELIM);
+ MetastoreShim.getMaterializedViewInfo(tableInfo, table, isOutputPadded);
}
return tableInfo.toString();
@@ -619,7 +625,7 @@ public class HiveMetadataFormatUtils {
* @param value The value to print - might contain newlines
* @param tableInfo The target builder
*/
- private static void formatOutput(String name, String value, StringBuilder
tableInfo) {
+ public static void formatOutput(String name, String value, StringBuilder
tableInfo) {
tableInfo.append(String.format("%-" + ALIGNMENT + "s",
name)).append(FIELD_DELIM);
int colNameLength = ALIGNMENT > name.length() ? ALIGNMENT : name.length();
indentMultilineValue(value, tableInfo, new int[]{0, colNameLength}, true);
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/describe-materialized-view.test
b/testdata/workloads/functional-query/queries/QueryTest/describe-materialized-view.test
new file mode 100644
index 000000000..03c0517f1
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/describe-materialized-view.test
@@ -0,0 +1,25 @@
+====
+---- QUERY
+# Test DESCRIBE FORMATTED Materialized View
+DESCRIBE FORMATTED functional_parquet.materialized_view
+---- RESULTS: VERIFY_IS_SUBSET
+'# Materialized View Information','NULL','NULL'
+'View Original Text: ','SELECT * FROM
functional_parquet.insert_only_transactional_table','NULL'
+'View Expanded Text: ','SELECT `insert_only_transactional_table`.`col1` FROM
`functional_parquet`.`insert_only_transactional_table`','NULL'
+'Rewrite Enabled: ','Yes ','NULL'
+'Outdated for Rewriting:','Unknown ','NULL'
+---- TYPES
+string,string,string
+====
+---- QUERY
+# Test DESCRIBE EXTENDED Materialized View
+DESCRIBE EXTENDED functional_parquet.materialized_view
+---- RESULTS: VERIFY_IS_SUBSET
+'# Materialized View Information','NULL','NULL'
+'View Original Text: ','SELECT * FROM
functional_parquet.insert_only_transactional_table','NULL'
+'View Expanded Text: ','SELECT `insert_only_transactional_table`.`col1` FROM
`functional_parquet`.`insert_only_transactional_table`','NULL'
+'Rewrite Enabled: ','Yes ','NULL'
+'Outdated for Rewriting:','Unknown ','NULL'
+---- TYPES
+string,string,string
+====
\ No newline at end of file
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 303a258c8..fdcf6263c 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -907,6 +907,10 @@ class TestDdlStatements(TestDdlBase):
comment = self._get_column_comment(table, 'j')
assert "comment4" == comment
+ def test_describe_materialized_view(self, vector, unique_database):
+ vector.get_value('exec_option')['abort_on_error'] = False
+ self.run_test_case('QueryTest/describe-materialized-view', vector,
+ use_db=unique_database,
multiple_impalad=self._use_multiple_impalad(vector))
# IMPALA-10811: RPC to submit query getting stuck for AWS NLB forever
# Test HS2, Beeswax and HS2-HTTP three clients.
