[
https://issues.apache.org/jira/browse/HIVE-24509?focusedWorklogId=530287&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-530287
]
ASF GitHub Bot logged work on HIVE-24509:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 02/Jan/21 21:57
Start Date: 02/Jan/21 21:57
Worklog Time Spent: 10m
Work Description: miklosgergely commented on a change in pull request
#1756:
URL: https://github.com/apache/hive/pull/1756#discussion_r550925420
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
##########
@@ -0,0 +1,575 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.text.StringEscapeUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.ql.ddl.ShowUtils;
+import org.apache.hadoop.hive.ql.ddl.ShowUtils.TextMetaDataTable;
+import org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableDesc;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.CheckConstraint;
+import org.apache.hadoop.hive.ql.metadata.CheckConstraint.CheckConstraintCol;
+import org.apache.hadoop.hive.ql.metadata.DefaultConstraint;
+import
org.apache.hadoop.hive.ql.metadata.DefaultConstraint.DefaultConstraintCol;
+import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.NotNullConstraint;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.metadata.UniqueConstraint;
+import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo.ForeignKeyCol;
+import org.apache.hadoop.hive.ql.metadata.UniqueConstraint.UniqueConstraintCol;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.common.util.HiveStringUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.hive.ql.ddl.ShowUtils.ALIGNMENT;
+import static
org.apache.hadoop.hive.ql.ddl.ShowUtils.DEFAULT_STRINGBUILDER_SIZE;
+import static org.apache.hadoop.hive.ql.ddl.ShowUtils.FIELD_DELIM;
+import static org.apache.hadoop.hive.ql.ddl.ShowUtils.LINE_DELIM;
+import static org.apache.hadoop.hive.ql.ddl.ShowUtils.formatOutput;
+
+/**
+ * Formats DESC TABLE results to text format.
+ */
+class TextDescTableFormatter extends DescTableFormatter {
+ @Override
+ public void describeTable(HiveConf conf, DataOutputStream out, String
columnPath, String tableName, Table table,
+ Partition partition, List<FieldSchema> columns, boolean isFormatted,
boolean isExtended, boolean isOutputPadded,
+ List<ColumnStatisticsObj> columnStats) throws HiveException {
+ try {
+ addStatsData(out, columnPath, columns, isFormatted, columnStats,
isOutputPadded);
+ addPartitionData(out, conf, columnPath, table, isFormatted,
isOutputPadded);
+
+ if (columnPath == null) {
+ if (isFormatted) {
+ addFormattedTableData(out, table, partition, isOutputPadded);
+ }
+
+ if (isExtended) {
+ out.write(Utilities.newLineCode);
+ addExtendedTableData(out, table, partition);
+ addExtendedConstraintData(out, table);
+ addExtendedStorageData(out, table);
+ }
+ }
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ }
+
+ private void addStatsData(DataOutputStream out, String columnPath,
List<FieldSchema> columns, boolean isFormatted,
+ List<ColumnStatisticsObj> columnStats, boolean isOutputPadded) throws
IOException {
+ String statsData = "";
+
+ TextMetaDataTable metaDataTable = new TextMetaDataTable();
+ boolean needColStats = isFormatted && columnPath != null;
+ if (needColStats) {
+ metaDataTable.addRow(DescTableDesc.COLUMN_STATISTICS_HEADERS.toArray(new
String[]{}));
+ } else if (isFormatted && !SessionState.get().isHiveServerQuery()) {
+ statsData += "# ";
+ metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(","));
+ }
+ for (FieldSchema column : columns) {
+ metaDataTable.addRow(ShowUtils.extractColumnValues(column, needColStats,
+ getColumnStatisticsObject(column.getName(), column.getType(),
columnStats)));
+ }
+ if (needColStats) {
+ metaDataTable.transpose();
+ }
+ statsData += metaDataTable.renderTable(isOutputPadded);
+ out.write(statsData.getBytes("UTF-8"));
+ }
+
+ private ColumnStatisticsObj getColumnStatisticsObject(String columnName,
String columnType,
+ List<ColumnStatisticsObj> columnStats) {
+ if (CollectionUtils.isNotEmpty(columnStats)) {
+ for (ColumnStatisticsObj columnStat : columnStats) {
+ if (columnStat.getColName().equalsIgnoreCase(columnName) &&
+ columnStat.getColType().equalsIgnoreCase(columnType)) {
+ return columnStat;
+ }
+ }
+ }
+ return null;
+ }
+
+ private void addPartitionData(DataOutputStream out, HiveConf conf, String
columnPath, Table table,
+ boolean isFormatted, boolean isOutputPadded) throws IOException {
+ String partitionData = "";
+ if (columnPath == null) {
+ List<FieldSchema> partitionColumns = table.isPartitioned() ?
table.getPartCols() : null;
+ if (CollectionUtils.isNotEmpty(partitionColumns) &&
+ conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY))
{
+ TextMetaDataTable metaDataTable = new TextMetaDataTable();
+ partitionData += LINE_DELIM + "# Partition Information" + LINE_DELIM +
"# ";
+ metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(","));
+ for (FieldSchema partitionColumn : partitionColumns) {
+ metaDataTable.addRow(ShowUtils.extractColumnValues(partitionColumn,
false, null));
+ }
+ partitionData += metaDataTable.renderTable(isOutputPadded);
+ }
+ } else {
+ String statsState =
table.getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ if (table.getParameters() != null && statsState != null) {
+ StringBuilder stringBuilder = new StringBuilder();
+ formatOutput(StatsSetupConst.COLUMN_STATS_ACCURATE,
+ isFormatted ? StringEscapeUtils.escapeJava(statsState) :
HiveStringUtils.escapeJava(statsState),
+ stringBuilder, isOutputPadded);
+ partitionData += stringBuilder.toString();
+ }
+ }
+ out.write(partitionData.getBytes("UTF-8"));
+ }
+
+ private void addFormattedTableData(DataOutputStream out, Table table,
Partition partition, boolean isOutputPadded)
+ throws IOException, UnsupportedEncodingException {
+ String formattedTableInfo = null;
+ if (partition != null) {
+ formattedTableInfo = getPartitionInformation(partition);
+ } else {
+ formattedTableInfo = getTableInformation(table, isOutputPadded);
+ }
+
+ if (table.getTableConstraintsInfo().isTableConstraintsInfoNotEmpty()) {
+ formattedTableInfo += getConstraintsInformation(table);
+ }
+ out.write(formattedTableInfo.getBytes("UTF-8"));
+ }
+
+ private String getTableInformation(Table table, boolean isOutputPadded) {
+ StringBuilder tableInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE);
+ tableInfo.append(LINE_DELIM).append("# Detailed Table
Information").append(LINE_DELIM);
+ getTableMetaDataInformation(tableInfo, table, isOutputPadded);
+
+ tableInfo.append(LINE_DELIM).append("# Storage
Information").append(LINE_DELIM);
+ getStorageDescriptorInfo(tableInfo, table.getTTable().getSd());
+
+ if (table.isView() || table.isMaterializedView()) {
+ tableInfo.append(LINE_DELIM + "# " + (table.isView() ? "" :
"Materialized ") + "View Information" + LINE_DELIM);
+ getViewInfo(tableInfo, table);
+ }
+
+ return tableInfo.toString();
+ }
+
+ private String getPartitionInformation(Partition partition) {
+ StringBuilder tableInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE);
+ tableInfo.append(LINE_DELIM + "# Detailed Partition Information" +
LINE_DELIM);
+ getPartitionMetaDataInformation(tableInfo, partition);
+
+ if (partition.getTable().getTableType() != TableType.VIRTUAL_VIEW) {
+ tableInfo.append(LINE_DELIM + "# Storage Information" + LINE_DELIM);
+ getStorageDescriptorInfo(tableInfo, partition.getTPartition().getSd());
+ }
+
+ return tableInfo.toString();
+ }
+
+ private void getViewInfo(StringBuilder tableInfo, Table table) {
+ formatOutput("Original Query:", table.getViewOriginalText(), tableInfo);
+ formatOutput("Expanded Query:", table.getViewExpandedText(), tableInfo);
+ if (table.isMaterializedView()) {
+ formatOutput("Rewrite Enabled:", table.isRewriteEnabled() ? "Yes" :
"No", tableInfo);
+ formatOutput("Outdated for Rewriting:", table.isOutdatedForRewriting()
== null ? "Unknown"
+ : table.isOutdatedForRewriting() ? "Yes" : "No", tableInfo);
+ }
+ }
+
+ private void getStorageDescriptorInfo(StringBuilder tableInfo,
StorageDescriptor storageDesc) {
+ formatOutput("SerDe Library:",
storageDesc.getSerdeInfo().getSerializationLib(), tableInfo);
+ formatOutput("InputFormat:", storageDesc.getInputFormat(), tableInfo);
+ formatOutput("OutputFormat:", storageDesc.getOutputFormat(), tableInfo);
+ formatOutput("Compressed:", storageDesc.isCompressed() ? "Yes" : "No",
tableInfo);
+ formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()),
tableInfo);
+ formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(),
tableInfo);
+ formatOutput("Sort Columns:", storageDesc.getSortCols().toString(),
tableInfo);
+
+ if (storageDesc.isStoredAsSubDirectories()) {
+ formatOutput("Stored As SubDirectories:", "Yes", tableInfo);
+ }
+
+ if (storageDesc.getSkewedInfo() != null) {
+ List<String> skewedColNames =
sortList(storageDesc.getSkewedInfo().getSkewedColNames());
+ if ((skewedColNames != null) && (skewedColNames.size() > 0)) {
+ formatOutput("Skewed Columns:", skewedColNames.toString(), tableInfo);
+ }
+
+ List<List<String>> skewedColValues = sortList(
+ storageDesc.getSkewedInfo().getSkewedColValues(), new
VectorComparator<String>());
+ if (CollectionUtils.isNotEmpty(skewedColValues)) {
+ formatOutput("Skewed Values:", skewedColValues.toString(), tableInfo);
+ }
+
+ Map<List<String>, String> skewedColMap = new TreeMap<>(new
VectorComparator<String>());
+
skewedColMap.putAll(storageDesc.getSkewedInfo().getSkewedColValueLocationMaps());
+ if (MapUtils.isNotEmpty(skewedColMap)) {
+ formatOutput("Skewed Value to Path:", skewedColMap.toString(),
tableInfo);
+ Map<List<String>, String> truncatedSkewedColMap =
+ new TreeMap<List<String>, String>(new VectorComparator<String>());
+ // walk through existing map to truncate path so that test won't mask
it then we can verify location is right
+ Set<Entry<List<String>, String>> entries = skewedColMap.entrySet();
+ for (Entry<List<String>, String> entry : entries) {
+ truncatedSkewedColMap.put(entry.getKey(),
PlanUtils.removePrefixFromWarehouseConfig(entry.getValue()));
+ }
+ formatOutput("Skewed Value to Truncated Path:",
truncatedSkewedColMap.toString(), tableInfo);
+ }
+ }
+
+ if (storageDesc.getSerdeInfo().getParametersSize() > 0) {
+ tableInfo.append("Storage Desc Params:" + LINE_DELIM);
+ displayAllParameters(storageDesc.getSerdeInfo().getParameters(),
tableInfo);
+ }
+ }
+
+ private void getTableMetaDataInformation(StringBuilder tableInfo, Table
table, boolean isOutputPadded) {
+ formatOutput("Database:", table.getDbName(), tableInfo);
+ formatOutput("OwnerType:", (table.getOwnerType() != null) ?
table.getOwnerType().name() : "null", tableInfo);
+ formatOutput("Owner:", table.getOwner(), tableInfo);
+ formatOutput("CreateTime:", formatDate(table.getTTable().getCreateTime()),
tableInfo);
+ formatOutput("LastAccessTime:",
formatDate(table.getTTable().getLastAccessTime()), tableInfo);
+ formatOutput("Retention:", Integer.toString(table.getRetention()),
tableInfo);
+
+ if (!table.isView()) {
+ formatOutput("Location:", table.getDataLocation().toString(), tableInfo);
+ }
+ formatOutput("Table Type:", table.getTableType().name(), tableInfo);
+
+ if (table.getParameters().size() > 0) {
+ tableInfo.append("Table Parameters:" + LINE_DELIM);
+ displayAllParameters(table.getParameters(), tableInfo, false,
isOutputPadded);
+ }
+ }
+
+ private void getPartitionMetaDataInformation(StringBuilder tableInfo,
Partition partition) {
+ formatOutput("Partition Value:", partition.getValues().toString(),
tableInfo);
+ formatOutput("Database:", partition.getTPartition().getDbName(),
tableInfo);
+ formatOutput("Table:", partition.getTable().getTableName(), tableInfo);
+ formatOutput("CreateTime:",
formatDate(partition.getTPartition().getCreateTime()), tableInfo);
+ formatOutput("LastAccessTime:",
formatDate(partition.getTPartition().getLastAccessTime()), tableInfo);
+ formatOutput("Location:", partition.getLocation(), tableInfo);
+
+ if (partition.getTPartition().getParameters().size() > 0) {
+ tableInfo.append("Partition Parameters:" + LINE_DELIM);
+ displayAllParameters(partition.getTPartition().getParameters(),
tableInfo);
+ }
+ }
+
+ private class VectorComparator<T extends Comparable<T>> implements
Comparator<List<T>>{
+ @Override
+ public int compare(List<T> listA, List<T> listB) {
+ for (int i = 0; i < listA.size() && i < listB.size(); i++) {
+ T valA = listA.get(i);
+ T valB = listB.get(i);
+ if (valA != null) {
+ int ret = valA.compareTo(valB);
+ if (ret != 0) {
+ return ret;
+ }
+ } else {
+ if (valB != null) {
+ return -1;
+ }
+ }
+ }
+ return Integer.compare(listA.size(), listB.size());
+ }
+ }
+
+ private <T extends Comparable<T>> List<T> sortList(List<T> list){
+ if (list == null || list.size() <= 1) {
+ return list;
+ }
+ List<T> ret = new ArrayList<>(list);
+ Collections.sort(ret);
+ return ret;
+ }
+
+ private <T> List<T> sortList(List<T> list, Comparator<T> comparator) {
+ if (list == null || list.size() <= 1) {
+ return list;
+ }
+ List<T> ret = new ArrayList<>(list);
+ Collections.sort(ret, comparator);
+ return ret;
+ }
+
+ private String formatDate(long timeInSeconds) {
+ if (timeInSeconds != 0) {
+ Date date = new Date(timeInSeconds * 1000);
+ return date.toString();
+ }
+ return "UNKNOWN";
+ }
+
+ private void displayAllParameters(Map<String, String> params, StringBuilder
tableInfo) {
+ displayAllParameters(params, tableInfo, true, false);
+ }
+
+ private void displayAllParameters(Map<String, String> params, StringBuilder
tableInfo, boolean escapeUnicode,
+ boolean isOutputPadded) {
+ List<String> keys = new ArrayList<String>(params.keySet());
+ Collections.sort(keys);
+ for (String key : keys) {
+ String value = params.get(key);
+ if (key.equals(StatsSetupConst.NUM_ERASURE_CODED_FILES)) {
+ if ("0".equals(value)) {
+ continue;
+ }
+ }
+ tableInfo.append(FIELD_DELIM); // Ensures all params are indented.
+ formatOutput(key, escapeUnicode ? StringEscapeUtils.escapeJava(value) :
HiveStringUtils.escapeJava(value),
+ tableInfo, isOutputPadded);
+ }
+ }
+
+ private String getConstraintsInformation(Table table) {
+ StringBuilder constraintsInfo = new
StringBuilder(DEFAULT_STRINGBUILDER_SIZE);
+
+ constraintsInfo.append(LINE_DELIM + "# Constraints" + LINE_DELIM);
+ if (PrimaryKeyInfo.isPrimaryKeyInfoNotEmpty(table.getPrimaryKeyInfo())) {
+ constraintsInfo.append(LINE_DELIM + "# Primary Key" + LINE_DELIM);
+ getPrimaryKeyInformation(constraintsInfo, table.getPrimaryKeyInfo());
+ }
+ if (ForeignKeyInfo.isForeignKeyInfoNotEmpty(table.getForeignKeyInfo())) {
+ constraintsInfo.append(LINE_DELIM + "# Foreign Keys" + LINE_DELIM);
+ getForeignKeysInformation(constraintsInfo, table.getForeignKeyInfo());
+ }
+ if (UniqueConstraint.isUniqueConstraintNotEmpty(table.getUniqueKeyInfo()))
{
+ constraintsInfo.append(LINE_DELIM + "# Unique Constraints" + LINE_DELIM);
+ getUniqueConstraintsInformation(constraintsInfo,
table.getUniqueKeyInfo());
+ }
+ if
(NotNullConstraint.isNotNullConstraintNotEmpty(table.getNotNullConstraint())) {
+ constraintsInfo.append(LINE_DELIM + "# Not Null Constraints" +
LINE_DELIM);
+ getNotNullConstraintsInformation(constraintsInfo,
table.getNotNullConstraint());
+ }
+ if
(DefaultConstraint.isCheckConstraintNotEmpty(table.getDefaultConstraint())) {
+ constraintsInfo.append(LINE_DELIM + "# Default Constraints" +
LINE_DELIM);
+ getDefaultConstraintsInformation(constraintsInfo,
table.getDefaultConstraint());
+ }
+ if (CheckConstraint.isCheckConstraintNotEmpty(table.getCheckConstraint()))
{
+ constraintsInfo.append(LINE_DELIM + "# Check Constraints" + LINE_DELIM);
+ getCheckConstraintsInformation(constraintsInfo,
table.getCheckConstraint());
+ }
+ return constraintsInfo.toString();
+ }
+
+ private void getPrimaryKeyInformation(StringBuilder constraintsInfo,
PrimaryKeyInfo constraint) {
+ formatOutput("Table:", constraint.getDatabaseName() + "." +
constraint.getTableName(), constraintsInfo);
+ formatOutput("Constraint Name:", constraint.getConstraintName(),
constraintsInfo);
+ Map<Integer, String> columnNames = constraint.getColNames();
+ String title = "Column Name:".intern();
Review comment:
Removed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 530287)
Time Spent: 5h 20m (was: 5h 10m)
> Move show specific codes under DDL and cut MetaDataFormatter classes to pieces
> ------------------------------------------------------------------------------
>
> Key: HIVE-24509
> URL: https://issues.apache.org/jira/browse/HIVE-24509
> Project: Hive
> Issue Type: Sub-task
> Components: Hive
> Reporter: Miklos Gergely
> Assignee: Miklos Gergely
> Priority: Major
> Labels: pull-request-available
> Time Spent: 5h 20m
> Remaining Estimate: 0h
>
> Lot of show ... specific codes are under the
> org.apache.hadoop.hive.ql.metadata.formatting package which are used only by
> these commands. Also the two MetaDataFormatters (JsonMetaDataFormatter,
> TextMetaDataFormatter) are trying to do everything, while they contain a lot
> of code duplications. Their functionalities should be put under the
> directories of the appropriate show commands.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
