This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 37f44a58f IMPALA-10918: Allow map type in SELECT list
37f44a58f is described below
commit 37f44a58f3f9d515e9c74455037c654542ac6566
Author: Daniel Becker <[email protected]>
AuthorDate: Tue Jul 12 17:31:48 2022 +0200
IMPALA-10918: Allow map type in SELECT list
Adding support for MAP types in the select list.
An example of how maps are printed:
{"k1":2,"k2":null}
Nested collection types (maps and arrays) are supported in any
combination. However, structs in collections and collections in structs
are not supported.
Limitations (other than map support) as described in the commit for
IMPALA-9498 still apply, the following are to be implemented later:
- Unify HS2 / Beeswax logic with the way STRUCTs are handled.
This could be done in a "final" logic that can handle
STRUCTS/ARRAYS nested to each other
- Implement "deep copy" and "deep serialize" for collections in BE.
This would enable all operators, e.g. ORDER BY and UNION.
Testing:
- modified the FE tests that checked that maps were not allowed in the
select list - now the test expect maps are allowed there
- added FE and EE tests involving maps based on the array tests
Change-Id: I921c647f1779add36e7f5df4ce6ca237dcfaf001
Reviewed-on: http://gerrit.cloudera.org:8080/18736
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
be/src/codegen/codegen-anyval.cc | 3 +-
be/src/runtime/raw-value.cc | 102 ++++---
be/src/runtime/raw-value.h | 9 +-
be/src/runtime/types.h | 1 +
be/src/service/hs2-util.cc | 16 +-
be/src/service/impala-beeswax-server.cc | 5 +-
be/src/service/impala-server.h | 4 +-
be/src/service/query-result-set.cc | 19 +-
be/src/service/query-result-set.h | 6 +-
.../java/org/apache/impala/analysis/Analyzer.java | 46 +--
.../org/apache/impala/analysis/InlineViewRef.java | 38 +--
.../org/apache/impala/analysis/SelectStmt.java | 11 +-
.../apache/impala/analysis/SetOperationStmt.java | 2 +-
.../main/java/org/apache/impala/catalog/Type.java | 5 +-
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 11 +-
.../impala/analysis/AnalyzeUpsertStmtTest.java | 5 -
.../org/apache/impala/analysis/AnalyzerTest.java | 7 +-
.../authorization/AuthorizationStmtTest.java | 34 +++
.../functional/functional_schema_template.sql | 54 +++-
.../datasets/functional/schema_constraints.csv | 7 +-
.../QueryTest/nested-array-in-select-list.test | 2 +-
.../QueryTest/nested-map-in-select-list.test | 326 +++++++++++++++++++++
.../ranger_column_masking_complex_types.test | 13 +
...anger_column_masking_struct_in_select_list.test | 32 +-
.../queries/QueryTest/struct-in-select-list.test | 6 +
tests/authorization/test_ranger.py | 4 +
tests/query_test/test_nested_types.py | 8 +-
27 files changed, 632 insertions(+), 144 deletions(-)
diff --git a/be/src/codegen/codegen-anyval.cc b/be/src/codegen/codegen-anyval.cc
index 92ec2d06b..b9fbc03e8 100644
--- a/be/src/codegen/codegen-anyval.cc
+++ b/be/src/codegen/codegen-anyval.cc
@@ -666,7 +666,8 @@ void CodegenAnyVal::StoreToNativePtr(llvm::Value*
raw_val_ptr, llvm::Value* pool
switch (type_.type) {
case TYPE_STRING:
case TYPE_VARCHAR:
- case TYPE_ARRAY: { // CollectionVal has same memory layout as StringVal.
+ case TYPE_ARRAY: // CollectionVal has same memory layout as StringVal.
+ case TYPE_MAP: { // CollectionVal has same memory layout as StringVal.
// Convert StringVal to StringValue
llvm::Value* string_value = llvm::Constant::getNullValue(raw_type);
llvm::Value* len = GetLen();
diff --git a/be/src/runtime/raw-value.cc b/be/src/runtime/raw-value.cc
index 6acc789a6..eba2b9d40 100644
--- a/be/src/runtime/raw-value.cc
+++ b/be/src/runtime/raw-value.cc
@@ -411,10 +411,52 @@ template void RawValue::WritePrimitive<false>(const void*
value, Tuple* tuple,
const SlotDescriptor* slot_desc, MemPool* pool,
std::vector<StringValue*>* string_values);
-void RawValue::PrintArrayValue(const CollectionValue* array_val,
- const TupleDescriptor* item_tuple_desc, int scale, stringstream *stream) {
+bool PrintNestedValueIfNull(const SlotDescriptor& slot_desc, Tuple* item,
+ stringstream* stream) {
+ bool is_null = item->IsNull(slot_desc.null_indicator_offset());
+ if (is_null) *stream << "NULL";
+ return is_null;
+}
+
+void PrintNonNullNestedCollection(const SlotDescriptor& slot_desc, Tuple*
item, int scale,
+ stringstream* stream) {
+ const CollectionValue* nested_collection_val =
+ item->GetCollectionSlot(slot_desc.tuple_offset());
+ DCHECK(nested_collection_val != nullptr);
+ const TupleDescriptor* child_item_tuple_desc =
+ slot_desc.children_tuple_descriptor();
+ DCHECK(child_item_tuple_desc != nullptr);
+ RawValue::PrintCollectionValue(nested_collection_val, child_item_tuple_desc,
scale,
+ stream, slot_desc.type().IsMapType());
+}
+
+void PrintNonNullNestedPrimitive(const SlotDescriptor& slot_desc, Tuple* item,
int scale,
+ stringstream* stream) {
+ RawValue::PrintValue(item->GetSlot(slot_desc.tuple_offset()),
slot_desc.type(), scale,
+ stream, true);
+}
+
+void PrintNestedValue(const SlotDescriptor& slot_desc, Tuple* item, int scale,
+ stringstream* stream) {
+ bool is_null = PrintNestedValueIfNull(slot_desc, item, stream);
+ if (is_null) return;
+
+ if (slot_desc.type().IsCollectionType()) {
+ // The item is also an array or a map, recurse deeper if not NULL.
+ PrintNonNullNestedCollection(slot_desc, item, scale, stream);
+ } else if (!slot_desc.type().IsComplexType()) {
+ // The item is a scalar, print it with the usual PrintValue.
+ PrintNonNullNestedPrimitive(slot_desc, item, scale, stream);
+ } else {
+ DCHECK(false);
+ }
+}
+
+void RawValue::PrintCollectionValue(const CollectionValue* coll_val,
+ const TupleDescriptor* item_tuple_desc, int scale, stringstream *stream,
+ bool is_map) {
DCHECK(item_tuple_desc != nullptr);
- if (array_val == nullptr) {
+ if (coll_val == nullptr) {
*stream << "NULL";
return;
}
@@ -422,42 +464,28 @@ void RawValue::PrintArrayValue(const CollectionValue*
array_val,
const vector<SlotDescriptor*>& slot_descs = item_tuple_desc->slots();
// TODO: This has to be changed once structs are supported too.
- DCHECK(slot_descs.size() == 1);
- DCHECK(slot_descs[0] != nullptr);
-
- *stream << "[";
- if (slot_descs[0]->type().IsArrayType()) {
- // The item is also an array, recurse deeper if not NULL.
- for (int i = 0; i < array_val->num_tuples; ++i) {
- Tuple* item = reinterpret_cast<Tuple*>(array_val->ptr + i *
item_byte_size);
- if (item->IsNull(slot_descs[0]->null_indicator_offset())) {
- *stream << "NULL";
- } else {
- const CollectionValue* nested_array_val =
- item->GetCollectionSlot(slot_descs[0]->tuple_offset());
- const TupleDescriptor* child_item_tuple_desc =
- slot_descs[0]->children_tuple_descriptor();
- DCHECK(child_item_tuple_desc != nullptr);
- PrintArrayValue(nested_array_val, child_item_tuple_desc, scale,
stream);
- }
- if (i < array_val->num_tuples - 1) *stream << ",";
- }
- } else if (!slot_descs[0]->type().IsComplexType()) {
- // The item is a scalar, print it with the usual PrintValue.
- for (int i = 0; i < array_val->num_tuples; ++i) {
- Tuple* item = reinterpret_cast<Tuple*>(array_val->ptr + i *
item_byte_size);
- if (item->IsNull(slot_descs[0]->null_indicator_offset())) {
- *stream << "NULL";
- } else {
- PrintValue(item->GetSlot(slot_descs[0]->tuple_offset()),
slot_descs[0]->type(),
- scale, stream, true);
- }
- if (i < array_val->num_tuples - 1) *stream << ",";
- }
+ if (is_map) {
+ DCHECK(slot_descs.size() == 2);
+ DCHECK(slot_descs[0] != nullptr);
+ DCHECK(slot_descs[1] != nullptr);
} else {
- DCHECK(false);
+ DCHECK(slot_descs.size() == 1);
+ DCHECK(slot_descs[0] != nullptr);
+ }
+
+ *stream << (is_map ? "{" : "[");
+ for (int i = 0; i < coll_val->num_tuples; ++i) {
+ Tuple* item = reinterpret_cast<Tuple*>(coll_val->ptr + i * item_byte_size);
+
+ PrintNestedValue(*slot_descs[0], item, scale, stream);
+ if (is_map) {
+ *stream << ":";
+ PrintNestedValue(*slot_descs[1], item, scale, stream);
+ }
+
+ if (i < coll_val->num_tuples - 1) *stream << ",";
}
- *stream << "]";
+ *stream << (is_map ? "}" : "]");
}
}
diff --git a/be/src/runtime/raw-value.h b/be/src/runtime/raw-value.h
index 78dfb15ad..8b99e7c25 100644
--- a/be/src/runtime/raw-value.h
+++ b/be/src/runtime/raw-value.h
@@ -67,10 +67,11 @@ class RawValue {
return str;
}
- /// Similar to PrintValue() but works with array values.
- /// Converts 'array_val' array into ascii and writes to 'stream'.
- static void PrintArrayValue(const CollectionValue* array_val,
- const TupleDescriptor* item_tuple_desc, int scale, std::stringstream
*stream);
+ /// Similar to PrintValue() but works with collection values.
+ /// Converts 'coll_val' collection into ascii and writes to 'stream'.
+ static void PrintCollectionValue(const CollectionValue* coll_val,
+ const TupleDescriptor* item_tuple_desc, int scale, std::stringstream
*stream,
+ bool is_map);
/// Writes the byte representation of a value to a stringstream
character-by-character
static void PrintValueAsBytes(const void* value, const ColumnType& type,
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index a8c68663e..3a4e95e86 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -247,6 +247,7 @@ struct ColumnType {
}
inline bool IsArrayType() const { return type == TYPE_ARRAY; }
+ inline bool IsMapType() const { return type == TYPE_MAP; }
inline bool IsVarLenType() const {
return IsVarLenStringType() || IsCollectionType();
diff --git a/be/src/service/hs2-util.cc b/be/src/service/hs2-util.cc
index ce9178e9a..ebb059662 100644
--- a/be/src/service/hs2-util.cc
+++ b/be/src/service/hs2-util.cc
@@ -421,9 +421,10 @@ static void
StructExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
}
}
-static void ArrayExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
+static void CollectionExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
const TColumnType& type, RowBatch* batch, int start_idx, int num_rows,
- uint32_t output_row_idx, apache::hive::service::cli::thrift::TColumn*
column) {
+ uint32_t output_row_idx, apache::hive::service::cli::thrift::TColumn*
column,
+ bool is_map) {
DCHECK(type.types.size() > 1);
ReserveSpace(num_rows, output_row_idx, &column->stringVal);
FOREACH_ROW_LIMIT(batch, start_idx, num_rows, it) {
@@ -439,7 +440,7 @@ static void
ArrayExprValuesToHS2TColumn(ScalarExprEvaluator* expr_eval,
CollectionValue value(coll_val);
// TODO: use rapidjson as in for structs
stringstream stream;
- RawValue::PrintArrayValue(&value, item_tuple_desc, -1, &stream);
+ RawValue::PrintCollectionValue(&value, item_tuple_desc, -1, &stream,
is_map);
column->stringVal.values.emplace_back(stream.str());
}
SetNullBit(output_row_idx, coll_val.is_null, &column->stringVal.nulls);
@@ -461,8 +462,12 @@ void impala::ExprValuesToHS2TColumn(ScalarExprEvaluator*
expr_eval,
expr_eval, type, batch, start_idx, num_rows, output_row_idx, column);
return;
case TTypeNodeType::ARRAY:
- ArrayExprValuesToHS2TColumn(
- expr_eval, type, batch, start_idx, num_rows, output_row_idx, column);
+ CollectionExprValuesToHS2TColumn(
+ expr_eval, type, batch, start_idx, num_rows, output_row_idx, column,
false);
+ return;
+ case TTypeNodeType::MAP:
+ CollectionExprValuesToHS2TColumn(
+ expr_eval, type, batch, start_idx, num_rows, output_row_idx, column,
true);
return;
default:
break;
@@ -939,6 +944,7 @@ thrift::TTypeEntry impala::ColumnToHs2Type(
}
case TYPE_STRUCT:
case TYPE_ARRAY:
+ case TYPE_MAP:
type_entry.__set_type(thrift::TTypeId::STRING_TYPE);
break;
case TYPE_BINARY:
diff --git a/be/src/service/impala-beeswax-server.cc
b/be/src/service/impala-beeswax-server.cc
index bd4a3b4ae..e73a41e11 100644
--- a/be/src/service/impala-beeswax-server.cc
+++ b/be/src/service/impala-beeswax-server.cc
@@ -206,7 +206,8 @@ string ImpalaServer::ColumnTypeToBeeswaxTypeString(const
TColumnType& type) {
DCHECK_EQ(TTypeNodeType::SCALAR, type.types[0].type);
DCHECK(type.types[0].__isset.scalar_type);
return TypeToOdbcString(type);
- } else if (type.types[0].type == TTypeNodeType::ARRAY) {
+ } else if (type.types[0].type == TTypeNodeType::ARRAY
+ || type.types[0].type == TTypeNodeType::MAP) {
DCHECK_GT(type.types.size(), 1);
// TODO (IMPALA-11041): consider returning the real type
return "string";
@@ -220,7 +221,7 @@ string ImpalaServer::ColumnTypeToBeeswaxTypeString(const
TColumnType& type) {
}
}
-// TODO: Handle complex types.
+// TODO: Handle struct types.
void ImpalaServer::get_results_metadata(ResultsMetadata& results_metadata,
const beeswax::QueryHandle& beeswax_handle) {
ScopedSessionState session_handle(this);
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index bff9166d1..37f41546b 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -886,8 +886,8 @@ class ImpalaServer : public ImpalaServiceIf,
/// Initializes the backend descriptor in 'be_desc' with the local backend
information.
void BuildLocalBackendDescriptorInternal(BackendDescriptorPB* be_desc);
- /// Converts a type to a string. The only complex type supported is array
which
- /// is returned as string (see IMPALA-11041).
+ /// Converts a type to a string. The only complex types supported are array
and map,
+ /// which are returned as strings (see IMPALA-11041).
std::string ColumnTypeToBeeswaxTypeString(const TColumnType& type);
/// Snapshot of a query's state, archived in the query log. Not mutated after
diff --git a/be/src/service/query-result-set.cc
b/be/src/service/query-result-set.cc
index 19f2b73d7..c67f416eb 100644
--- a/be/src/service/query-result-set.cc
+++ b/be/src/service/query-result-set.cc
@@ -197,11 +197,9 @@ Status AsciiQueryResultSet::AddRows(const
vector<ScalarExprEvaluator*>& expr_eva
&out_stream);
} else if (metadata_.columns[i].columnType.types.size() > 1) {
ColumnType col_type =
ColumnType::FromThrift(metadata_.columns[i].columnType);
- // TODO: Implement map (IMPALA-10918) type.
- DCHECK(col_type.IsArrayType());
- if (col_type.IsArrayType()) {
- PrintArrayValue(expr_evals[i], it.Get(), scales[i], &out_stream);
- }
+ DCHECK(col_type.IsArrayType() || col_type.IsMapType());
+ PrintCollectionValue(expr_evals[i], it.Get(), scales[i], &out_stream,
+ col_type.IsMapType());
} else {
DCHECK(false);
}
@@ -212,8 +210,8 @@ Status AsciiQueryResultSet::AddRows(const
vector<ScalarExprEvaluator*>& expr_eva
return Status::OK();
}
-void QueryResultSet::PrintArrayValue(ScalarExprEvaluator* expr_eval,
- const TupleRow* row, int scale, stringstream *stream) {
+void QueryResultSet::PrintCollectionValue(ScalarExprEvaluator* expr_eval,
+ const TupleRow* row, int scale, stringstream *stream, bool is_map) {
const ScalarExpr& scalar_expr = expr_eval->root();
// Currently scalar_expr can be only a slot ref as no functions return
arrays.
DCHECK(scalar_expr.IsSlotRef());
@@ -222,7 +220,7 @@ void QueryResultSet::PrintArrayValue(ScalarExprEvaluator*
expr_eval,
const CollectionValue* array_val =
static_cast<const CollectionValue*>(expr_eval->GetValue(row));
- RawValue::PrintArrayValue(array_val, item_tuple_desc, scale, stream);
+ RawValue::PrintCollectionValue(array_val, item_tuple_desc, scale, stream,
is_map);
}
Status AsciiQueryResultSet::AddOneRow(const TResultRow& row) {
@@ -447,8 +445,9 @@ void HS2ColumnarResultSet::InitColumns() {
DCHECK(type_nodes.size() > 0);
ThriftTColumn col_output;
if (type_nodes[0].type == TTypeNodeType::STRUCT
- || type_nodes[0].type == TTypeNodeType::ARRAY) {
- // Return structs and arrays as string.
+ || type_nodes[0].type == TTypeNodeType::ARRAY
+ || type_nodes[0].type == TTypeNodeType::MAP) {
+ // Return structs, arrays and maps as string.
col_output.__isset.stringVal = true;
} else {
DCHECK(type_nodes.size() == 1);
diff --git a/be/src/service/query-result-set.h
b/be/src/service/query-result-set.h
index 282f7f615..176e18c48 100644
--- a/be/src/service/query-result-set.h
+++ b/be/src/service/query-result-set.h
@@ -78,10 +78,10 @@ class QueryResultSet {
apache::hive::service::cli::thrift::TRowSet* rowset);
protected:
- /// Wrapper to call RawValue::PrintArrayValue for a given collection column.
+ /// Wrapper to call RawValue::PrintCollectionValue for a given collection
column.
/// expr_eval must be a SlotRef on a collection slot.
- static void PrintArrayValue(ScalarExprEvaluator* expr_eval, const TupleRow*
row,
- int scale, std::stringstream *stream);
+ static void PrintCollectionValue(ScalarExprEvaluator* expr_eval, const
TupleRow* row,
+ int scale, std::stringstream *stream, bool is_map);
};
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 25b880f1f..91f27b470 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -1465,8 +1465,8 @@ public class Analyzer {
SlotDescriptor existingSlotDesc = slotPathMap_.get(key);
if (existingSlotDesc != null) return existingSlotDesc;
- if (slotPath.destType().isArrayType()) {
- SlotDescriptor result = registerArraySlotRef(slotPath);
+ if (slotPath.destType().isCollectionType()) {
+ SlotDescriptor result = registerCollectionSlotRef(slotPath);
result.setPath(slotPath);
slotPathMap_.put(slotPath.getFullyQualifiedRawPath(), result);
registerColumnPrivReq(result);
@@ -1652,13 +1652,13 @@ public class Analyzer {
}
/**
- * Registers an array and its descendants.
+ * Registers a collection and its descendants.
* Creates a CollectionTableRef for all collections on the path.
*/
- private SlotDescriptor registerArraySlotRef(Path slotPath)
+ private SlotDescriptor registerCollectionSlotRef(Path slotPath)
throws AnalysisException {
Preconditions.checkState(slotPath.isResolved());
- Preconditions.checkState(slotPath.destType().isArrayType());
+ Preconditions.checkState(slotPath.destType().isCollectionType());
List<String> rawPath = slotPath.getRawPath();
List<String> collectionTableRawPath = new ArrayList<>();
TupleDescriptor rootDesc = slotPath.getRootDesc();
@@ -1678,28 +1678,38 @@ public class Analyzer {
SlotDescriptor desc = ((SlotRef) collTblRef.getCollectionExpr()).getDesc();
desc.setIsMaterializedRecursively(true);
- // Resolve path
- List<String> rawPathToItem = new ArrayList<String>();
- rawPathToItem.add(Path.ARRAY_ITEM_FIELD_NAME);
+ if (slotPath.destType().isArrayType()) {
+ // Resolve path
+ List<String> rawPathToItem = Arrays.asList(Path.ARRAY_ITEM_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToItem);
+ } else {
+ Preconditions.checkState(slotPath.destType().isMapType());
+
+ List<String> rawPathToKey = Arrays.asList(Path.MAP_KEY_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToKey);
+
+ List<String> rawPathToValue = Arrays.asList(Path.MAP_VALUE_FIELD_NAME);
+ resolveAndRegisterDescendantPath(collTblRef, rawPathToValue);
+ }
+
+ return desc;
+ }
- Path resolvedPathToItem = new Path(collTblRef.getDesc(), rawPathToItem);
- boolean isResolved = resolvedPathToItem.resolve();
+ private void resolveAndRegisterDescendantPath(CollectionTableRef collTblRef,
+ List<String> rawPath) throws AnalysisException {
+ Path resolvedPath = new Path(collTblRef.getDesc(), rawPath);
+ boolean isResolved = resolvedPath.resolve();
Preconditions.checkState(isResolved);
- if (resolvedPathToItem.destType().isStructType()) {
+ if (resolvedPath.destType().isStructType()) {
throw new AnalysisException(
"STRUCT type inside collection types is not supported.");
}
- if (resolvedPathToItem.destType().isMapType()) {
- throw new AnalysisException(
- "MAP type inside collection types is not supported.");
- }
- if (resolvedPathToItem.destType().isBinary()) {
+ if (resolvedPath.destType().isBinary()) {
throw new AnalysisException(
"Binary type inside collection types is not supported
(IMPALA-11491).");
}
- registerSlotRef(resolvedPathToItem, false);
- return desc;
+ registerSlotRef(resolvedPath, false);
}
/**
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index fb65f2a26..944348e5c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -304,7 +304,7 @@ public class InlineViewRef extends TableRef {
analyzer.createAuxEqPredicate(new SlotRef(slotDesc), colExpr.clone());
}
- if (colExpr.getType().isArrayType()) {
+ if (colExpr.getType().isCollectionType()) {
// Calling registerSlotRef() above created a new SlotDescriptor +
TupleDescriptor
// hierarchy for Array types. Walk through this hiararchy and add all
slot refs
// to smap_ and baseTblSmap_.
@@ -322,22 +322,26 @@ public class InlineViewRef extends TableRef {
if (itemTupleDesc != null) {
Preconditions.checkState(srcItemTupleDesc != null);
Preconditions.checkState(baseTableItemTupleDesc != null);
- // Assume that there is only a single slot as struct in nested in
- // arrays is not yet supported in select lists.
- Preconditions.checkState(itemTupleDesc.getSlots().size() == 1);
- Preconditions.checkState(srcItemTupleDesc.getSlots().size() == 1);
- Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() ==
1);
- SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(0);
- SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(0);
- SlotDescriptor baseTableItemSlotDesc =
baseTableItemTupleDesc.getSlots().get(0);
- SlotRef itemSlotRef = new SlotRef(itemSlotDesc);
- SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc);
- SlotRef beseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc);
- smap_.put(itemSlotRef, srcItemSlotRef);
- baseTblSmap_.put(itemSlotRef, beseTableItemSlotRef);
- if (createAuxPredicate(colExpr)) {
- analyzer.createAuxEqPredicate(
- new SlotRef(itemSlotDesc), srcItemSlotRef.clone());
+
+ final int num_slots = itemTupleDesc.getSlots().size();
+ Preconditions.checkState(srcItemTupleDesc.getSlots().size() ==
num_slots);
+ Preconditions.checkState(baseTableItemTupleDesc.getSlots().size() ==
num_slots);
+
+ // There is one slot for arrays and two for maps. When we add support
for structs
+ // in collections in the select list, there may be more slots.
+ for (int i = 0; i < num_slots; i++) {
+ SlotDescriptor itemSlotDesc = itemTupleDesc.getSlots().get(i);
+ SlotDescriptor srcItemSlotDesc = srcItemTupleDesc.getSlots().get(i);
+ SlotDescriptor baseTableItemSlotDesc =
baseTableItemTupleDesc.getSlots().get(i);
+ SlotRef itemSlotRef = new SlotRef(itemSlotDesc);
+ SlotRef srcItemSlotRef = new SlotRef(srcItemSlotDesc);
+ SlotRef beseTableItemSlotRef = new SlotRef(baseTableItemSlotDesc);
+ smap_.put(itemSlotRef, srcItemSlotRef);
+ baseTblSmap_.put(itemSlotRef, beseTableItemSlotRef);
+ if (createAuxPredicate(colExpr)) {
+ analyzer.createAuxEqPredicate(
+ new SlotRef(itemSlotDesc), srcItemSlotRef.clone());
+ }
}
}
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index 8c2b4dcfc..8774fbb39 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -39,6 +39,7 @@ import org.apache.impala.catalog.FeIcebergTable.Utils;
import org.apache.impala.catalog.FeKuduTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
+import org.apache.impala.catalog.MapType;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.TableLoadingException;
@@ -488,10 +489,12 @@ public class SelectStmt extends QueryStmt {
expr.getType().toSql() + "' in '" + expr.toSql() + "'.");
}
} else if (expr.getType().isMapType()) {
- throw new AnalysisException(String.format(
- "Expr '%s' in select list returns a map type '%s'.\n" +
- "Map type is not allowed in the select list.",
- expr.toSql(), expr.getType().toSql()));
+ MapType mapType = (MapType) expr.getType();
+ if (!mapType.getKeyType().isSupported()
+ || !mapType.getValueType().isSupported()) {
+ throw new AnalysisException("Unsupported type '" +
+ expr.getType().toSql() + "' in '" + expr.toSql() + "'.");
+ }
} else if (expr.getType().isStructType()) {
if
(!analyzer_.getQueryCtx().client_request.query_options.disable_codegen) {
throw new AnalysisException("Struct type in select list is not
allowed " +
diff --git a/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java
b/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java
index c8d6b943e..ccf81c30b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SetOperationStmt.java
@@ -627,7 +627,7 @@ public class SetOperationStmt extends QueryStmt {
SlotDescriptor slotDesc = analyzer.addSlotDescriptor(tupleDesc);
slotDesc.setLabel(getColLabels().get(i));
slotDesc.setType(expr.getType());
- if (expr.getType().isArrayType()) {
+ if (expr.getType().isCollectionType()) {
slotDesc.setItemTupleDesc(((SlotRef)expr).getDesc().getItemTupleDesc());
}
slotDesc.setStats(columnStats.get(i));
diff --git a/fe/src/main/java/org/apache/impala/catalog/Type.java
b/fe/src/main/java/org/apache/impala/catalog/Type.java
index 1068ab8d6..7e4c2fb6f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Type.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Type.java
@@ -351,7 +351,7 @@ public abstract class Type {
* casting between any two types other than both decimals.
*
*
- * TODO: Support map and struct types.
+ * TODO: Support struct types.
*/
public static Type getAssignmentCompatibleType(
Type t1, Type t2, boolean strict, boolean strictDecimal) {
@@ -361,6 +361,9 @@ public abstract class Type {
} else if (t1.isArrayType() && t2.isArrayType()) {
// Only support exact match for array types.
if (t1.equals(t2)) return t2;
+ } else if (t1.isMapType() && t2.isMapType()) {
+ // Only support exact match for map types.
+ if (t1.equals(t2)) return t2;
}
return ScalarType.INVALID;
}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 87df7da18..30dfb15be 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -2292,8 +2292,7 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"Cannot create table 't': Type CHAR(5) is not supported in Kudu");
AnalysisError("create table t primary key (id) partition by hash
partitions 3" +
" stored as kudu as select id, m from
functional.complextypes_fileformat",
- "Expr 'm' in select list returns a map type 'MAP<STRING,BIGINT>'.\n" +
- "Map type is not allowed in the select list.");
+ "Cannot create table 't': Type MAP<STRING,BIGINT> is not supported in
Kudu");
AnalysisError("create table t primary key (id) partition by hash
partitions 3" +
" stored as kudu as select id, a from
functional.complextypes_fileformat",
"Cannot create table 't': Type ARRAY<INT> is not supported in Kudu");
@@ -3092,13 +3091,9 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalyzesOk("create view functional.foo (a) as " +
"select int_array_col " +
"from functional.allcomplextypes");
- // View cannot have map typed columns because map-typed exprs are
- // not supported in the select list.
- AnalysisError("create view functional.foo (a) as " +
+ AnalyzesOk("create view functional.foo (a) as " +
"select int_map_col " +
- "from functional.allcomplextypes",
- "Expr 'int_map_col' in select list returns a map type
'MAP<STRING,INT>'.\n" +
- "Map type is not allowed in the select list.");
+ "from functional.allcomplextypes");
// It's allowed to do the same with struct as it is supported in the
select list.
AnalysisContext ctx = createAnalysisCtx();
// TODO: Turning Codegen OFF could be removed once the Codegen support is
implemented
diff --git
a/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
b/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
index 1dcabdc15..8ad03e6f7 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
@@ -120,10 +120,5 @@ public class AnalyzeUpsertStmtTest extends AnalyzerTest {
"'b.int_array_col' correlated with an outer block as well as an " +
"uncorrelated one 'functional.alltypestiny':\n" +
"SELECT item FROM b.int_array_col, functional.alltypestiny");
- // Illegal map-typed expr
- AnalysisError("upsert into functional_kudu.testtbl " +
- "select int_map_col from functional.allcomplextypes",
- "Expr 'int_map_col' in select list returns a map type
'MAP<STRING,INT>'.\n" +
- "Map type is not allowed in the select list.");
}
}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
index ad91500eb..0fcae9d30 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
@@ -303,15 +303,12 @@ public class AnalyzerTest extends FrontendTestBase {
"select binary_member_col from
functional_parquet.binary_in_complex_types",
"Struct containing a BINARY type is not allowed in the select list " +
"(IMPALA-11491).");
- // TODO: change error message once IMPALA-10918 is finished.
AnalysisError(
"select binary_key_col from
functional_parquet.binary_in_complex_types",
- "Expr 'binary_key_col' in select list returns a map type
'MAP<BINARY,INT>'." +
- "\nMap type is not allowed in the select list.");
+ "Binary type inside collection types is not supported
(IMPALA-11491).");
AnalysisError(
"select binary_value_col from
functional_parquet.binary_in_complex_types",
- "Expr 'binary_value_col' in select list returns a map type
'MAP<INT,BINARY>'." +
- "\nMap type is not allowed in the select list.");
+ "Binary type inside collection types is not supported
(IMPALA-11491).");
for (ScalarType t: Type.getUnsupportedTypes()) {
// Create/Alter table.
diff --git
a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
index 0bb4e39fd..b1b27ee2b 100644
---
a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
+++
b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
@@ -718,6 +718,40 @@ public class AuthorizationStmtTest extends
AuthorizationTestBase {
.error(selectError("functional.allcomplextypes"),
onColumn("functional",
"allcomplextypes", new String[]{"array_array_col"},
TPrivilegeLevel.SELECT));
+ // Select with map types in select list.
+ authorize("select int_map_col, array_map_col, map_map_col "
+ + "from functional.allcomplextypes")
+ .ok(onServer(TPrivilegeLevel.ALL))
+ .ok(onServer(TPrivilegeLevel.OWNER))
+ .ok(onServer(TPrivilegeLevel.SELECT))
+ .ok(onDatabase("functional", TPrivilegeLevel.ALL))
+ .ok(onDatabase("functional", TPrivilegeLevel.OWNER))
+ .ok(onDatabase("functional", TPrivilegeLevel.SELECT))
+ .ok(onTable("functional", "allcomplextypes", TPrivilegeLevel.ALL))
+ .ok(onTable("functional", "allcomplextypes", TPrivilegeLevel.OWNER))
+ .ok(onTable("functional", "allcomplextypes", TPrivilegeLevel.SELECT))
+ .ok(onColumn("functional", "allcomplextypes",
+ new String[]{"int_map_col", "array_map_col", "map_map_col"},
+ TPrivilegeLevel.SELECT))
+ .error(selectError("functional.allcomplextypes"))
+ .error(selectError("functional.allcomplextypes"), onServer(allExcept(
+ TPrivilegeLevel.ALL, TPrivilegeLevel.OWNER,
TPrivilegeLevel.SELECT)))
+ .error(selectError("functional.allcomplextypes"), onDatabase(
+ "functional", allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.OWNER,
+ TPrivilegeLevel.SELECT)))
+ .error(selectError("functional.allcomplextypes"), onTable("functional",
+ "allcomplextypes", allExcept(TPrivilegeLevel.ALL,
TPrivilegeLevel.OWNER,
+ TPrivilegeLevel.SELECT)))
+ .error(selectError("functional.allcomplextypes"),
onColumn("functional",
+ "allcomplextypes", new String[]{"int_map_col"},
TPrivilegeLevel.SELECT))
+ .error(selectError("functional.allcomplextypes"),
onColumn("functional",
+ "allcomplextypes", new String[]{"array_map_col"},
TPrivilegeLevel.SELECT))
+ .error(selectError("functional.allcomplextypes"),
onColumn("functional",
+ "allcomplextypes", new String[]{"map_map_col"},
TPrivilegeLevel.SELECT))
+ .error(selectError("functional.allcomplextypes"),
onColumn("functional",
+ "allcomplextypes", new String[]{"int_map_col", "map_map_col"},
+ TPrivilegeLevel.SELECT));
+
for (AuthzTest authzTest: new AuthzTest[]{
// Select with cross join.
authorize("select * from functional.alltypes union all " +
diff --git a/testdata/datasets/functional/functional_schema_template.sql
b/testdata/datasets/functional/functional_schema_template.sql
index fdb5b6f48..4f1ff5cd0 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -3430,6 +3430,16 @@ AS SELECT id, int_array, int_array_array FROM
{db_name}{db_suffix}.complextypest
---- DATASET
functional
---- BASE_TABLE_NAME
+complextypes_maps_view
+---- CREATE
+DROP VIEW IF EXISTS {db_name}{db_suffix}.{table_name};
+CREATE VIEW {db_name}{db_suffix}.{table_name}
+AS SELECT id, int_map, int_map_array FROM {db_name}{db_suffix}.complextypestbl;
+---- LOAD
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
iceberg_v2_delete_positional
---- CREATE
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
@@ -3636,26 +3646,40 @@ select count(*) as mv_count from
{db_name}{db_suffix}.{table_name};
---- DATASET
functional
---- BASE_TABLE_NAME
-array_tbl
+collection_tbl
---- COLUMNS
id INT
-int_1d ARRAY<INT>
-int_2d ARRAY<ARRAY<INT>>
-int_3d ARRAY<ARRAY<ARRAY<INT>>>
-string_1d ARRAY<STRING>
-string_2d ARRAY<ARRAY<STRING>>
-string_3d ARRAY<ARRAY<ARRAY<STRING>>>
+arr_int_1d ARRAY<INT>
+arr_int_2d ARRAY<ARRAY<INT>>
+arr_int_3d ARRAY<ARRAY<ARRAY<INT>>>
+arr_string_1d ARRAY<STRING>
+arr_string_2d ARRAY<ARRAY<STRING>>
+arr_string_3d ARRAY<ARRAY<ARRAY<STRING>>>
+map_1d MAP<INT, STRING>
+map_2d MAP<INT,MAP<INT,STRING>>
+map_3d MAP<INT,MAP<INT,MAP<INT,STRING>>>
+map_map_array MAP<INT,MAP<INT,ARRAY<INT>>>
---- DEPENDENT_LOAD_HIVE
-- It would be nice to insert NULLs, but I couldn't find a way in Hive.
INSERT INTO {db_name}{db_suffix}.{table_name} VALUES
- (1,
- array(1, 2, NULL),
- array(array(1, 2, NULL), array(3)),
- array(array(array(1, 2, NULL), array(3)), array(array(4))),
- array("1", "2", NULL),
- array(array("1", "2", NULL), array("3")),
- array(array(array("1", "2", NULL), array("3")), array(array("4")))
- );
+ (1,
+ array(1, 2, NULL),
+ array(array(1, 2, NULL), array(3)),
+ array(array(array(1, 2, NULL), array(3)), array(array(4))),
+ array("1", "2", NULL),
+ array(array("1", "2", NULL), array("3")),
+ array(array(array("1", "2", NULL), array("3")), array(array("4"))),
+ map(1, "first", 2, "second"),
+ map(1, map(10, "ten", 20, "twenty"), 2, map(30, "thirty", 40, "forty")),
+ map(
+ 1, map(10, map(100, "hundred", 200, "two hundred"), 20, map(300, "three
hundred", 400, "four hundred")),
+ 2, map(30, map(500, "five hundred", 600, "six hundred"), 40, map(700,
"seven hundred", 800, "eight hundred"))
+ ),
+ map(
+ 1, map(10, array(100, 200), 20, array(300, 400)),
+ 2, map(30, array(500, 600), 40, array(700, 800))
+ )
+ );
---- LOAD
====
---- DATASET
diff --git a/testdata/datasets/functional/schema_constraints.csv
b/testdata/datasets/functional/schema_constraints.csv
index c42602db0..7b3a265be 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -344,8 +344,11 @@ table_name:alltypessmall_bool_sorted,
constraint:restrict_to, table_format:orc/d
table_name:complextypes_arrays_only_view, constraint:restrict_to,
table_format:parquet/none/none
table_name:complextypes_arrays_only_view, constraint:restrict_to,
table_format:orc/def/block
-table_name:array_tbl, constraint:restrict_to, table_format:parquet/none/none
-table_name:array_tbl, constraint:restrict_to, table_format:orc/def/block
+table_name:collection_tbl, constraint:restrict_to,
table_format:parquet/none/none
+table_name:collection_tbl, constraint:restrict_to, table_format:orc/def/block
+
+table_name:complextypes_maps_view, constraint:restrict_to,
table_format:parquet/none/none
+table_name:complextypes_maps_view, constraint:restrict_to,
table_format:orc/def/block
# 'alltypestiny_negative' only used in ORC tests.
table_name:alltypestiny_negative, constraint:restrict_to,
table_format:orc/def/block
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
index 8904119d6..14c1c54d1 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test
@@ -386,7 +386,7 @@ INT,INT,STRING
---- QUERY
# Regression test for:
# IMPALA-11434: "More than 1 2d arrays in select list causes analysis error"
-select id, int_1d, int_2d, int_3d, string_1d, string_2d, string_3d from
array_tbl;
+select id, arr_int_1d, arr_int_2d, arr_int_3d, arr_string_1d, arr_string_2d,
arr_string_3d from collection_tbl;
---- RESULTS
1,'[1,2,NULL]','[[1,2,NULL],[3]]','[[[1,2,NULL],[3]],[[4]]]','["1","2",NULL]','[["1","2",NULL],["3"]]','[[["1","2",NULL],["3"]],[["4"]]]'
---- TYPES
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test
b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test
new file mode 100644
index 000000000..622bac4b1
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test
@@ -0,0 +1,326 @@
+====
+---- QUERY
+select id, int_map from complextypestbl
+---- RESULTS
+1,'{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}'
+3,'{}'
+4,'{}'
+5,'{}'
+6,'NULL'
+7,'{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+select id, int_map from complextypestbl where id=1
+---- RESULTS
+1,'{"k1":1,"k2":100}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+select id, int_map, int_map_array from complextypestbl
+---- RESULTS
+1,'{"k1":1,"k2":100}','[{"k1":1}]'
+2,'{"k1":2,"k2":NULL}','[{"k3":NULL,"k1":1},NULL,{}]'
+3,'{}','[NULL,NULL]'
+4,'{}','[]'
+5,'{}','NULL'
+6,'NULL','NULL'
+7,'{"k1":NULL,"k3":NULL}','NULL'
+8,'{"k1":-1}','[{},{"k1":1},{},{}]'
+---- TYPES
+bigint,string,string
+====
+---- QUERY
+# Sorting is not supported yet for collections: IMPALA-10939
+select id, int_map_array, int_map from complextypestbl order by id
+---- CATCH
+IllegalStateException: Sorting is not supported if the select list contains
collection columns.
+====
+---- QUERY
+# Same collection used twice in a select list.
+select id, int_map, int_map from complextypestbl
+---- RESULTS
+1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}'
+3,'{}','{}'
+4,'{}','{}'
+5,'{}','{}'
+6,'NULL','NULL'
+7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}','{"k1":-1}'
+---- TYPES
+bigint,string,string
+====
+---- QUERY
+# Same collection used from two versions of the same table/
+select t1.id, t1.int_map, t2.int_map
+ from complextypestbl t1 join complextypestbl t2
+ on t1.id = t2.id
+---- RESULTS
+1,'{"k1":1,"k2":100}','{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}','{"k1":2,"k2":NULL}'
+3,'{}','{}'
+4,'{}','{}'
+5,'{}','{}'
+6,'NULL','NULL'
+7,'{"k1":NULL,"k3":NULL}','{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}','{"k1":-1}'
+---- TYPES
+bigint,string,string
+====
+---- QUERY
+select id, int_map from complextypestbl union all select id, int_map from
complextypestbl
+---- RESULTS
+1,'{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}'
+3,'{}'
+4,'{}'
+5,'{}'
+6,'NULL'
+7,'{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}'
+1,'{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}'
+3,'{}'
+4,'{}'
+5,'{}'
+6,'NULL'
+7,'{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+# TODO: only UNION ALL is supported. UNION needs several utility functions in
the BE, so
+# for now we reject it in the FE.
+select id, int_map from complextypestbl union select id, int_map from
complextypestbl;
+---- CATCH
+IllegalStateException: UNION, EXCEPT and INTERSECT are not supported for
collection types
+====
+---- QUERY
+# Changing a column to a different type leads "non-pass-through" union that
does a
+# deepcopy on the tuple, which is not yet implemented in BE for arrays. This
case is
+# currently caught in the planner.
+select id, int_map from complextypestbl
+ union all select cast(id as tinyint), int_map from complextypestbl
+---- CATCH
+IllegalStateException: only pass-through UNION ALL is supported for array
columns
+====
+---- QUERY
+# Constants in the select list of unions also lead to "non-pass-through" union.
+select 1, int_map from complextypestbl
+ union all select 2, int_map from complextypestbl;
+---- CATCH
+IllegalStateException: only pass-through UNION ALL is supported for array
columns
+====
+---- QUERY
+select 1 from (select int_map from complextypestbl) s
+---- RESULTS
+1
+1
+1
+1
+1
+1
+1
+1
+---- TYPES
+tinyint
+====
+---- QUERY
+select id, int_map from (select id, int_map from complextypestbl) s;
+---- RESULTS
+1,'{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}'
+3,'{}'
+4,'{}'
+5,'{}'
+6,'NULL'
+7,'{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+with s as (select id, t.int_map from complextypestbl t)
+select id, int_map from s;
+---- RESULTS
+1,'{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}'
+3,'{}'
+4,'{}'
+5,'{}'
+6,'NULL'
+7,'{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+select id, int_map from complextypes_maps_view;
+---- RESULTS
+1,'{"k1":1,"k2":100}'
+2,'{"k1":2,"k2":NULL}'
+3,'{}'
+4,'{}'
+5,'{}'
+6,'NULL'
+7,'{"k1":NULL,"k3":NULL}'
+8,'{"k1":-1}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+# Unnesting map returned by view.
+select id, m.key, m.value from complextypes_maps_view v, v.int_map m;
+---- RESULTS
+1,'k1',1
+1,'k2',100
+2,'k1',2
+2,'k2',NULL
+7,'k1',NULL
+7,'k3',NULL
+8,'k1',-1
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+# Unnesting map returned from WITH clause and predicate in inner query.
+with v as (select id, int_map from complextypestbl where id=1)
+select v.id, a.key, a.value from v, v.int_map a;
+---- RESULTS
+1,'k1',1
+1,'k2',100
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+# Unnesting map returned from WITH clause and predicate in outer query.
+with v as (select id, int_map from complextypestbl)
+select v.id, a.key, a.value from v, v.int_map a where id=1;
+---- RESULTS
+1,'k1',1
+1,'k2',100
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+# Unnesting map returned from WITH clause on item.
+with v as (select id, int_map from complextypestbl)
+select v.id, a.key, a.value from v, v.int_map a where a.key='k1'
+---- RESULTS
+1,'k1',1
+2,'k1',2
+7,'k1',NULL
+8,'k1',-1
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+# Unnesting map returned by view wrapped in inline view.
+select v.id, a.key, a.value from
+ (select id, int_map from complextypes_maps_view) v, v.int_map a;
+---- RESULTS
+1,'k1',1
+1,'k2',100
+2,'k1',2
+2,'k2',NULL
+7,'k1',NULL
+7,'k3',NULL
+8,'k1',-1
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+# Unnesting map returned by view wrapped in inline view + WITH clause.
+with v2 as (select id, int_map from complextypes_maps_view)
+select v.id, a.key, a.value from (select id, int_map from v2) v, v.int_map a;
+---- RESULTS
+1,'k1',1
+1,'k2',100
+2,'k1',2
+2,'k2',NULL
+7,'k1',NULL
+7,'k3',NULL
+8,'k1',-1
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+# Unnesting map returned by view wrapped in inline view + WITH clause.
+with v2 as (select id, int_map from complextypes_maps_view)
+select v.id, a.key, a.value from
+ (select id, int_map from v2 where id=1) v, v.int_map a
+ where a.key='k1';
+---- RESULTS
+1,'k1',1
+---- TYPES
+bigint,string,int
+====
+---- QUERY
+select item from unnest(complextypestbl.int_map_array)
+---- RESULTS
+'{"k1":1}'
+'{"k3":NULL,"k1":1}'
+'NULL'
+'{}'
+'NULL'
+'NULL'
+'{}'
+'{"k1":1}'
+'{}'
+'{}'
+---- TYPES
+string
+====
+---- QUERY
+select id, a.key, a.value from complextypes_maps_view t left join t.int_map a;
+---- RESULTS
+1,'k1',1
+1,'k2',100
+2,'k1',2
+2,'k2',NULL
+3,'NULL',NULL
+4,'NULL',NULL
+5,'NULL',NULL
+6,'NULL',NULL
+7,'k1',NULL
+7,'k3',NULL
+8,'k1',-1
+---- TYPES
+BIGINT,STRING,INT
+====
+---- QUERY
+select id, a2.key, a2.value from complextypes_maps_view v, v.int_map_array a1,
a1.item a2;
+---- RESULTS
+1,'k1',1
+2,'k3',NULL
+2,'k1',1
+8,'k1',1
+---- TYPES
+BIGINT,STRING,INT
+====
+---- QUERY
+# Regression test for:
+# IMPALA-11434: "More than 1 2d arrays in select list causes analysis error"
+select id, map_1d, map_2d, map_3d, arr_int_3d, map_map_array from
collection_tbl;
+---- RESULTS
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{1:{10:{100:"hundred",200:"two
hundred"},20:{300:"three hundred",400:"four hundred"}},2:{30:{500:"five
hundred",600:"six hundred"},40:{700:"seven hundred",800:"eight
hundred"}}}','[[[1,2,NULL],[3]],[[4]]]','{1:{10:[100,200],20:[300,400]},2:{30:[500,600],40:[700,800]}}'
+---- TYPES
+INT,STRING,STRING,STRING,STRING,STRING
+=====
+---- QUERY
+select id, map_1d, map_2d, mma.value mma_value, ma.value ma_value
+from collection_tbl c, c.map_map_array mma, mma.value ma;
+---- RESULTS
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{10:[100,200],20:[300,400]}','[100,200]'
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{10:[100,200],20:[300,400]}','[300,400]'
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{30:[500,600],40:[700,800]}','[500,600]'
+1,'{1:"first",2:"second"}','{1:{10:"ten",20:"twenty"},2:{30:"thirty",40:"forty"}}','{30:[500,600],40:[700,800]}','[700,800]'
+---- TYPES
+INT,STRING,STRING,STRING,STRING
+=====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
index 3f9fd0606..316f573b4 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_complex_types.test
@@ -632,3 +632,16 @@ select id, a.item from complextypes_arrays_only_view v,
v.int_array a;
---- TYPES
BIGINT,INT
====
+---- QUERY
+select id, a.key, a.value from complextypes_maps_view v, v.int_map a;
+---- RESULTS
+100,'k1',1
+100,'k2',100
+200,'k1',2
+200,'k2',NULL
+700,'k1',NULL
+700,'k3',NULL
+800,'k1',-1
+---- TYPES
+BIGINT,STRING,INT
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
index 1e3186ff4..573f7cc2d 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
@@ -46,4 +46,34 @@ NULL,'NULL'
NULL,'[NULL,[5,6]]'
---- TYPES
bigint,string
-====
\ No newline at end of file
+====
+---- QUERY
+select id, int_map from functional_orc_def.complextypestbl
+---- RESULTS
+NULL,'{"k1":-1}'
+NULL,'{"k1":1,"k2":100}'
+NULL,'{"k1":2,"k2":NULL}'
+NULL,'{}'
+NULL,'{}'
+NULL,'{}'
+NULL,'NULL'
+NULL,'{"k1":NULL,"k3":NULL}'
+---- TYPES
+bigint,string
+====
+---- QUERY
+# There is a column mask added for int_map_array, but it is ignored.
+# It is not clear whether we should support this (RANGER-3525).
+select id, int_map_array from functional_orc_def.complextypestbl
+---- RESULTS
+NULL,'[{},{"k1":1},{},{}]'
+NULL,'[{"k1":1}]'
+NULL,'[{"k3":NULL,"k1":1},NULL,{}]'
+NULL,'[NULL,NULL]'
+NULL,'[]'
+NULL,'NULL'
+NULL,'NULL'
+NULL,'NULL'
+---- TYPES
+bigint,string
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
index deddd5159..f2b5ed498 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/struct-in-select-list.test
@@ -642,3 +642,9 @@ select nested_struct.c.d from complextypestbl;
---- CATCH
AnalysisException: STRUCT type inside collection types is not supported.
====
+---- QUERY
+# Structs inside maps are not yet supported.
+select nested_struct.g from complextypestbl;
+---- CATCH
+AnalysisException: STRUCT type inside collection types is not supported.
+====
diff --git a/tests/authorization/test_ranger.py
b/tests/authorization/test_ranger.py
index 9a989395c..23c5a1e73 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -1809,6 +1809,10 @@ class
TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
unique_name + str(policy_cnt), user, "functional_orc_def",
"complextypestbl", "int_array_array", "MASK_NULL")
policy_cnt += 1
+ TestRanger._add_column_masking_policy(
+ unique_name + str(policy_cnt), user, "functional_orc_def",
+ "complextypestbl", "int_array_map", "MASK_NULL")
+ policy_cnt += 1
self.execute_query_expect_success(admin_client, "refresh authorization",
user=ADMIN)
self.run_test_case("QueryTest/ranger_column_masking_struct_in_select_list",
vector,
diff --git a/tests/query_test/test_nested_types.py
b/tests/query_test/test_nested_types.py
index 0097fa117..9b3ca59ca 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -152,7 +152,7 @@ class TestNestedStructsInSelectList(ImpalaTestSuite):
self.run_test_case('QueryTest/nested-struct-in-select-list', new_vector)
-class TestNestedTArraysInSelectList(ImpalaTestSuite):
+class TestNestedCollectionsInSelectList(ImpalaTestSuite):
"""Functional tests for nested arrays provided in the select list."""
@classmethod
def get_workload(self):
@@ -160,7 +160,7 @@ class TestNestedTArraysInSelectList(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
- super(TestNestedTArraysInSelectList, cls).add_test_dimensions()
+ super(TestNestedCollectionsInSelectList, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_constraint(lambda v:
v.get_value('table_format').file_format in ['parquet', 'orc'])
cls.ImpalaTestMatrix.add_dimension(
@@ -176,6 +176,10 @@ class TestNestedTArraysInSelectList(ImpalaTestSuite):
"""Queries where an array column is in the select list"""
self.run_test_case('QueryTest/nested-array-in-select-list', vector)
+ def test_map_in_select_list(self, vector, unique_database):
+ """Queries where a map column is in the select list"""
+ self.run_test_case('QueryTest/nested-map-in-select-list', vector)
+
# Moved this to a separate test class from TestNestedTypesInSelectList because
this needs
# a narrower test vector.
