This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new f705496e3 IMPALA-11796: Support structs in select list with Beeswax
f705496e3 is described below
commit f705496e34ac474e8e1c999619e3b928c5e39e0f
Author: Daniel Becker <[email protected]>
AuthorDate: Tue Dec 13 14:35:27 2022 +0100
IMPALA-11796: Support structs in select list with Beeswax
Structs have been allowed in the select list with the HS2 protocol but
not with Beeswax.
IMPALA-11717 makes the printing of complex types (including structs)
independent of the protocol, so we can add support for structs with
Beeswax.
Testing:
- enabled struct tests in test_nested_types.py for Beeswax
Change-Id: I579f7a6ab9bebda306a4acba80d423c1da4ea94c
Reviewed-on: http://gerrit.cloudera.org:8080/19352
Reviewed-by: Daniel Becker <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
be/src/service/impala-beeswax-server.cc | 7 ++----
be/src/service/query-result-set.cc | 42 ++++++++++++++++++---------------
be/src/service/query-result-set.h | 10 ++++----
tests/query_test/test_nested_types.py | 29 -----------------------
4 files changed, 30 insertions(+), 58 deletions(-)
diff --git a/be/src/service/impala-beeswax-server.cc
b/be/src/service/impala-beeswax-server.cc
index e73a41e11..da3c05a98 100644
--- a/be/src/service/impala-beeswax-server.cc
+++ b/be/src/service/impala-beeswax-server.cc
@@ -207,14 +207,11 @@ string ImpalaServer::ColumnTypeToBeeswaxTypeString(const
TColumnType& type) {
DCHECK(type.types[0].__isset.scalar_type);
return TypeToOdbcString(type);
} else if (type.types[0].type == TTypeNodeType::ARRAY
- || type.types[0].type == TTypeNodeType::MAP) {
+ || type.types[0].type == TTypeNodeType::MAP
+ || type.types[0].type == TTypeNodeType::STRUCT) {
DCHECK_GT(type.types.size(), 1);
// TODO (IMPALA-11041): consider returning the real type
return "string";
- } else if (type.types[0].type == TTypeNodeType::STRUCT) {
- DCHECK_GT(type.types.size(), 1);
- RaiseBeeswaxException("Returning struct types is not supported through the
"
- "beeswax interface", SQLSTATE_GENERAL_ERROR);
} else {
DCHECK(false);
return "";
diff --git a/be/src/service/query-result-set.cc
b/be/src/service/query-result-set.cc
index 0897a4d53..17f37c208 100644
--- a/be/src/service/query-result-set.cc
+++ b/be/src/service/query-result-set.cc
@@ -202,8 +202,7 @@ Status AsciiQueryResultSet::AddRows(const
vector<ScalarExprEvaluator*>& expr_eva
&out_stream);
} else if (metadata_.columns[i].columnType.types.size() > 1) {
ColumnType col_type =
ColumnType::FromThrift(metadata_.columns[i].columnType);
- DCHECK(col_type.IsArrayType() || col_type.IsMapType());
- PrintCollectionValue(expr_evals[i], it.Get(), &out_stream,
col_type.type);
+ PrintComplexValue(expr_evals[i], it.Get(), &out_stream, col_type);
} else {
DCHECK(false);
}
@@ -214,29 +213,34 @@ Status AsciiQueryResultSet::AddRows(const
vector<ScalarExprEvaluator*>& expr_eva
return Status::OK();
}
-void QueryResultSet::PrintCollectionValue(ScalarExprEvaluator* expr_eval,
- const TupleRow* row, stringstream *stream, PrimitiveType collection_type) {
- DCHECK(collection_type == PrimitiveType::TYPE_ARRAY
- || collection_type == PrimitiveType::TYPE_MAP);
+void QueryResultSet::PrintComplexValue(ScalarExprEvaluator* expr_eval,
+ const TupleRow* row, stringstream *stream, const ColumnType& type) {
+ DCHECK(type.IsComplexType());
const ScalarExpr& scalar_expr = expr_eval->root();
- // Currently scalar_expr can be only a slot ref as no functions return
arrays.
+ // Currently scalar_expr can be only a slot ref as no functions return
complex types.
DCHECK(scalar_expr.IsSlotRef());
- const TupleDescriptor* item_tuple_desc =
scalar_expr.GetCollectionTupleDesc();
- DCHECK(item_tuple_desc != nullptr);
- const CollectionValue* array_val =
- static_cast<const CollectionValue*>(expr_eval->GetValue(row));
+ void* value = expr_eval->GetValue(row);
- if (array_val != nullptr) {
- rapidjson::BasicOStreamWrapper<stringstream> wrapped_stream(*stream);
- rapidjson::Writer<rapidjson::BasicOStreamWrapper<stringstream>> writer(
- wrapped_stream);
+ if (value == nullptr) {
+ (*stream) << RawValue::NullLiteral(/*top-level*/ true);
+ return;
+ }
+
+ rapidjson::BasicOStreamWrapper<stringstream> wrapped_stream(*stream);
+ rapidjson::Writer<rapidjson::BasicOStreamWrapper<stringstream>>
writer(wrapped_stream);
+
+ if (type.IsCollectionType()) {
+ const CollectionValue* collection_val = static_cast<const
CollectionValue*>(value);
+ const TupleDescriptor* item_tuple_desc =
scalar_expr.GetCollectionTupleDesc();
+ DCHECK(item_tuple_desc != nullptr);
ComplexValueWriter<rapidjson::BasicOStreamWrapper<stringstream>>
- ::CollectionValueToJSON(*array_val,
- collection_type,
- item_tuple_desc, &writer);
+ ::CollectionValueToJSON(*collection_val, type.type, item_tuple_desc,
&writer);
} else {
- (*stream) << RawValue::NullLiteral(/*top-level*/ true);
+ DCHECK(type.IsStructType());
+ const StructVal* struct_val = static_cast<const StructVal*>(value);
+ ComplexValueWriter<rapidjson::BasicOStreamWrapper<stringstream>>
+ ::StructValToJSON(*struct_val, type, &writer);
}
}
diff --git a/be/src/service/query-result-set.h
b/be/src/service/query-result-set.h
index 32f36fd73..50f2f4430 100644
--- a/be/src/service/query-result-set.h
+++ b/be/src/service/query-result-set.h
@@ -79,11 +79,11 @@ class QueryResultSet {
apache::hive::service::cli::thrift::TRowSet* rowset);
protected:
- /// Wrapper to callComplexValueWriter::CollectionValueToJSON() for a given
collection
- /// column. expr_eval must be a SlotRef on a collection slot.
- static void PrintCollectionValue(ScalarExprEvaluator* expr_eval, const
TupleRow* row,
- std::stringstream *stream, PrimitiveType collection_type);
-
+ /// Wrapper to call ComplexValueWriter::CollectionValueToJSON() or
+ /// ComplexValueWriter::StructValToJSON() for a given complex column.
expr_eval must be
+ /// a SlotRef on a complex-typed (collection or struct) slot.
+ static void PrintComplexValue(ScalarExprEvaluator* expr_eval, const
TupleRow* row,
+ std::stringstream *stream, const ColumnType& type);
};
}
diff --git a/tests/query_test/test_nested_types.py
b/tests/query_test/test_nested_types.py
index b45a4e2cb..8c1c50e9f 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -129,8 +129,6 @@ class TestNestedStructsInSelectList(ImpalaTestSuite):
create_exec_option_dimension_from_dict({
'disable_codegen': ['False', 'True']}))
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
- cls.ImpalaTestMatrix.add_constraint(lambda v:
- v.get_value('protocol') == 'hs2')
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('orc_schema_resolution',
0, 1))
cls.ImpalaTestMatrix.add_constraint(orc_schema_resolution_constraint)
@@ -187,33 +185,6 @@ class TestNestedCollectionsInSelectList(ImpalaTestSuite):
self.run_test_case('QueryTest/map_null_keys', vector)
-# Moved this to a separate test class from TestNestedTypesInSelectList because
this needs
-# a narrower test vector.
-class TestNestedTypesInSelectListWithBeeswax(ImpalaTestSuite):
- """Functional tests for nested types provided in the select list."""
- @classmethod
- def get_workload(self):
- return 'functional-query'
-
- @classmethod
- def add_test_dimensions(cls):
- super(TestNestedTypesInSelectListWithBeeswax, cls).add_test_dimensions()
- cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
- cls.ImpalaTestMatrix.add_constraint(lambda v:
- v.get_value('protocol') == 'beeswax')
-
cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
- cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
- disable_codegen_options=[True]))
-
- def test_struct_with_beeswax(self, vector):
- expected_err = "Returning struct types is not supported through the
beeswax " + \
- "interface"
- err = self.execute_query_expect_failure(self.client,
- "select tiny_struct from functional_orc_def.complextypes_structs",
- vector.get_value('exec_option'))
- assert expected_err in str(err)
-
-
class TestComputeStatsWithNestedTypes(ImpalaTestSuite):
"""Functional tests for running compute stats on tables that have nested
types in the
columns."""
