This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 405d8df83ab [feat](Variant) add variant_enable_nested_group property
and fix variant nullable issue (#61047)
405d8df83ab is described below
commit 405d8df83aba541e32ee4aff688481d10e8b276c
Author: lihangyu <[email protected]>
AuthorDate: Thu Mar 5 12:07:54 2026 +0800
[feat](Variant) add variant_enable_nested_group property and fix variant
nullable issue (#61047)
---
.../variant/variant_column_writer_impl.cpp | 27 +++++++++++-------
be/src/olap/tablet_meta.cpp | 3 ++
be/src/olap/tablet_schema.cpp | 4 +++
be/src/olap/tablet_schema.h | 6 ++++
.../vec/functions/array/function_array_utils.cpp | 5 +++-
.../java/org/apache/doris/catalog/ScalarType.java | 7 +++++
.../java/org/apache/doris/catalog/VariantType.java | 18 ++++++++++--
.../main/java/org/apache/doris/catalog/Column.java | 9 ++++++
.../apache/doris/common/util/PropertyAnalyzer.java | 33 ++++++++++++++++++++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 16 +++++++++--
.../org/apache/doris/nereids/types/DataType.java | 3 +-
.../apache/doris/nereids/types/VariantType.java | 11 ++++++--
.../java/org/apache/doris/qe/SessionVariable.java | 13 +++++++++
.../ExternalFileTableValuedFunction.java | 3 +-
gensrc/proto/olap_file.proto | 1 +
gensrc/thrift/Descriptors.thrift | 1 +
16 files changed, 140 insertions(+), 20 deletions(-)
diff --git
a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
index f5de9c1e651..e29bdb0141f 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
@@ -1308,11 +1308,13 @@ Status VariantColumnWriterImpl::finalize() {
column->parent_unique_id() == current_variant_uid;
});
if (!has_extracted_columns) {
- RETURN_IF_ERROR(build_nested_group_routing_plan(*ptr,
&_nested_group_routing_plan));
+ if (_tablet_column->variant_enable_nested_group()) {
+ RETURN_IF_ERROR(build_nested_group_routing_plan(*ptr,
&_nested_group_routing_plan));
- // Root NG dedup is handled in _process_root_column() — see the
- // has_root_ng check there. We intentionally do NOT modify the
in-memory
- // root data here because _nested_group_provider->prepare() needs it.
+ // Root NG dedup is handled in _process_root_column() — see the
+ // has_root_ng check there. We intentionally do NOT modify the
in-memory
+ // root data here because _nested_group_provider->prepare() needs
it.
+ }
}
RETURN_IF_ERROR(ptr->pick_subcolumns_to_sparse_column(
@@ -1341,9 +1343,12 @@ Status VariantColumnWriterImpl::finalize() {
}
// NestedGroup write behavior is determined by the injected provider
implementation.
- RETURN_IF_ERROR(_nested_group_provider->prepare(
- *ptr, /*include_jsonb_subcolumns=*/true, _tablet_column, _opts,
- olap_data_convertor.get(), num_rows, &column_id, &_statistics));
+ // Only invoke the provider when nested group writing is enabled.
+ if (_tablet_column->variant_enable_nested_group()) {
+ RETURN_IF_ERROR(_nested_group_provider->prepare(
+ *ptr, /*include_jsonb_subcolumns=*/true, _tablet_column, _opts,
+ olap_data_convertor.get(), num_rows, &column_id,
&_statistics));
+ }
if (_binary_writer) {
_binary_writer->merge_stats_to(&_statistics);
}
@@ -1558,9 +1563,11 @@ Status VariantSubcolumnWriter::finalize() {
_opts.meta->set_num_rows(ptr->rows());
++column_id;
- RETURN_IF_ERROR(_nested_group_provider->prepare(*ptr,
/*include_jsonb_subcolumns=*/false,
- &flush_column, _opts,
olap_data_convertor.get(),
- ptr->rows(), &column_id,
&_statistics));
+ if (parent_column.variant_enable_nested_group()) {
+ RETURN_IF_ERROR(_nested_group_provider->prepare(
+ *ptr, /*include_jsonb_subcolumns=*/false, &flush_column, _opts,
+ olap_data_convertor.get(), ptr->rows(), &column_id,
&_statistics));
+ }
_statistics.to_pb(_opts.meta->mutable_variant_statistics());
_is_finalized = true;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 8a4c9da9fd2..be701d54f7c 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -562,6 +562,9 @@ void TabletMeta::init_column_from_tcolumn(uint32_t
unique_id, const TColumn& tco
if (tcolumn.__isset.variant_doc_hash_shard_count) {
column->set_variant_doc_hash_shard_count(tcolumn.variant_doc_hash_shard_count);
}
+ if (tcolumn.__isset.variant_enable_nested_group) {
+
column->set_variant_enable_nested_group(tcolumn.variant_enable_nested_group);
+ }
}
void TabletMeta::remove_rowset_delete_bitmap(const RowsetId& rowset_id, const
Version& version) {
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 051b8b075cf..a7295b6a941 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -678,6 +678,9 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
if (column.has_variant_doc_hash_shard_count()) {
_variant.doc_hash_shard_count = column.variant_doc_hash_shard_count();
}
+ if (column.has_variant_enable_nested_group()) {
+ _variant.enable_nested_group = column.variant_enable_nested_group();
+ }
if (column.has_pattern_type()) {
_pattern_type = column.pattern_type();
}
@@ -764,6 +767,7 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
column->set_variant_enable_doc_mode(_variant.enable_doc_mode);
column->set_variant_doc_materialization_min_rows(_variant.doc_materialization_min_rows);
column->set_variant_doc_hash_shard_count(_variant.doc_hash_shard_count);
+ column->set_variant_enable_nested_group(_variant.enable_nested_group);
}
void TabletColumn::add_sub_column(TabletColumn& sub_column) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 5fdd35f8a6d..96baf5c45f3 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -80,6 +80,8 @@ public:
bool enable_doc_mode = false;
int64_t doc_materialization_min_rows = 0;
int32_t doc_hash_shard_count = 64;
+
+ bool enable_nested_group = false;
};
TabletColumn();
@@ -279,6 +281,10 @@ public:
_variant.enable_typed_paths_to_sparse =
variant_enable_typed_paths_to_sparse;
}
+ bool variant_enable_nested_group() const { return
_variant.enable_nested_group; }
+
+ void set_variant_enable_nested_group(bool val) {
_variant.enable_nested_group = val; }
+
bool is_decimal() const { return _is_decimal; }
private:
diff --git a/be/src/vec/functions/array/function_array_utils.cpp
b/be/src/vec/functions/array/function_array_utils.cpp
index 7620d7179c8..fc96cb7288c 100644
--- a/be/src/vec/functions/array/function_array_utils.cpp
+++ b/be/src/vec/functions/array/function_array_utils.cpp
@@ -27,6 +27,7 @@
#include "vec/columns/column_variant.h"
#include "vec/columns/column_vector.h"
#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_nullable.h"
namespace doris::vectorized {
@@ -58,7 +59,9 @@ bool extract_column_array_info(const IColumn& src,
ColumnArrayExecutionData& dat
data.nested_type->get_primitive_type() != PrimitiveType::TYPE_VARIANT)
{
// set variant root column/type to from column/type
auto variant = ColumnVariant::create(true /*always nullable*/);
- variant->create_root(data.nested_type,
make_nullable(data.nested_col)->assume_mutable());
+ auto nullable_nested_type = make_nullable(data.nested_type);
+ auto nullable_col = make_nullable(data.nested_col);
+ variant->create_root(nullable_nested_type,
std::move(*nullable_col).mutate());
data.nested_col = variant->get_ptr();
}
return true;
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
index 99f7ac60a21..dc73f384e14 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -1066,4 +1066,11 @@ public class ScalarType extends Type {
// Backward-compatible default bucket count used by BE doc snapshot
writer/reader.
return 128;
}
+
+ public boolean getVariantEnableNestedGroup() {
+ if (this instanceof VariantType) {
+ return ((VariantType) this).getEnableNestedGroup();
+ }
+ return false; // Default to false for backward compatibility.
+ }
}
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
index da062d21489..69917dbf4b7 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
@@ -61,6 +61,9 @@ public class VariantType extends ScalarType {
@SerializedName(value = "variantDocShardCount")
private final int variantDocShardCount;
+ @SerializedName(value = "enableNestedGroup")
+ private final boolean enableNestedGroup;
+
private Map<String, String> properties = Maps.newHashMap();
public VariantType() {
@@ -73,6 +76,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(ArrayList<VariantField> fields) {
@@ -89,6 +93,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(Map<String, String> properties) {
@@ -102,6 +107,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(ArrayList<VariantField> fields, Map<String, String>
properties) {
@@ -119,6 +125,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(ArrayList<VariantField> fields, int
variantMaxSubcolumnsCount,
@@ -127,7 +134,8 @@ public class VariantType extends ScalarType {
int
variantSparseHashShardCount,
boolean
enableVariantDocMode,
long
variantDocMaterializationMinRows,
- int
variantDocShardCount) {
+ int
variantDocShardCount,
+ boolean
enableNestedGroup) {
super(PrimitiveType.VARIANT);
Preconditions.checkNotNull(fields);
this.predefinedFields = fields;
@@ -141,6 +149,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = enableVariantDocMode;
this.variantDocMaterializationMinRows =
variantDocMaterializationMinRows;
this.variantDocShardCount = variantDocShardCount;
+ this.enableNestedGroup = enableNestedGroup;
}
@Override
@@ -218,7 +227,8 @@ public class VariantType extends ScalarType {
&& variantMaxSubcolumnsCount ==
otherVariantType.variantMaxSubcolumnsCount
&& enableTypedPathsToSparse ==
otherVariantType.enableTypedPathsToSparse
&& enableVariantDocMode ==
otherVariantType.enableVariantDocMode
- && variantDocMaterializationMinRows ==
otherVariantType.variantDocMaterializationMinRows;
+ && variantDocMaterializationMinRows ==
otherVariantType.variantDocMaterializationMinRows
+ && enableNestedGroup == otherVariantType.enableNestedGroup;
}
@Override
@@ -257,4 +267,8 @@ public class VariantType extends ScalarType {
public int getVariantDocShardCount() {
return variantDocShardCount;
}
+
+ public boolean getEnableNestedGroup() {
+ return enableNestedGroup;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 457259856cf..bf525428061 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -659,6 +659,7 @@ public class Column implements GsonPostProcessable {
tColumn.setVariantEnableDocMode(this.getVariantEnableDocMode());
tColumn.setVariantDocMaterializationMinRows(this.getvariantDocMaterializationMinRows());
tColumn.setVariantDocHashShardCount(this.getVariantDocShardCount());
+
tColumn.setVariantEnableNestedGroup(this.getVariantEnableNestedGroup());
// ATTN:
// Currently, this `toThrift()` method is only used from
CreateReplicaTask.
// And CreateReplicaTask does not need `defineExpr` field.
@@ -886,6 +887,7 @@ public class Column implements GsonPostProcessable {
builder.setVariantEnableDocMode(this.getVariantEnableDocMode());
builder.setVariantDocMaterializationMinRows(this.getvariantDocMaterializationMinRows());
builder.setVariantDocHashShardCount(this.getVariantDocShardCount());
+
builder.setVariantEnableNestedGroup(this.getVariantEnableNestedGroup());
// variant may contain predefined structured fields
addChildren(builder);
}
@@ -977,6 +979,9 @@ public class Column implements GsonPostProcessable {
if (this.getVariantDocShardCount() !=
other.getVariantDocShardCount()) {
throw new DdlException("Can not change variant doc snapshot
shard count");
}
+ if (this.getVariantEnableNestedGroup() !=
other.getVariantEnableNestedGroup()) {
+ throw new DdlException("Can not change variant enable nested
group");
+ }
if (CollectionUtils.isNotEmpty(this.getChildren()) ||
CollectionUtils.isNotEmpty(other.getChildren())) {
throw new DdlException("Can not change variant schema
templates");
}
@@ -1336,6 +1341,10 @@ public class Column implements GsonPostProcessable {
return type.isVariantType() ? ((ScalarType)
type).getVariantDocShardCount() : 128;
}
+ public boolean getVariantEnableNestedGroup() {
+ return type.isVariantType() ? ((ScalarType)
type).getVariantEnableNestedGroup() : false;
+ }
+
public void setFieldPatternType(TPatternType type) {
fieldPatternType = type;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 0d31eeb769b..4a9818aed7d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -259,6 +259,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT =
"variant_max_subcolumns_count";
public static final String PROPERTIES_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE
= "variant_enable_typed_paths_to_sparse";
+ public static final String PROPERTIES_VARIANT_ENABLE_NESTED_GROUP =
"variant_enable_nested_group";
public static final String PROPERTIES_TDE_ALGORITHM = "tde_algorithm";
public static final String AES256 = "AES256";
public static final String SM4 = "SM4";
@@ -2063,6 +2064,21 @@ public class PropertyAnalyzer {
return enableTypedPathsToSparse;
}
+ public static boolean analyzeEnableNestedGroup(Map<String, String>
properties,
+ boolean defaultValue) throws AnalysisException {
+ boolean enableNestedGroup = defaultValue;
+ if (properties != null &&
properties.containsKey(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP)) {
+ String enableNestedGroupStr =
properties.get(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP);
+ try {
+ enableNestedGroup = Boolean.parseBoolean(enableNestedGroupStr);
+ } catch (Exception e) {
+ throw new AnalysisException("variant_enable_nested_group must
be `true` or `false`");
+ }
+ properties.remove(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP);
+ }
+ return enableNestedGroup;
+ }
+
public static int analyzeVariantMaxSparseColumnStatisticsSize(Map<String,
String> properties, int defuatValue)
throws AnalysisException {
int maxSparseColumnStatisticsSize = defuatValue;
@@ -2176,6 +2192,23 @@ public class PropertyAnalyzer {
+ "cannot be set together");
}
}
+ // variant_enable_nested_group=true is mutually exclusive with
+ // variant_enable_doc_mode=true and variant_max_subcolumns_count>0
+ if (properties != null &&
properties.containsKey(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP)
+ &&
"true".equalsIgnoreCase(properties.get(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP)))
{
+ if (properties.containsKey(PROPERTIES_VARIANT_ENABLE_DOC_MODE)
+ &&
"true".equalsIgnoreCase(properties.get(PROPERTIES_VARIANT_ENABLE_DOC_MODE))) {
+ throw new AnalysisException("variant_enable_nested_group and
variant_enable_doc_mode "
+ + "cannot both be true");
+ }
+ if
(properties.containsKey(PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT)) {
+ int count =
Integer.parseInt(properties.get(PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT));
+ if (count > 0) {
+ throw new AnalysisException("variant_enable_nested_group
cannot be true when "
+ + "variant_max_subcolumns_count > 0");
+ }
+ }
+ }
}
public static TEncryptionAlgorithm analyzeTDEAlgorithm(Map<String, String>
properties) throws AnalysisException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index af20d91c38c..2675541b4f8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -5155,6 +5155,9 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
int variantDocHashShardCount = ConnectContext.get() == null ? 128 :
ConnectContext.get().getSessionVariable().getDefaultVariantDocHashShardCount();
+ boolean enableNestedGroup = ConnectContext.get() == null ? true :
+
ConnectContext.get().getSessionVariable().getDefaultVariantEnableNestedGroup();
+
try {
// validate properties: variant_enable_doc_mode cannot be set
together with other properties
PropertyAnalyzer.validateVariantProperties(properties);
@@ -5173,10 +5176,17 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
.analyzeVariantDocMaterializationMinRows(properties,
variantDocMaterializationMinRows);
variantDocHashShardCount = PropertyAnalyzer
.analyzeVariantDocHashShardCount(properties,
variantDocHashShardCount);
+ enableNestedGroup = PropertyAnalyzer
+ .analyzeEnableNestedGroup(properties, enableNestedGroup);
} catch (org.apache.doris.common.AnalysisException e) {
throw new NotSupportedException(e.getMessage());
}
+ if (enableNestedGroup) {
+ throw new NotSupportedException(
+ "variant_enable_nested_group is not supported now");
+ }
+
// When doc mode is enabled, disable subcolumn extraction and sparse
column features
if (enableVariantDocMode) {
// Disable sparse column features
@@ -5214,12 +5224,14 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
+ " and " +
PropertyAnalyzer.PROPERTIES_VARIANT_SPARSE_HASH_SHARD_COUNT
+ " and " +
PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_DOC_MODE
+ " and " +
PropertyAnalyzer.PROPERTIES_VARIANT_DOC_MATERIALIZATION_MIN_ROWS
- + " and " +
PropertyAnalyzer.PROPERTIES_VARIANT_DOC_HASH_SHARD_COUNT);
+ + " and " +
PropertyAnalyzer.PROPERTIES_VARIANT_DOC_HASH_SHARD_COUNT
+ + " and " +
PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_NESTED_GROUP);
}
return new VariantType(fields, variantMaxSubcolumnsCount,
enableTypedPathsToSparse,
variantMaxSparseColumnStatisticsSize,
variantSparseHashShardCount,
- enableVariantDocMode, variantDocMaterializationMinRows,
variantDocHashShardCount);
+ enableVariantDocMode, variantDocMaterializationMinRows,
variantDocHashShardCount,
+ enableNestedGroup);
}
private static boolean isSupportedVariantDocModeType(DataType type) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 20095922991..61041674db2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -493,7 +493,8 @@ public abstract class DataType {
((org.apache.doris.catalog.VariantType)
type).getVariantSparseHashShardCount(),
((org.apache.doris.catalog.VariantType)
type).getEnableVariantDocMode(),
((org.apache.doris.catalog.VariantType)
type).getvariantDocMaterializationMinRows(),
- ((org.apache.doris.catalog.VariantType)
type).getVariantDocShardCount());
+ ((org.apache.doris.catalog.VariantType)
type).getVariantDocShardCount(),
+ ((org.apache.doris.catalog.VariantType)
type).getEnableNestedGroup());
}
return VariantType.INSTANCE;
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
index 4ac0e597e82..16f7d72076f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
@@ -55,6 +55,7 @@ public class VariantType extends PrimitiveType {
private final boolean enableVariantDocMode;
private final long variantDocMaterializationMinRows;
private final int variantDocShardCount;
+ private final boolean enableNestedGroup;
/**
* Creates a Variant type without predefined fields and only configures
the max subcolumn limit.
@@ -70,6 +71,7 @@ public class VariantType extends PrimitiveType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
/**
@@ -84,6 +86,7 @@ public class VariantType extends PrimitiveType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
/**
@@ -100,7 +103,8 @@ public class VariantType extends PrimitiveType {
public VariantType(List<VariantField> fields, int
variantMaxSubcolumnsCount,
boolean enableTypedPathsToSparse, int
variantMaxSparseColumnStatisticsSize,
int variantSparseHashShardCount, boolean enableVariantDocMode,
- long variantDocMaterializationMinRows, int variantDocShardCount) {
+ long variantDocMaterializationMinRows, int variantDocShardCount,
+ boolean enableNestedGroup) {
this.predefinedFields =
ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be
null"));
this.variantMaxSubcolumnsCount = variantMaxSubcolumnsCount;
this.enableTypedPathsToSparse = enableTypedPathsToSparse;
@@ -109,6 +113,7 @@ public class VariantType extends PrimitiveType {
this.enableVariantDocMode = enableVariantDocMode;
this.variantDocMaterializationMinRows =
variantDocMaterializationMinRows;
this.variantDocShardCount = variantDocShardCount;
+ this.enableNestedGroup = enableNestedGroup;
}
@Override
@@ -117,7 +122,7 @@ public class VariantType extends PrimitiveType {
.collect(Collectors.toList()),
variantMaxSubcolumnsCount, enableTypedPathsToSparse,
variantMaxSparseColumnStatisticsSize,
variantSparseHashShardCount,
enableVariantDocMode,
variantDocMaterializationMinRows,
- variantDocShardCount);
+ variantDocShardCount, enableNestedGroup);
}
@Override
@@ -126,7 +131,7 @@ public class VariantType extends PrimitiveType {
.map(VariantField::toCatalogDataType)
.collect(Collectors.toCollection(ArrayList::new)),
variantMaxSubcolumnsCount, enableTypedPathsToSparse,
variantMaxSparseColumnStatisticsSize,
variantSparseHashShardCount, enableVariantDocMode,
- variantDocMaterializationMinRows, variantDocShardCount);
+ variantDocMaterializationMinRows, variantDocShardCount,
enableNestedGroup);
return type;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 362b69533d2..f92d4566f34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -952,6 +952,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String DEFAULT_VARIANT_DOC_HASH_SHARD_COUNT =
"default_variant_doc_hash_shard_count";
+ public static final String DEFAULT_VARIANT_ENABLE_NESTED_GROUP =
"default_variant_enable_nested_group";
+
public static final String MULTI_DISTINCT_STRATEGY =
"multi_distinct_strategy";
public static final String AGG_PHASE = "agg_phase";
@@ -3443,6 +3445,13 @@ public class SessionVariable implements Serializable,
Writable {
)
public int defaultVariantDocHashShardCount = 64;
+ @VariableMgr.VarAttr(
+ name = DEFAULT_VARIANT_ENABLE_NESTED_GROUP,
+ needForward = true,
+ fuzzy = true
+ )
+ public boolean defaultVariantEnableNestedGroup = false;
+
@VariableMgr.VarAttr(
name = "use_v3_storage_format",
fuzzy = true,
@@ -6153,6 +6162,10 @@ public class SessionVariable implements Serializable,
Writable {
return defaultVariantDocHashShardCount;
}
+ public boolean getDefaultVariantEnableNestedGroup() {
+ return defaultVariantEnableNestedGroup;
+ }
+
public void readAffectQueryResultVariables(BiConsumer<String, Object>
variablesReader) {
for (Field affectQueryResultField : affectQueryResultFields) {
String name = affectQueryResultField.getName();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 5470c83222f..df1a6bd6a10 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -402,7 +402,8 @@ public abstract class ExternalFileTableValuedFunction
extends TableValuedFunctio
/*variantSparseHashShardCount*/ 0,
/*variantEnableDocMode*/ false,
/*variantDocMaterializationMinRows*/ 0,
- /*variantDocShardCount*/ 0);
+ /*variantDocShardCount*/ 0,
+ /*enableNestedGroup*/ false);
parsedNodes = 1;
} else {
type =
ScalarType.createType(PrimitiveType.fromThrift(tPrimitiveType),
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index ff54f54aed7..aadacd37879 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -390,6 +390,7 @@ message ColumnPB {
optional int64 variant_doc_materialization_min_rows = 32;
// Number of buckets used to store doc map in variant doc mode.
optional int32 variant_doc_hash_shard_count = 33 [default = 64];
+ optional bool variant_enable_nested_group = 34 [default = false];
}
// Dictionary of Schema info, to reduce TabletSchemaCloudPB fdb kv size
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index 6f71a29f72a..eb12834fa18 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -99,6 +99,7 @@ struct TColumn {
26: optional bool variant_enable_doc_mode
27: optional i64 variant_doc_materialization_min_rows
28: optional i32 variant_doc_hash_shard_count
+ 29: optional bool variant_enable_nested_group
}
struct TSlotDescriptor {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
