This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6f569f8d585 [Exec](meta) Support set propertiy dict_page_size by
create table (#52243)
6f569f8d585 is described below
commit 6f569f8d585e51f40070dd726e635834b9d49170
Author: HappenLee <[email protected]>
AuthorDate: Fri Jul 11 19:20:12 2025 +0800
[Exec](meta) Support set propertiy dict_page_size by create table (#52243)
Now we can set dict page size in create tableļ¼
```
CREATE TABLE `customer_bak` (
`c_custkey` int NOT NULL,
`c_name` varchar(26) NOT NULL,
`c_address` varchar(41) NOT NULL,
`c_city` varchar(11) NOT NULL,
`c_nation` varchar(16) NOT NULL,
`c_region` varchar(13) NOT NULL,
`c_phone` varchar(16) NOT NULL,
`c_mktsegment` varchar(11) NOT NULL
) ENGINE = OLAP DUPLICATE KEY(`c_custkey`) COMMENT 'OLAP' DISTRIBUTED BY
HASH(`c_custkey`) BUCKETS 12 PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"storage_dict_page_size" = "148576",
"storage_page_size" = "148576", "min_load_replica_num" = "-1",
"is_being_synced" = "false", "colocate_with" = "groupa2",
"storage_medium" = "hdd", "storage_format" = "V2",
"inverted_index_storage_format" = "V2",
"light_schema_change" = "true", "disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000",
"group_commit_data_bytes" = "134217728"
);
```
some case if dict page size too bigger cause too mush memory, some case
if dict page too small, the query performance not good. so we need set
the dict page size by user self.
---
be/src/cloud/pb_convert.cpp | 4 ++
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 6 ++-
be/src/olap/rowset/segment_v2/column_writer.cpp | 1 +
be/src/olap/rowset/segment_v2/column_writer.h | 5 ++-
be/src/olap/rowset/segment_v2/options.h | 6 +--
be/src/olap/rowset/segment_v2/segment_writer.cpp | 1 +
.../rowset/segment_v2/vertical_segment_writer.cpp | 1 +
be/src/olap/tablet_meta.cpp | 3 ++
be/src/olap/tablet_schema.cpp | 4 ++
be/src/olap/tablet_schema.h | 5 +++
.../org/apache/doris/alter/CloudRollupJobV2.java | 3 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../analysis/ModifyTablePropertiesClause.java | 5 ++-
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 6 +++
.../java/org/apache/doris/catalog/OlapTable.java | 14 ++++++
.../org/apache/doris/catalog/TableProperty.java | 14 ++++++
.../cloud/datasource/CloudInternalCatalog.java | 6 ++-
.../apache/doris/common/util/PropertyAnalyzer.java | 21 +++++++++
.../apache/doris/datasource/InternalCatalog.java | 15 ++++++-
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../commands/info/ModifyTablePropertiesOp.java | 5 ++-
.../org/apache/doris/task/CreateReplicaTask.java | 6 ++-
.../apache/doris/common/PropertyAnalyzerTest.java | 48 +++++++++++++++++++++
.../java/org/apache/doris/task/AgentTaskTest.java | 5 ++-
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 1 +
.../data/query_p0/system/test_table_properties.out | Bin 15733 -> 16201 bytes
30 files changed, 180 insertions(+), 22 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index fffba6a8c0a..aaff6842760 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -336,6 +336,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
const TabletSchemaPB
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}
@@ -367,6 +368,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
TabletSchemaPB&& in)
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}
@@ -411,6 +413,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
const TabletSchemaCloudPB
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}
@@ -443,6 +446,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
TabletSchemaCloudPB&& in)
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
+ out->set_storage_dict_page_size(in.storage_dict_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index 83ac882141d..bf526bad7e4 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -51,8 +51,10 @@ Status BinaryDictPageBuilder::init() {
&data_page_builder_ptr, _options));
_data_page_builder.reset(data_page_builder_ptr);
PageBuilderOptions dict_builder_options;
- dict_builder_options.data_page_size =
- std::min(_options.data_page_size, _options.dict_page_size);
+ // here the binary plain page is used to store the dictionary items so
+ // the data page size is set to the same as the dict page size
+ dict_builder_options.data_page_size = _options.dict_page_size;
+ dict_builder_options.dict_page_size = _options.dict_page_size;
dict_builder_options.is_dict_page = true;
PageBuilder* dict_builder_ptr = nullptr;
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp
b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 13f21fc45d3..6ea2a4fe54a 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -424,6 +424,7 @@ Status ScalarColumnWriter::init() {
// create page builder
PageBuilderOptions opts;
opts.data_page_size = _opts.data_page_size;
+ opts.dict_page_size = _opts.dict_page_size;
RETURN_IF_ERROR(_encoding_info->create_page_builder(opts, &page_builder));
if (page_builder == nullptr) {
return Status::NotSupported("Failed to create page builder for type {}
and encoding {}",
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h
b/be/src/olap/rowset/segment_v2/column_writer.h
index 52939055864..072e5e2bb0b 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -53,7 +53,8 @@ struct ColumnWriterOptions {
// - input:
column_id/unique_id/type/length/encoding/compression/is_nullable members
// - output: encoding/indexes/dict_page members
ColumnMetaPB* meta = nullptr;
- size_t data_page_size = 64 * 1024;
+ size_t data_page_size = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ size_t dict_page_size = STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
// store compressed page only when space saving is above the threshold.
// space saving = 1 - compressed_size / uncompressed_size
double compression_min_space_saving = 0.1;
@@ -71,7 +72,7 @@ struct ColumnWriterOptions {
std::string to_string() const {
std::stringstream ss;
ss << std::boolalpha << "meta=" << meta->DebugString()
- << ", data_page_size=" << data_page_size
+ << ", data_page_size=" << data_page_size << ", dict_page_size=" <<
dict_page_size
<< ", compression_min_space_saving = " <<
compression_min_space_saving
<< ", need_zone_map=" << need_zone_map << ", need_bitmap_index=" <<
need_bitmap_index
<< ", need_bloom_filter" << need_bloom_filter;
diff --git a/be/src/olap/rowset/segment_v2/options.h
b/be/src/olap/rowset/segment_v2/options.h
index 164a8a27fc1..ad8c4d90448 100644
--- a/be/src/olap/rowset/segment_v2/options.h
+++ b/be/src/olap/rowset/segment_v2/options.h
@@ -22,15 +22,15 @@
namespace doris {
namespace segment_v2 {
-static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M
static constexpr size_t STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536;
+static constexpr size_t STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE = 256 * 1024l;
constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row store
page size: 16KB
struct PageBuilderOptions {
- size_t data_page_size = DEFAULT_PAGE_SIZE;
+ size_t data_page_size = STORAGE_PAGE_SIZE_DEFAULT_VALUE;
- size_t dict_page_size = DEFAULT_PAGE_SIZE;
+ size_t dict_page_size = STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
bool need_check_bitmap = true;
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 4e442975cc6..3e9576e229d 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -253,6 +253,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid,
const TabletColumn& co
if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
opts.data_page_size = storage_page_size;
}
+ opts.dict_page_size = _tablet_schema->storage_dict_page_size();
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int64_t>(
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 9eab36bd2d2..5fa9a34d5eb 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -253,6 +253,7 @@ Status
VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
if (storage_page_size >= 4096 && storage_page_size <= 10485760) {
opts.data_page_size = storage_page_size;
}
+ opts.dict_page_size = _tablet_schema->storage_dict_page_size();
DBUG_EXECUTE_IF("VerticalSegmentWriter._create_column_writer.storage_page_size",
{
auto table_id =
DebugPoints::instance()->get_debug_param_or_default<int64_t>(
"VerticalSegmentWriter._create_column_writer.storage_page_size", "table_id",
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index cd507245772..1a0e21c04db 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -345,6 +345,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
if (tablet_schema.__isset.storage_page_size) {
schema->set_storage_page_size(tablet_schema.storage_page_size);
}
+ if (tablet_schema.__isset.storage_dict_page_size) {
+
schema->set_storage_dict_page_size(tablet_schema.storage_dict_page_size);
+ }
if (tablet_schema.__isset.skip_write_index_on_load) {
schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 8390c230865..1ec1d3942a9 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1078,6 +1078,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_compression_type = schema.compression_type();
_row_store_page_size = schema.row_store_page_size();
_storage_page_size = schema.storage_page_size();
+ _storage_dict_page_size = schema.storage_dict_page_size();
_schema_version = schema.schema_version();
// Default to V1 inverted index storage format for backward compatibility
if not specified in schema.
if (!schema.has_inverted_index_storage_format()) {
@@ -1154,6 +1155,7 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
_sort_col_num = ori_tablet_schema.sort_col_num();
_row_store_page_size = ori_tablet_schema.row_store_page_size();
_storage_page_size = ori_tablet_schema.storage_page_size();
+ _storage_dict_page_size = ori_tablet_schema.storage_dict_page_size();
_enable_variant_flatten_nested =
ori_tablet_schema.variant_flatten_nested();
// copy from table_schema_param
@@ -1319,6 +1321,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
tablet_schema_pb->set_compression_type(_compression_type);
tablet_schema_pb->set_row_store_page_size(_row_store_page_size);
tablet_schema_pb->set_storage_page_size(_storage_page_size);
+ tablet_schema_pb->set_storage_dict_page_size(_storage_dict_page_size);
tablet_schema_pb->set_version_col_idx(_version_col_idx);
tablet_schema_pb->set_skip_bitmap_col_idx(_skip_bitmap_col_idx);
tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format);
@@ -1584,6 +1587,7 @@ bool operator==(const TabletSchema& a, const
TabletSchema& b) {
if (a._store_row_column != b._store_row_column) return false;
if (a._row_store_page_size != b._row_store_page_size) return false;
if (a._storage_page_size != b._storage_page_size) return false;
+ if (a._storage_dict_page_size != b._storage_dict_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return
false;
if (a._enable_variant_flatten_nested != b._enable_variant_flatten_nested)
return false;
return true;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 85feb62d51a..f2d0de0691d 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -397,6 +397,10 @@ public:
long row_store_page_size() const { return _row_store_page_size; }
void set_storage_page_size(long storage_page_size) { _storage_page_size =
storage_page_size; }
long storage_page_size() const { return _storage_page_size; }
+ void set_storage_dict_page_size(long storage_dict_page_size) {
+ _storage_dict_page_size = storage_dict_page_size;
+ }
+ long storage_dict_page_size() const { return _storage_dict_page_size; }
bool has_global_row_id() const {
for (auto [col_name, _] : _field_name_to_index) {
if
(col_name.start_with(StringRef(BeConsts::GLOBAL_ROWID_COL.data(),
@@ -592,6 +596,7 @@ private:
segment_v2::CompressionTypePB _compression_type =
segment_v2::CompressionTypePB::LZ4F;
long _row_store_page_size = segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
long _storage_page_size = segment_v2::STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ long _storage_dict_page_size =
segment_v2::STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
size_t _next_column_unique_id = 0;
std::string _auto_increment_column;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index 894528177e8..75aaac17c1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -233,7 +233,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
null,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(), null,
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index b57fe1609d6..1d54573d2ac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -254,7 +254,8 @@ public class CloudSchemaChangeJobV2 extends
SchemaChangeJobV2 {
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
clusterKeyUids,
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index f7762ea6dd5..facf3d25493 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -285,7 +285,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
objectPool,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 1472ad218a6..b1d85e28f08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -322,7 +322,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
objectPool,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index f2f9a2ac9e4..b8dc8c92e9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -376,8 +376,9 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN))
{
throw new AnalysisException("You can not modify property
'enable_unique_key_skip_bitmap_column'.");
- } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
- throw new AnalysisException("You can not modify
storage_page_size");
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)
+ ||
properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+ throw new AnalysisException("You can not modify
storage_page_size|storage_dict_page_size");
} else {
throw new AnalysisException("Unknown table property: " +
properties.keySet());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 216f38b3263..5c30627e3e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1417,7 +1417,8 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
objectPool,
localTbl.rowStorePageSize(),
localTbl.variantEnableFlattenNested(),
- localTbl.storagePageSize());
+ localTbl.storagePageSize(),
+ localTbl.storageDictPageSize());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
if (baseTabletRef != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 119910a49e5..52b45f58396 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3806,6 +3806,12 @@ public class Env {
sb.append(olapTable.storagePageSize()).append("\"");
}
+ // storage dict page size
+ if (olapTable.storageDictPageSize() !=
PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE) {
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE).append("\"
= \"");
+ sb.append(olapTable.storageDictPageSize()).append("\"");
+ }
+
// skip inverted index on load
if (olapTable.skipWriteIndexOnLoad()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).append("\"
= \"");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index e9073798149..bbc8fbe231f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2815,6 +2815,20 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
}
+ public void setStorageDictPageSize(long storageDictPageSize) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
+ Long.valueOf(storageDictPageSize).toString());
+ tableProperty.buildStorageDictPageSize();
+ }
+
+ public long storageDictPageSize() {
+ if (tableProperty != null) {
+ return tableProperty.storageDictPageSize();
+ }
+ return PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+ }
+
public void setStorageFormat(TStorageFormat storageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT,
storageFormat.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index d8dd9e10402..a74e81ce939 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -103,6 +103,8 @@ public class TableProperty implements GsonPostProcessable {
private long storagePageSize =
PropertyAnalyzer.STORAGE_PAGE_SIZE_DEFAULT_VALUE;
+ private long storageDictPageSize =
PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+
private String compactionPolicy =
PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY;
private long timeSeriesCompactionGoalSizeMbytes
@@ -331,6 +333,17 @@ public class TableProperty implements GsonPostProcessable {
return storagePageSize;
}
+ public TableProperty buildStorageDictPageSize() {
+ storageDictPageSize = Long.parseLong(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
+
Long.toString(PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE)));
+ return this;
+ }
+
+ public long storageDictPageSize() {
+ return storageDictPageSize;
+ }
+
public TableProperty buildSkipWriteIndexOnLoad() {
skipWriteIndexOnLoad = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
"false"));
@@ -724,6 +737,7 @@ public class TableProperty implements GsonPostProcessable {
buildRowStoreColumns();
buildRowStorePageSize();
buildStoragePageSize();
+ buildStorageDictPageSize();
buildSkipWriteIndexOnLoad();
buildCompactionPolicy();
buildTimeSeriesCompactionGoalSizeMbytes();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 0353856d5e0..6ef644fd59e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -182,7 +182,8 @@ public class CloudInternalCatalog extends InternalCatalog {
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(), clusterKeyUids,
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
requestBuilder.addTabletMetas(builder);
}
requestBuilder.setDbId(dbId);
@@ -215,7 +216,7 @@ public class CloudInternalCatalog extends InternalCatalog {
List<Integer> rowStoreColumnUniqueIds,
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat,
long pageSize,
boolean variantEnableFlattenNested, List<Integer> clusterKeyUids,
- long storagePageSize) throws DdlException {
+ long storagePageSize, long storageDictPageSize) throws
DdlException {
OlapFile.TabletMetaCloudPB.Builder builder =
OlapFile.TabletMetaCloudPB.newBuilder();
builder.setTableId(tableId);
builder.setIndexId(indexId);
@@ -358,6 +359,7 @@ public class CloudInternalCatalog extends InternalCatalog {
}
schemaBuilder.setRowStorePageSize(pageSize);
schemaBuilder.setStoragePageSize(storagePageSize);
+ schemaBuilder.setStorageDictPageSize(storageDictPageSize);
schemaBuilder.setEnableVariantFlattenNested(variantEnableFlattenNested);
if (!CollectionUtils.isEmpty(clusterKeyUids)) {
schemaBuilder.addAllClusterKeyUids(clusterKeyUids);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 01b230457ec..0c08f4cfa42 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -106,6 +106,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_STORAGE_PAGE_SIZE =
"storage_page_size";
public static final long STORAGE_PAGE_SIZE_DEFAULT_VALUE = 65536L;
+ public static final String PROPERTIES_STORAGE_DICT_PAGE_SIZE =
"storage_dict_page_size";
+ public static final long STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE = 262144L;
+
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE =
"distribution_type";
@@ -1113,6 +1116,24 @@ public class PropertyAnalyzer {
return storagePageSize;
}
+ public static long analyzeStorageDictPageSize(Map<String, String>
properties) throws AnalysisException {
+ long storageDictPageSize = STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+ if (properties != null &&
properties.containsKey(PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+ String storageDictPageSizeStr =
properties.get(PROPERTIES_STORAGE_DICT_PAGE_SIZE);
+ try {
+ storageDictPageSize = Long.parseLong(storageDictPageSizeStr);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid storage dict page size: "
+ storageDictPageSizeStr);
+ }
+ if (storageDictPageSize < 0 || storageDictPageSize > 104857600) {
+ throw new AnalysisException("Storage dict page size must be
between 0 and 100MB.");
+ }
+ storageDictPageSize = alignTo4K(storageDictPageSize);
+ properties.remove(PROPERTIES_STORAGE_DICT_PAGE_SIZE);
+ }
+ return storageDictPageSize;
+ }
+
// analyzeStorageFormat will parse the storage format from properties
// sql: alter table tablet_name set ("storage_format" = "v2")
// Use this sql to convert all tablets(base and rollup index) to a new
format segment
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 98b385e4f2e..223bc72c6fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1465,6 +1465,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE,
Long.toString(olapTable.storagePageSize()));
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
+ Long.toString(olapTable.storageDictPageSize()));
+ }
if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD))
{
properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
olapTable.skipWriteIndexOnLoad().toString());
@@ -1995,7 +1999,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool, tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize());
+ tbl.storagePageSize(),
+ tbl.storageDictPageSize());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
@@ -2505,6 +2510,14 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
olapTable.setStoragePageSize(storagePageSize);
+ long storageDictPageSize =
PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE;
+ try {
+ storageDictPageSize =
PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ olapTable.setStorageDictPageSize(storageDictPageSize);
+
// check data sort properties
int keyColumnSize =
CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames()) ?
keysDesc.keysColumnSize() :
keysDesc.getClusterKeysColumnNames().size();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 54b7cd1582e..f290d717446 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -1081,7 +1081,8 @@ public class ReportHandler extends Daemon {
objectPool,
olapTable.rowStorePageSize(),
olapTable.variantEnableFlattenNested(),
- olapTable.storagePageSize());
+ olapTable.storagePageSize(),
+ olapTable.storageDictPageSize());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
.getInvertedIndexFileStorageFormat());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
index 931aef61941..52aa6aa6594 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
@@ -369,8 +369,9 @@ public class ModifyTablePropertiesOp extends AlterTableOp {
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN))
{
throw new AnalysisException("You can not modify property
'enable_unique_key_skip_bitmap_column'.");
- } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
- throw new AnalysisException("You can not modify
storage_page_size");
+ } else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)
+ ||
properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE)) {
+ throw new AnalysisException("You can not modify
storage_page_size|storage_dict_page_size property.");
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM)) {
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 17c99435879..36e84be5942 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -67,6 +67,7 @@ public class CreateReplicaTask extends AgentTask {
private TCompressionType compressionType;
private long rowStorePageSize;
private long storagePageSize;
+ private long storageDictPageSize;
private List<Column> columns;
@@ -158,7 +159,8 @@ public class CreateReplicaTask extends AgentTask {
Map<Object, Object> objectPool,
long rowStorePageSize,
boolean variantEnableFlattenNested,
- long storagePageSize) {
+ long storagePageSize,
+ long storageDictPageSize) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
this.replicaId = replicaId;
@@ -207,6 +209,7 @@ public class CreateReplicaTask extends AgentTask {
this.rowStorePageSize = rowStorePageSize;
this.variantEnableFlattenNested = variantEnableFlattenNested;
this.storagePageSize = storagePageSize;
+ this.storageDictPageSize = storageDictPageSize;
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -369,6 +372,7 @@ public class CreateReplicaTask extends AgentTask {
tSchema.setStoreRowColumn(storeRowColumn);
tSchema.setRowStorePageSize(rowStorePageSize);
tSchema.setStoragePageSize(storagePageSize);
+ tSchema.setStorageDictPageSize(storageDictPageSize);
createTabletReq.setTabletSchema(tSchema);
createTabletReq.setVersion(version);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 6d708aa0826..ab7291eaf16 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -191,6 +191,54 @@ public class PropertyAnalyzerTest {
Assert.assertEquals(Tag.DEFAULT_BACKEND_TAG.value,
tagMap.get(Tag.TYPE_LOCATION));
}
+ @Test
+ public void testStorageDictPageSize() throws AnalysisException {
+ Map<String, String> properties = Maps.newHashMap();
+
+ // Test default value
+
Assert.assertEquals(PropertyAnalyzer.STORAGE_DICT_PAGE_SIZE_DEFAULT_VALUE,
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test valid value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"8192"); // 8KB
+ Assert.assertEquals(8192,
PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test lower boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"4096"); // 4KB
+ Assert.assertEquals(4096,
PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test upper boundary value
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"10485760"); // 10MB
+ Assert.assertEquals(10485760,
PropertyAnalyzer.analyzeStorageDictPageSize(properties));
+
+ // Test invalid number format
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"invalid");
+ try {
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid storage dict
page size"));
+ }
+
+ // Test value below minimum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"-1024"); // 1KB
+ try {
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage dict page size
must be between 0 and 100MB"));
+ }
+
+ // Test value above maximum limit
+ properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_DICT_PAGE_SIZE,
"209715200"); // 200MB
+ try {
+ PropertyAnalyzer.analyzeStorageDictPageSize(properties);
+ Assert.fail("Expected an AnalysisException to be thrown");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("Storage dict page size
must be between 0 and 100MB"));
+ }
+ }
+
@Test
public void testStoragePageSize() throws AnalysisException {
Map<String, String> properties = Maps.newHashMap();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index ce3d07c85d7..55c14a658e6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -76,6 +76,9 @@ public class AgentTaskTest {
private TStorageType storageType = TStorageType.COLUMN;
private long rowStorePageSize = 16384L;
private long storagePageSize = 65536L;
+
+ private long storageDictPageSize = 262144L;
+
private List<Column> columns;
private MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(3);
@@ -112,7 +115,7 @@ public class AgentTaskTest {
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false,
- storagePageSize);
+ storagePageSize, storageDictPageSize);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 494f80224f6..787e851835b 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -425,6 +425,7 @@ message TabletSchemaPB {
optional bool enable_variant_flatten_nested = 28 [default=false];
optional int32 skip_bitmap_col_idx = 29 [default = -1];
optional int64 storage_page_size = 30 [default=65536];
+ optional int64 storage_dict_page_size = 31 [default=262144];
}
message TabletSchemaCloudPB {
@@ -458,6 +459,7 @@ message TabletSchemaCloudPB {
optional bool enable_variant_flatten_nested = 29 [default=false];
optional int32 skip_bitmap_col_idx = 30 [default = -1];
optional int64 storage_page_size = 31 [default=65536];
+ optional int64 storage_dict_page_size = 32 [default=262144];
optional bool is_dynamic_schema = 100 [default=false];
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 808f645fd68..f807c0c89e7 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -50,6 +50,7 @@ struct TTabletSchema {
21: optional i64 row_store_page_size = 16384
22: optional bool variant_enable_flatten_nested = false
23: optional i64 storage_page_size = 65536
+ 24: optional i64 storage_dict_page_size = 262144
}
// this enum stands for different storage format in src_backends
diff --git a/regression-test/data/query_p0/system/test_table_properties.out
b/regression-test/data/query_p0/system/test_table_properties.out
index 70413bc4864..b4981be72a8 100644
Binary files a/regression-test/data/query_p0/system/test_table_properties.out
and b/regression-test/data/query_p0/system/test_table_properties.out differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
