This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new bf67db243de branch-3.0: [fix](auto bucket) Configurable parameters for
partition size estimation and number of buckets in auto bucket #50566 (#51230)
bf67db243de is described below
commit bf67db243de46c4153f147129d3f65b973e88d23
Author: deardeng <[email protected]>
AuthorDate: Thu May 29 09:55:08 2025 +0800
branch-3.0: [fix](auto bucket) Configurable parameters for partition size
estimation and number of buckets in auto bucket #50566 (#51230)
cherry pick from #50566
---
.../main/java/org/apache/doris/common/Config.java | 11 +++++++++++
.../apache/doris/common/util/AutoBucketUtils.java | 18 ++++++++++++++----
.../doris/catalog/DynamicPartitionTableTest.java | 4 ++--
.../doris/common/util/AutoBucketUtilsTest.java | 20 +++++++++++++++++---
4 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 3cbbf183a1c..1833e3d009e 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2637,6 +2637,17 @@ public class Config extends ConfigBase {
})
public static int autobucket_max_buckets = 128;
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "Auto Buckets中按照partition size去估算bucket数，存算一体partition size
5G估算一个bucket，"
+ + "但存算分离下partition size 10G估算一个bucket。
若配置小于0，会在在代码中会自适应存算一体模式默认5G，在存算分离默认10G",
+ "In Auto Buckets, the number of buckets is estimated based on the
partition size. "
+ + "For storage and computing integration, a partition size of 5G
is estimated as one bucket."
+ + " but for cloud, a partition size of 10G is estimated as one
bucket. "
+ + "If the configuration is less than 0, the code will have an
adaptive non-cloud mode with a default of 5G,"
+ + " and in cloud mode with a default of 10G."
+ })
+ public static int autobucket_partition_size_per_bucket_gb = -1;
+
@ConfField(description = {"Arrow Flight
Server中所有用户token的缓存上限，超过后LRU淘汰，默认值为512, "
+ "并强制限制小于 qe_max_connection/2, 避免`Reach limit of connections`, "
+ "因为arrow flight sql是无状态的协议，连接通常不会主动断开，"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java
index 19c4c4bf369..c6b8e69feae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java
@@ -74,14 +74,14 @@ public class AutoBucketUtils {
ImmutableMap<String, DiskInfo> disks = backend.getDisks();
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE &&
diskInfo.hasPathHash()) {
- buckets += (diskInfo.getAvailableCapacityB() - 1) / (50 *
SIZE_1GB) + 1;
+ buckets += (int) ((diskInfo.getAvailableCapacityB() - 1) /
(50 * SIZE_1GB) + 1);
}
}
}
return buckets;
}
- private static int convertParitionSizeToBucketsNum(long partitionSize) {
+ private static int convertPartitionSizeToBucketsNum(long partitionSize) {
partitionSize /= 5; // for compression 5:1
// <= 100MB, 1 bucket
@@ -92,12 +92,22 @@ public class AutoBucketUtils {
} else if (partitionSize <= SIZE_1GB) {
return 2;
} else {
- return (int) ((partitionSize - 1) / SIZE_1GB + 1);
+ int partitionSizePerBucket =
Config.autobucket_partition_size_per_bucket_gb;
+ if (partitionSizePerBucket <= 0) {
+ if (Config.isCloudMode()) {
+ partitionSizePerBucket = 10;
+ } else {
+ partitionSizePerBucket = 5;
+ }
+ logger.debug("autobucket_partition_size_per_bucket_gb <= 0,
use adaptive {}",
+ partitionSizePerBucket);
+ }
+ return (int) ((partitionSize - 1) / (partitionSizePerBucket *
SIZE_1GB) + 1);
}
}
public static int getBucketsNum(long partitionSize) {
- int bucketsNumByPartitionSize =
convertParitionSizeToBucketsNum(partitionSize);
+ int bucketsNumByPartitionSize =
convertPartitionSizeToBucketsNum(partitionSize);
int bucketsNumByBE = Config.isCloudMode() ? Integer.MAX_VALUE :
getBucketsNumByBEDisks();
int bucketsNum = Math.min(Config.autobucket_max_buckets,
Math.min(bucketsNumByPartitionSize, bucketsNumByBE));
int beNum = getBENum();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 4217342133b..9495c048f4b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -1824,7 +1824,7 @@ public class DynamicPartitionTableTest {
partitions = Lists.newArrayList(table.getAllPartitions());
partitions.sort(Comparator.comparing(Partition::getId));
Assert.assertEquals(54, partitions.size());
- // 100GB total, 1GB per bucket, should 100 buckets.
- Assert.assertEquals(100, partitions.get(partitions.size() -
1).getDistributionInfo().getBucketNum());
+ // 100GB total, 5GB per bucket, should 20 buckets.
+ Assert.assertEquals(20, partitions.get(partitions.size() -
1).getDistributionInfo().getBucketNum());
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
index f03a4282d9a..bd3e7f9e2a5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
@@ -216,12 +216,14 @@ public class AutoBucketUtilsTest {
int bucketNum = getPartitionBucketNum(tableName);
Assert.assertEquals(FeConstants.default_bucket_num, bucketNum);
}
-
// Some of these tests will report
// java.lang.IllegalArgumentException: Value of type
org.apache.doris.catalog.
// Env incompatible with return type com.google.common.collect.
- // ImmutableMap of
org.apache.doris.system.SystemInfoService#getBackendsInCluster(String)
+ // ImmutableMap of
org.apache.doris.system.SystemInfoService#getAllBackendsByAllCluster(String)
// Occasional failure, so ignore these tests
+ // It works on Mac and development machine, but it reports an error on CI
pipeline. I don't know what it is,
+ // so @Ignore
+
@Ignore
@Test
public void test100MB(@Mocked Env env, @Mocked EditLog editLog, @Mocked
SystemInfoService systemInfoService)
@@ -309,6 +311,18 @@ public class AutoBucketUtilsTest {
long estimatePartitionSize = AutoBucketUtils.SIZE_1TB;
ImmutableMap<Long, Backend> backends = createBackends(200, 7, 4 *
AutoBucketUtils.SIZE_1TB);
expectations(env, editLog, systemInfoService, backends);
- Assert.assertEquals(200,
AutoBucketUtils.getBucketsNum(estimatePartitionSize));
+ Assert.assertEquals(128,
AutoBucketUtils.getBucketsNum(estimatePartitionSize));
+ }
+
+ @Ignore
+ @Test
+ public void test1T_1_In_Cloud(@Mocked Env env, @Mocked EditLog editLog,
@Mocked SystemInfoService systemInfoService)
+ throws Exception {
+ Config.autobucket_partition_size_per_bucket_gb = 5;
+ Config.cloud_unique_id = "cloud_mode";
+ long estimatePartitionSize = AutoBucketUtils.SIZE_1TB;
+ ImmutableMap<Long, Backend> backends = createBackends(10, 7, 4 *
AutoBucketUtils.SIZE_1TB);
+ expectations(env, editLog, systemInfoService, backends);
+ Assert.assertEquals(41,
AutoBucketUtils.getBucketsNum(estimatePartitionSize));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
