This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e4933bee9a707071ee6bf61fc9559a6d0b904c7c
Author: zhangdong <493738...@qq.com>
AuthorDate: Fri Oct 13 10:24:01 2023 +0800
[Enhance](resource group)db support replication_allocation (#25195)
- db support replication_allocation,when create table,if not set
`replication_num` or `replication_allocation `,will use it in db
- fix partition property will disappear when table partition is not null
---
docs/en/docs/admin-manual/multi-tenant.md | 39 ++++++++-
.../Alter/ALTER-DATABASE.md | 18 ++++
.../Create/CREATE-DATABASE.md | 8 ++
docs/zh-CN/docs/admin-manual/multi-tenant.md | 37 +++++++-
.../Alter/ALTER-DATABASE.md | 18 ++++
.../Create/CREATE-DATABASE.md | 8 ++
.../doris/analysis/AlterDatabasePropertyStmt.java | 3 -
.../org/apache/doris/analysis/CreateTableStmt.java | 49 ++++++++++-
.../apache/doris/analysis/SinglePartitionDesc.java | 10 ++-
.../java/org/apache/doris/catalog/Database.java | 47 +++++-----
.../apache/doris/common/util/PropertyAnalyzer.java | 13 +++
.../apache/doris/planner/ResourceTagQueryTest.java | 99 ++++++++++++++++++++++
12 files changed, 317 insertions(+), 32 deletions(-)
diff --git a/docs/en/docs/admin-manual/multi-tenant.md
b/docs/en/docs/admin-manual/multi-tenant.md
index 41663a1e4fc..d0402bbf445 100644
--- a/docs/en/docs/admin-manual/multi-tenant.md
+++ b/docs/en/docs/admin-manual/multi-tenant.md
@@ -115,7 +115,15 @@ Node resource division refers to setting tags for BE nodes
in a Doris cluster, a
│ │
└────────────────────────────────────────────────────┘
```
-
+
+ For the convenience of setting the data distribution strategy for tables, a
unified data distribution strategy can be set at the database level, but the
priority of setting tables is higher than that of the database
+
+ ```sql
+ CREATE DATABASE db_name PROPERTIES (
+ "replication_allocation" = "tag.location.group_a:1, tag.location.group_b:1"
+ )
+ ```
+
3. Use different resource groups for data query
After the execution of the first two steps is completed, we can limit a
user's query by setting the user's resource usage permissions, and can only use
the nodes in the specified resource group to execute.
@@ -187,7 +195,7 @@ Through memory and CPU resource limits. We can divide user
queries into more fin
## Best practices and forward compatibility
-Tag division and CPU limitation are new features in version 0.15. In order to
ensure a smooth upgrade from the old version, Doris has made the following
forward compatibility:
+### Tag division and CPU limitation are new features in version 0.15. In order
to ensure a smooth upgrade from the old version, Doris has made the following
forward compatibility:
1. Each BE node will have a default Tag: `"tag.location": "default"`.
2. The BE node added through the `alter system add backend` statement will
also set Tag: `"tag.location": "default"` by default.
@@ -232,3 +240,30 @@ Here we give an example of the steps to start using the
resource division functi
Through the above 4 steps, we can smoothly use the resource division function
after the original cluster is upgraded.
+
+### How to conveniently set replica distribution strategies when there are
many tables
+
+ For example, there is a db1 with four tables under it, and the replica
distribution strategy required for table1 is `group_a:1,group_b:2`, the replica
distribution strategy required for tables 2, 3, and 4 is `group_c:1,group_b:2`
+
+ Then you can use the following statement to create db1:
+
+ ```sql
+ CREATE DATABASE db1 PROPERTIES (
+ "replication_allocation" = "tag.location.group_a:1, tag.location.group_b:2"
+ )
+ ```
+
+ Create table1 using the following statement:
+
+ ```sql
+ CREATE TABLE table1
+ (k1 int, k2 int)
+ distributed by hash(k1) buckets 1
+ properties(
+ "replication_allocation"="tag.location.group_c:1, tag.location.group_b:2"
+ )
+ ```
+
+ The table creation statements for table2, table3, and table4 do not need to
specify `replication_allocation` again.
+
+ Note: Changing the replica distribution policy of the database will not
affect existing tables.
\ No newline at end of file
diff --git
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
index af2e0467e97..f1b361e3edf 100644
---
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
+++
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
@@ -56,6 +56,12 @@ illustrate:
After renaming the database, use the REVOKE and GRANT commands to modify
the appropriate user permissions, if necessary.
The default data quota for the database is 1024GB, and the default replica
quota is 1073741824.
+4) Modify the properties of an existing database
+
+```sql
+ALTER DATABASE db_name SET PROPERTIES ("key"="value", ...);
+```
+
### Example
1. Set the specified database data volume quota
@@ -82,6 +88,18 @@ ALTER DATABASE example_db RENAME example_db2;
ALTER DATABASE example_db SET REPLICA QUOTA 102400;
```
+4. Modify the default replica distribution policy for tables in db (this
operation only applies to newly created tables and will not modify existing
tables in db)
+
+```sql
+ALTER DATABASE example_db SET PROPERTIES("replication_allocation" =
"tag.location.default:2");
+```
+
+5. Cancel the default replica distribution policy for tables in db (this
operation only applies to newly created tables and will not modify existing
tables in db)
+
+```sql
+ALTER DATABASE example_db SET PROPERTIES("replication_allocation" = "");
+```
+
### Keywords
```text
diff --git
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
index 6243680e1cc..0305b197db8 100644
---
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
+++
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
@@ -59,6 +59,14 @@ CREATE DATABASE [IF NOT EXISTS] db_name
- `iceberg.hive.metastore.uris` : hive metastore service address;
- `iceberg.catalog.type`: The default is `HIVE_CATALOG`; currently only
`HIVE_CATALOG` is supported, and more Iceberg catalog types will be supported
in the future.
+- If you want to specify the default replica distribution for tables in db,
you need to specify `replication_allocation` (the `replication_allocation`
attribute of table will have higher priority than db)
+
+ ```sql
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:3"
+ )
+ ```
+
### Example
1. Create a new database db_test
diff --git a/docs/zh-CN/docs/admin-manual/multi-tenant.md
b/docs/zh-CN/docs/admin-manual/multi-tenant.md
index acc1775c0a6..a7ae8314f59 100644
--- a/docs/zh-CN/docs/admin-manual/multi-tenant.md
+++ b/docs/zh-CN/docs/admin-manual/multi-tenant.md
@@ -115,6 +115,14 @@ FE 不参与用户数据的处理计算等工作，因此是一个资源消耗
│ │
└────────────────────────────────────────────────────┘
```
+
+ 为了方便设置table的数据分布策略，可以在database层面设置统一的数据分布策略，但是table设置的优先级高于database
+
+ ```sql
+ CREATE DATABASE db_name PROPERTIES (
+ "replication_allocation" = "tag.location.group_a:1, tag.location.group_b:1"
+ )
+ ```
3. 使用不同资源组进行数据查询
@@ -186,7 +194,7 @@ FE 不参与用户数据的处理计算等工作，因此是一个资源消耗
## 最佳实践和向前兼容
-Tag 划分和 CPU 限制是 0.15 版本中的新功能。为了保证可以从老版本平滑升级，Doris 做了如下的向前兼容：
+### Tag 划分和 CPU 限制是 0.15 版本中的新功能。为了保证可以从老版本平滑升级，Doris 做了如下的向前兼容：
1. 每个 BE 节点会有一个默认的 Tag：`"tag.location": "default"`。
2. 通过 `alter system add backend` 语句新增的 BE 节点也会默认设置 Tag：`"tag.location":
"default"`。
@@ -230,3 +238,30 @@ Tag 划分和 CPU 限制是 0.15 版本中的新功能。为了保证可以从
等数据重分布完毕后。我们就可以开始设置用户的资源标签权限了。因为默认情况下，用户的 `resource_tags.location`
属性为空，即可以访问任意 Tag 的 BE。所以在前面步骤中，不会影响到已有用户的正常查询。当 `resource_tags.location`
属性非空时，用户将被限制访问指定 Tag 的 BE。
通过以上4步，我们可以较为平滑的在原有集群升级后，使用资源划分功能。
+
+### table数量很多时如何方便的设置副本分布策略
+
+ 比如有一个 db1,db1下有四个table，table1需要的副本分布策略为
`group_a:1,group_b:2`，table2，table3, table4需要的副本分布策略为 `group_c:1,group_b:2`
+
+ 那么可以使用如下语句创建db1：
+
+ ```sql
+ CREATE DATABASE db1 PROPERTIES (
+ "replication_allocation" = "tag.location.group_a:1, tag.location.group_b:2"
+ )
+ ```
+
+ 使用如下语句创建table1：
+
+ ```sql
+ CREATE TABLE table1
+ (k1 int, k2 int)
+ distributed by hash(k1) buckets 1
+ properties(
+ "replication_allocation"="tag.location.group_c:1, tag.location.group_b:2"
+ )
+ ```
+
+ table2，table3,table4的建表语句无需再指定`replication_allocation`。
+
+ 注意事项：更改database的副本分布策略不会对已有的table产生影响。
\ No newline at end of file
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
index 0555a45024f..90b2a93e0fb 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE.md
@@ -56,6 +56,12 @@ ALTER DATABASE db_name SET REPLICA QUOTA quota;
重命名数据库后，如需要，请使用 REVOKE 和 GRANT 命令修改相应的用户权限。
数据库的默认数据量配额为1024GB，默认副本数量配额为1073741824。
+4) 对已有 database 的 property 进行修改操作
+
+```sql
+ALTER DATABASE db_name SET PROPERTIES ("key"="value", ...);
+```
+
### Example
1. 设置指定数据库数据量配额
@@ -82,6 +88,18 @@ ALTER DATABASE example_db RENAME example_db2;
ALTER DATABASE example_db SET REPLICA QUOTA 102400;
```
+4. 修改db下table的默认副本分布策略（该操作仅对新建的table生效，不会修改db下已存在的table）
+
+```sql
+ALTER DATABASE example_db SET PROPERTIES("replication_allocation" =
"tag.location.default:2");
+```
+
+5. 取消db下table的默认副本分布策略（该操作仅对新建的table生效，不会修改db下已存在的table）
+
+```sql
+ALTER DATABASE example_db SET PROPERTIES("replication_allocation" = "");
+```
+
### Keywords
```text
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
index ecd65de9cb0..fafe4d5846f 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-DATABASE.md
@@ -59,6 +59,14 @@ CREATE DATABASE [IF NOT EXISTS] db_name
- `iceberg.hive.metastore.uris` ：hive metastore 服务地址;
- `iceberg.catalog.type`： 默认为 `HIVE_CATALOG`；当前仅支持 `HIVE_CATALOG`，后续会支持更多
Iceberg catalog 类型。
+-
如果要为db下的table指定默认的副本分布策略，需要指定`replication_allocation`（table的`replication_allocation`属性优先级会高于db）
+
+ ```sql
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default:3"
+ )
+ ```
+
### Example
1. 新建数据库 db_test
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
index 625309f1069..28d65652081 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
@@ -70,9 +70,6 @@ public class AlterDatabasePropertyStmt extends DdlStmt {
// clone properties for analyse
Map<String, String> analysisProperties = new HashMap<String,
String>(properties);
PropertyAnalyzer.analyzeBinlogConfig(analysisProperties);
- if (!analysisProperties.isEmpty()) {
- throw new UserException("Invalid property name or value: " +
analysisProperties);
- }
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 28ab8c97bc9..07ba5da9dd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.analysis.IndexDef.IndexType;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
@@ -40,6 +41,7 @@ import org.apache.doris.common.util.ParseUtil;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -49,6 +51,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -325,7 +328,7 @@ public class CreateTableStmt extends DdlStmt {
boolean enableDuplicateWithoutKeysByDefault = false;
if (properties != null) {
enableDuplicateWithoutKeysByDefault =
-
PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
+
PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
}
//pre-block creation with column type ALL
for (ColumnDef columnDef : columnDefs) {
@@ -623,9 +626,10 @@ public class CreateTableStmt extends DdlStmt {
}
}
- private Map<String, String> rewriteReplicaAllocationProperties(Map<String,
String> properties) {
+ private Map<String, String> rewriteReplicaAllocationProperties(Map<String,
String> properties)
+ throws AnalysisException {
if (Config.force_olap_table_replication_num <= 0) {
- return properties;
+ return rewriteReplicaAllocationPropertiesByDatabase(properties);
}
// if force_olap_table_replication_num is set, use this value to
rewrite the replication_num or
// replication_allocation properties
@@ -651,6 +655,45 @@ public class CreateTableStmt extends DdlStmt {
return newProperties;
}
+ private Map<String, String>
rewriteReplicaAllocationPropertiesByDatabase(Map<String, String> properties)
+ throws AnalysisException {
+ // if table contain `replication_allocation` or
`replication_allocation`,not need rewrite by db
+ if (properties != null &&
(properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)
+ ||
properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM))) {
+ return properties;
+ }
+ CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalogNullable(tableName.getCtl());
+ if (catalog == null) {
+ return properties;
+ }
+ DatabaseIf db = catalog.getDbNullable(tableName.getDb());
+ if (db == null) {
+ return properties;
+ }
+ // if db not have properties,not need rewrite
+ if (db.getDbProperties() == null) {
+ return properties;
+ }
+ Map<String, String> dbProperties =
db.getDbProperties().getProperties();
+ if (dbProperties == null) {
+ return properties;
+ }
+ if (properties == null) {
+ properties = Maps.newHashMap();
+ }
+ if
(dbProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)
&& StringUtils
+
.isNotEmpty(dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)))
{
+ properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
+
dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION));
+ }
+ if
(dbProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM) &&
StringUtils
+
.isNotEmpty(dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM))) {
+ properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
+
dbProperties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM));
+ }
+ return properties;
+ }
+
private void analyzeEngineName() throws AnalysisException {
if (Strings.isNullOrEmpty(engineName)) {
engineName = "olap";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
index c70ef3f56de..ac506628b1c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
@@ -29,6 +29,7 @@ import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Joiner;
import com.google.common.base.Joiner.MapJoiner;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import java.util.Map;
@@ -126,9 +127,16 @@ public class SinglePartitionDesc implements
AllPartitionDesc {
partitionKeyDesc.analyze(partColNum);
+ Map<String, String> mergedMap = Maps.newHashMap();
+ // Should putAll `otherProperties` before `this.properties`,
+ // because the priority of partition is higher than table
if (otherProperties != null) {
- this.properties = otherProperties;
+ mergedMap.putAll(otherProperties);
}
+ if (this.properties != null) {
+ mergedMap.putAll(this.properties);
+ }
+ this.properties = mergedMap;
// analyze data property
partitionDataProperty =
PropertyAnalyzer.analyzeDataProperty(properties,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 92100a98987..14e72edf767 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -28,6 +28,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.persist.CreateTableInfo;
import org.apache.doris.persist.gson.GsonUtils;
@@ -858,33 +859,35 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
public void replayUpdateDbProperties(Map<String, String> properties) {
dbProperties.updateProperties(properties);
- binlogConfig = dbProperties.getBinlogConfig();
+ if (PropertyAnalyzer.hasBinlogConfig(properties)) {
+ binlogConfig = dbProperties.getBinlogConfig();
+ }
}
public boolean updateDbProperties(Map<String, String> properties) throws
DdlException {
- BinlogConfig oldBinlogConfig = getBinlogConfig();
- BinlogConfig newBinlogConfig = BinlogConfig.fromProperties(properties);
- if (oldBinlogConfig.equals(newBinlogConfig)) {
- return false;
- }
-
- if (newBinlogConfig.isEnable() && !oldBinlogConfig.isEnable()) {
- // check all tables binlog enable is true
- for (Table table : idToTable.values()) {
- if (table.getType() != TableType.OLAP) {
- continue;
- }
+ if (PropertyAnalyzer.hasBinlogConfig(properties)) {
+ BinlogConfig oldBinlogConfig = getBinlogConfig();
+ BinlogConfig newBinlogConfig =
BinlogConfig.fromProperties(properties);
+
+ if (newBinlogConfig.isEnable() && !oldBinlogConfig.isEnable()) {
+ // check all tables binlog enable is true
+ for (Table table : idToTable.values()) {
+ if (table.getType() != TableType.OLAP) {
+ continue;
+ }
- OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- if (!olapTable.getBinlogConfig().isEnable()) {
- String errMsg = String.format("binlog is not enable in
table[%s] in db [%s]", table.getName(),
- getFullName());
- throw new DdlException(errMsg);
+ OlapTable olapTable = (OlapTable) table;
+ olapTable.readLock();
+ try {
+ if (!olapTable.getBinlogConfig().isEnable()) {
+ String errMsg = String
+ .format("binlog is not enable in table[%s]
in db [%s]", table.getName(),
+ getFullName());
+ throw new DdlException(errMsg);
+ }
+ } finally {
+ olapTable.readUnlock();
}
- } finally {
- olapTable.readUnlock();
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 6a9c4bada1b..04fb984ef4d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -899,6 +899,19 @@ public class PropertyAnalyzer {
return tagMap;
}
+ public static boolean hasBinlogConfig(Map<String, String> properties) {
+ if (properties == null || properties.isEmpty()) {
+ return false;
+ }
+
+ for (String key : properties.keySet()) {
+ if (key.startsWith(PROPERTIES_BINLOG_PREFIX)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static Map<String, String> analyzeBinlogConfig(Map<String, String>
properties) throws AnalysisException {
if (properties == null || properties.isEmpty()) {
return null;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
index ef4a58e5342..8bcd302a181 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.planner;
+import org.apache.doris.analysis.AlterDatabasePropertyStmt;
import org.apache.doris.analysis.AlterSystemStmt;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
@@ -32,6 +33,7 @@ import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
@@ -279,6 +281,103 @@ public class ResourceTagQueryTest {
List<List<String>> userProps =
Env.getCurrentEnv().getAuth().getUserProperties(Auth.ROOT_USER);
Assert.assertEquals(10, userProps.size());
+
+ // now :
+ // be1 be2 be3 ==>tag1;
+ // be4,be5 ==> default;
+ // root ==> zone1;
+
+ // create database
+ String createDbStmtStr
+ = "create database test_prop
PROPERTIES('replication_allocation' = 'tag.location.default:3');";
+ CreateDbStmt createDbStmt = (CreateDbStmt)
UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
+ Env.getCurrentEnv().createDb(createDbStmt);
+
+ // create table with default tag
+ String createTableStr2 = "create table test_prop.tbl2\n"
+ + "(k1 date, k2 int)\n"
+ + "distributed by hash(k2) buckets 2;";
+ // table will inherit db prop, only have 2 default be, so
`AnalysisException`
+ ExceptionChecker.expectThrows(AnalysisException.class, () ->
createTable(createTableStr2));
+ //alter db change `replication_allocation`
+ String alterDbStmtStr
+ = "alter database test_prop set
PROPERTIES('replication_allocation' = 'tag.location.default:2');";
+ AlterDatabasePropertyStmt alterDbStmt = (AlterDatabasePropertyStmt)
UtFrameUtils
+ .parseAndAnalyzeStmt(alterDbStmtStr, connectContext);
+ Env.getCurrentEnv().alterDatabaseProperty(alterDbStmt);
+ ExceptionChecker.expectThrowsNoException(() ->
createTable(createTableStr2));
+ Database propDb =
Env.getCurrentInternalCatalog().getDbNullable("default_cluster:test_prop");
+ OlapTable tbl2 = (OlapTable) propDb.getTableNullable("tbl2");
+ // should same with db
+ Map<Tag, Short> tbl2ExpectedAllocMap = Maps.newHashMap();
+ tbl2ExpectedAllocMap.put(Tag.DEFAULT_BACKEND_TAG, (short) 2);
+ for (Partition partition : tbl2.getPartitions()) {
+ ReplicaAllocation replicaAllocation =
tbl2.getPartitionInfo().getReplicaAllocation(partition.getId());
+ Map<Tag, Short> allocMap = replicaAllocation.getAllocMap();
+ Assert.assertEquals(tbl2ExpectedAllocMap, allocMap);
+ }
+ // can not query due root ==> zone1
+ queryStr = "explain select * from test_prop.tbl2";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
queryStr);
+ System.out.println(explainString);
+ Assert.assertTrue(explainString.contains("have no queryable
replicas"));
+
+ // The priority of table is higher than db,should same with table
+ String createTableStr3 = "create table test_prop.tbl3\n"
+ + "(k1 date, k2 int)\n"
+ + "distributed by hash(k2) buckets 2
PROPERTIES('replication_allocation' = 'tag.location.zone1:3');";
+ ExceptionChecker.expectThrowsNoException(() ->
createTable(createTableStr3));
+ OlapTable tbl3 = (OlapTable) propDb.getTableNullable("tbl3");
+ Map<Tag, Short> tbl3ExpectedAllocMap = Maps.newHashMap();
+ tbl3ExpectedAllocMap.put(tag1, (short) 3);
+ for (Partition partition : tbl3.getPartitions()) {
+ ReplicaAllocation replicaAllocation =
tbl3.getPartitionInfo().getReplicaAllocation(partition.getId());
+ Map<Tag, Short> allocMap = replicaAllocation.getAllocMap();
+ Assert.assertEquals(tbl3ExpectedAllocMap, allocMap);
+ }
+ // can still query
+ queryStr = "explain select * from test_prop.tbl3";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext,
queryStr);
+ System.out.println(explainString);
+ Assert.assertTrue(explainString.contains("tablets=2/2"));
+ //alter db change `replication_allocation` to null
+ alterDbStmtStr = "alter database test_prop set
PROPERTIES('replication_allocation' = '');";
+ alterDbStmt = (AlterDatabasePropertyStmt) UtFrameUtils
+ .parseAndAnalyzeStmt(alterDbStmtStr, connectContext);
+ Env.getCurrentEnv().alterDatabaseProperty(alterDbStmt);
+ // create table with default tag
+ String createTableStr4 = "create table test_prop.tbl4\n"
+ + "(k1 date, k2 int)\n"
+ + "distributed by hash(k2) buckets 2;";
+ // only have 2 default tag be, default need 3, so error
+ ExceptionChecker.expectThrows(DdlException.class, () ->
createTable(createTableStr4));
+
+ // now db not set `PROPERTIES`
+ // The priority of partition is higher than table
+ String createStr5 = "create table test_prop.tbl5\n"
+ + "(k1 date, k2 int)\n"
+ + "partition by range(k1)\n"
+ + "(\n"
+ + " partition p1 values less than(\"2021-06-01\")
('replication_allocation' = 'tag.location.zone1:1'),\n"
+ + " partition p2 values less than(\"2021-08-01\")\n"
+ + ")\n"
+ + "distributed by hash(k2) buckets 2 "
+ + "PROPERTIES('replication_allocation' =
'tag.location.default:1');";
+ ExceptionChecker.expectThrowsNoException(() ->
createTable(createStr5));
+ OlapTable tbl5 = (OlapTable) propDb.getTableNullable("tbl5");
+ Map<Tag, Short> p1ExpectedAllocMap = Maps.newHashMap();
+ p1ExpectedAllocMap.put(tag1, (short) 1);
+ Map<Tag, Short> p2ExpectedAllocMap = Maps.newHashMap();
+ p2ExpectedAllocMap.put(Tag.DEFAULT_BACKEND_TAG, (short) 1);
+ for (Partition partition : tbl5.getPartitions()) {
+ ReplicaAllocation replicaAllocation =
tbl5.getPartitionInfo().getReplicaAllocation(partition.getId());
+ Map<Tag, Short> allocMap = replicaAllocation.getAllocMap();
+ if (partition.getName().equals("p1")) {
+ Assert.assertEquals(p1ExpectedAllocMap, allocMap);
+ } else {
+ Assert.assertEquals(p2ExpectedAllocMap, allocMap);
+ }
+ }
}
private void checkTableReplicaAllocation(OlapTable tbl) throws
InterruptedException {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
