This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new fbf6c081eee branch-3.0: [feature](vault) Support database level
storage vault #45325 (#49305)
fbf6c081eee is described below
commit fbf6c081eee472d084a1db0e3efebd9d1a47c5b4
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Mar 22 17:01:36 2025 +0800
branch-3.0: [feature](vault) Support database level storage vault #45325
(#49305)
Cherry-picked from #45325
Co-authored-by: Ganlin Zhao <36554565+glzha...@users.noreply.github.com>
---
.../java/org/apache/doris/catalog/Database.java | 24 ++
.../apache/doris/common/util/PropertyAnalyzer.java | 53 +++--
.../apache/doris/datasource/InternalCatalog.java | 3 +-
.../vault_p0/database/test_database_vault.groovy | 246 +++++++++++++++++++++
4 files changed, 310 insertions(+), 16 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 4a6bc249b4a..bb7b540a131 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.cloud.catalog.CloudEnv;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@@ -39,6 +40,7 @@ import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -52,6 +54,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -1000,6 +1003,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>,
}
}
+ checkStorageVault(properties);
replayUpdateDbProperties(properties);
return true;
}
@@ -1008,6 +1012,26 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>,
return binlogConfig;
}
+ public void checkStorageVault(Map<String, String> properties) throws
DdlException {
+ Env env = Env.getCurrentEnv();
+ if (!Config.isCloudMode() || !((CloudEnv)
env).getEnableStorageVault()) {
+ return;
+ }
+
+ Map<String, String> propertiesCheck = new HashMap<>(properties);
+ String storageVaultName =
PropertyAnalyzer.analyzeStorageVaultName(propertiesCheck);
+ if (Strings.isNullOrEmpty(storageVaultName)) {
+ return;
+ }
+
+ String storageVaultId =
env.getStorageVaultMgr().getVaultIdByName(storageVaultName);
+ if (Strings.isNullOrEmpty(storageVaultId)) {
+ throw new DdlException("Storage vault '" + storageVaultName + "'
does not exist. "
+ + "You can use `SHOW STORAGE VAULT` to get all available
vaults, "
+ + "or create a new one with `CREATE STORAGE VAULT`.");
+ }
+ }
+
public String toJson() {
return GsonUtils.GSON.toJson(this);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index e74d909e1ff..ec9b986915c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
+import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EnvFactory;
@@ -61,6 +62,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -1187,28 +1189,49 @@ public class PropertyAnalyzer {
return storagePolicy;
}
- /**
- * @param properties
- * @return <storageVaultName, storageVaultId>
- * @throws AnalysisException
- */
- public static Pair<String, String> analyzeStorageVault(Map<String, String>
properties) throws AnalysisException {
+ public static String analyzeStorageVaultName(Map<String, String>
properties) {
String storageVaultName = null;
if (properties != null &&
properties.containsKey(PROPERTIES_STORAGE_VAULT_NAME)) {
storageVaultName = properties.get(PROPERTIES_STORAGE_VAULT_NAME);
properties.remove(PROPERTIES_STORAGE_VAULT_NAME);
}
+ return storageVaultName;
+ }
+
+ /**
+ * @param properties, db
+ * @return <storageVaultName, storageVaultId>
+ * @throws AnalysisException
+ */
+ public static Pair<String, String> analyzeStorageVault(Map<String, String>
properties, Database db)
+ throws AnalysisException {
+ String storageVaultName = analyzeStorageVaultName(properties);
+ String storageVaultId = null;
+
if (Strings.isNullOrEmpty(storageVaultName)) {
- // If user does not specify one storage vault then FE would use
the default vault
- Pair<String, String> info =
Env.getCurrentEnv().getStorageVaultMgr().getDefaultStorageVault();
- if (info == null) {
- throw new AnalysisException("No default storage vault."
- + " You can use `SHOW STORAGE VAULT` to get all
available vaults,"
- + " and pick one set default vault with `SET
<vault_name> AS DEFAULT STORAGE VAULT`");
+ // If user does not specify one storage vault then FE would check
db's storage vault then the default vault
+ // the storage vault inherit order is as follows: table -> db ->
default
+ if (db.getDbProperties() != null) {
+ Map<String, String> dbProperties = new
HashMap<>(db.getDbProperties().getProperties());
+ storageVaultName =
PropertyAnalyzer.analyzeStorageVaultName(dbProperties);
+ }
+
+ if (!Strings.isNullOrEmpty(storageVaultName)) {
+ storageVaultId =
Env.getCurrentEnv().getStorageVaultMgr().getVaultIdByName(storageVaultName);
+ LOG.info("Using database[{}] storage vault: name={}, id={}",
+ db.getName(), storageVaultName, storageVaultId);
+ } else {
+ // continue to check default vault
+ Pair<String, String> info =
Env.getCurrentEnv().getStorageVaultMgr().getDefaultStorageVault();
+ if (info == null) {
+ throw new AnalysisException("No default storage vault."
+ + " You can use `SHOW STORAGE VAULT` to get all
available vaults,"
+ + " and pick one set default vault with `SET
<vault_name> AS DEFAULT STORAGE VAULT`");
+ }
+ storageVaultName = info.first;
+ LOG.info("Using default storage vault, name:{} id:{}",
info.first, info.second);
}
- storageVaultName = info.first;
- LOG.info("Using default storage vault, name:{} id:{}", info.first,
info.second);
}
if (Strings.isNullOrEmpty(storageVaultName)) {
@@ -1217,7 +1240,7 @@ public class PropertyAnalyzer {
+ " and pick one to set the table property
`\"storage_vault_name\" = \"<vault_name>\"`");
}
- String storageVaultId =
Env.getCurrentEnv().getStorageVaultMgr().getVaultIdByName(storageVaultName);
+ storageVaultId =
Env.getCurrentEnv().getStorageVaultMgr().getVaultIdByName(storageVaultName);
if (Strings.isNullOrEmpty(storageVaultId)) {
throw new AnalysisException("Storage vault '" + storageVaultName +
"' does not exist. "
+ "You can use `SHOW STORAGE VAULT` to get all available
vaults, "
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 0419c219629..c01443caa4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -431,6 +431,7 @@ public class InternalCatalog implements CatalogIf<Database>
{
long id = Env.getCurrentEnv().getNextId();
Database db = new Database(id, fullDbName);
// check and analyze database properties before create database
+ db.checkStorageVault(properties);
db.setDbProperties(new DatabaseProperty(properties));
if (!tryLock(false)) {
@@ -2738,7 +2739,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
if (Config.isCloudMode() && ((CloudEnv) env).getEnableStorageVault()) {
// <storageVaultName, storageVaultId>
- Pair<String, String> storageVaultInfoPair =
PropertyAnalyzer.analyzeStorageVault(properties);
+ Pair<String, String> storageVaultInfoPair =
PropertyAnalyzer.analyzeStorageVault(properties, db);
// Check if user has storage vault usage privilege
if (ConnectContext.get() != null && !env.getAccessManager()
diff --git
a/regression-test/suites/vault_p0/database/test_database_vault.groovy
b/regression-test/suites/vault_p0/database/test_database_vault.groovy
new file mode 100644
index 00000000000..16c45dfbd88
--- /dev/null
+++ b/regression-test/suites/vault_p0/database/test_database_vault.groovy
@@ -0,0 +1,246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.util.stream.Collectors;
+
+suite("test_database_vault", "nonConcurrent") {
+ if (!isCloudMode()) {
+ logger.info("skip test_database_vault case because not cloud mode")
+ return
+ }
+
+ if (!enableStoragevault()) {
+ logger.info("skip test_database_vault case")
+ return
+ }
+
+ def db1 = "test_db_vault_db1"
+ def db2 = "test_db_vault_db2"
+ def vault1 = "test_db_vault_vault1"
+ def vault2 = "test_db_vault_vault2"
+ def vault3 = "test_db_vault_vault3"
+ def table1 = "test_db_vault_table1"
+ def table2 = "test_db_vault_table2"
+ def table3 = "test_db_vault_table3"
+
+ try {
+
+ sql """
+ UNSET DEFAULT STORAGE VAULT
+ """
+
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS ${vault1}
+ PROPERTIES (
+ "type"="hdfs",
+ "fs.defaultFS"="${getHmsHdfsFs()}",
+ "path_prefix" = "test_db_vault_vault1_prefix",
+ "hadoop.username" = "${getHmsUser()}"
+ );
+ """
+
+ def storageVaults = (sql " SHOW STORAGE VAULT; ").stream().map(row ->
row[0]).collect(Collectors.toSet())
+ assertTrue(storageVaults.contains(vault1))
+
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS ${vault2}
+ PROPERTIES (
+ "type"="hdfs",
+ "fs.defaultFS"="${getHmsHdfsFs()}",
+ "path_prefix" = "test_db_vault_vault2_prefix",
+ "hadoop.username" = "${getHmsUser()}"
+ );
+ """
+
+ storageVaults = (sql " SHOW STORAGE VAULT; ").stream().map(row ->
row[0]).collect(Collectors.toSet())
+ assertTrue(storageVaults.contains(vault2))
+
+ sql "DROP DATABASE IF EXISTS ${db1}"
+ sql "DROP DATABASE IF EXISTS ${db2}"
+ // Create database with vault, vault should exist
+ expectExceptionLike({
+ sql """
+ CREATE DATABASE IF NOT EXISTS ${db1}
+ PROPERTIES (
+ "storage_vault_name" = "non_exist_vault"
+ )
+ """
+ }, "not exist")
+
+ sql """
+ CREATE DATABASE IF NOT EXISTS ${db1}
+ PROPERTIES (
+ "storage_vault_name" = "${vault2}"
+ )
+ """
+
+ sql """
+ CREATE DATABASE IF NOT EXISTS ${db2}
+ """
+
+ def createDbSql = (sql " SHOW CREATE DATABASE ${db1};
").stream().map(row -> row[1]).collect(Collectors.toSet())
+ assertTrue(createDbSql.first().contains(vault2))
+
+ // Alter database property with storage vault
+ expectExceptionLike({
+ sql """
+ ALTER DATABASE ${db1} SET
+ PROPERTIES (
+ "storage_vault_name" = "non_exist_vault"
+ )
+ """
+ }, "not exist")
+
+ sql """
+ ALTER DATABASE ${db1} SET
+ PROPERTIES (
+ "storage_vault_name" = "${vault1}"
+ )
+ """
+
+ createDbSql = (sql " SHOW CREATE DATABASE ${db1}; ").stream().map(row
-> row[1]).collect(Collectors.toSet())
+ assertTrue(createDbSql.first().contains(vault1))
+
+ sql """
+ ALTER DATABASE ${db2} SET
+ PROPERTIES (
+ "storage_vault_name" = "${vault2}"
+ )
+ """
+
+ createDbSql = (sql " SHOW CREATE DATABASE ${db2}; ").stream().map(row
-> row[1]).collect(Collectors.toSet())
+ assertTrue(createDbSql.first().contains(vault2))
+
+ // Create table with/without specifying vault
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db1}.${table1} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES (
+ "storage_vault_name" = ${vault2}
+ )
+ """
+ def createTableSql = (sql " SHOW CREATE TABLE ${db1}.${table1};
").stream().map(row -> row[1]).collect(Collectors.toSet())
+ assertTrue(createTableSql.first().contains(vault2))
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db1}.${table2} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ """
+ createTableSql = (sql " SHOW CREATE TABLE ${db1}.${table2};
").stream().map(row -> row[1]).collect(Collectors.toSet())
+ assertTrue(createTableSql.first().contains(vault1))
+
+ // Unset Database storage vault
+ sql """
+ ALTER DATABASE ${db2} SET
+ PROPERTIES (
+ "storage_vault_name" = ""
+ )
+ """
+ expectExceptionLike({
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db2}.${table1} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ """
+ }, "No default storage vault")
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db2}.${table2} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES (
+ "storage_vault_name" = ${vault1}
+ )
+ """
+ createTableSql = (sql " SHOW CREATE TABLE ${db2}.${table2};
").stream().map(row -> row[1]).collect(Collectors.toSet())
+ assertTrue(createTableSql.first().contains(vault1))
+
+ sql """
+ ALTER DATABASE ${db2} SET
+ PROPERTIES (
+ "storage_vault_name" = "${vault2}"
+ )
+ """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db2}.${table1} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ """
+ createTableSql = (sql " SHOW CREATE TABLE ${db2}.${table1};
").stream().map(row -> row[1]).collect(Collectors.toSet())
+ assertTrue(createTableSql.first().contains(vault2))
+
+ // Rename storage vault
+ sql """
+ ALTER STORAGE VAULT ${vault2}
+ PROPERTIES (
+ "type" = "hdfs",
+ "VAULT_NAME" = "${vault3}"
+ );
+ """
+
+ expectExceptionLike({
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db2}.${table3} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ """
+ }, "does not exist")
+
+ sql """
+ ALTER DATABASE ${db2} SET
+ PROPERTIES (
+ "storage_vault_name" = "${vault3}"
+ );
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${db2}.${table3} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ """
+
+ createTableSql = (sql " SHOW CREATE TABLE ${db2}.${table3};
").stream().map(row -> row[1]).collect(Collectors.toSet())
+ assertTrue(createTableSql.first().contains(vault3))
+ } finally {
+ sql "DROP DATABASE IF EXISTS ${db1}"
+ sql "DROP DATABASE IF EXISTS ${db2}"
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
