This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 76b0393cc9f [Enhancement](tvf)catalog tvf implements user permission
checks and hides sensitive information (#41497) (#41599)
76b0393cc9f is described below
commit 76b0393cc9f9add9fff6b178ac2b7da6baf53f09
Author: daidai <[email protected]>
AuthorDate: Wed Oct 9 19:06:35 2024 +0800
[Enhancement](tvf)catalog tvf implements user permission checks and hides
sensitive information (#41497) (#41599)
bp #41497
---
be/src/vec/exec/scan/vmeta_scanner.cpp | 1 +
.../org/apache/doris/datasource/CatalogMgr.java | 85 +++++++++++++---------
.../doris/tablefunction/MetadataGenerator.java | 9 ++-
.../external_table_p0/tvf/test_catalogs_tvf.out | 40 ++++++++++
.../external_table_p0/tvf/test_catalogs_tvf.groovy | 68 ++++++++++++++++-
5 files changed, 162 insertions(+), 41 deletions(-)
diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp
b/be/src/vec/exec/scan/vmeta_scanner.cpp
index 74fed8c80c7..180d18b0c2c 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.cpp
+++ b/be/src/vec/exec/scan/vmeta_scanner.cpp
@@ -400,6 +400,7 @@ Status VMetaScanner::_build_catalogs_metadata_request(const
TMetaScanRange& meta
// create TMetadataTableRequestParams
TMetadataTableRequestParams metadata_table_params;
metadata_table_params.__set_metadata_type(TMetadataType::CATALOGS);
+ metadata_table_params.__set_current_user_ident(_user_identity);
request->__set_metada_table_params(metadata_table_params);
return Status::OK();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index d9b118b03c5..7f63ea92fd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.analysis.DropCatalogStmt;
import org.apache.doris.analysis.ShowCatalogStmt;
import org.apache.doris.analysis.ShowCreateCatalogStmt;
+import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EnvFactory;
@@ -371,30 +372,27 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
matcher =
PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(),
CaseSensibility.CATALOG.getCaseSensibility());
}
- for (CatalogIf catalog : nameToCatalog.values()) {
- if (Env.getCurrentEnv().getAccessManager()
- .checkCtlPriv(ConnectContext.get(),
catalog.getName(), PrivPredicate.SHOW)) {
- String name = catalog.getName();
- // Filter catalog name
- if (matcher != null && !matcher.match(name)) {
- continue;
- }
- List<String> row = Lists.newArrayList();
- row.add(String.valueOf(catalog.getId()));
- row.add(name);
- row.add(catalog.getType());
- if (name.equals(currentCtlg)) {
- row.add("Yes");
- } else {
- row.add("No");
- }
- Map<String, String> props = catalog.getProperties();
- String createTime =
props.getOrDefault(ExternalCatalog.CREATE_TIME, FeConstants.null_string);
- row.add(createTime);
-
row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
- row.add(catalog.getComment());
- rows.add(row);
+ for (CatalogIf catalog :
listCatalogsWithCheckPriv(ConnectContext.get().getCurrentUserIdentity())) {
+ String name = catalog.getName();
+ // Filter catalog name
+ if (matcher != null && !matcher.match(name)) {
+ continue;
+ }
+ List<String> row = Lists.newArrayList();
+ row.add(String.valueOf(catalog.getId()));
+ row.add(name);
+ row.add(catalog.getType());
+ if (name.equals(currentCtlg)) {
+ row.add("Yes");
+ } else {
+ row.add("No");
}
+ Map<String, String> props = catalog.getProperties();
+ String createTime =
props.getOrDefault(ExternalCatalog.CREATE_TIME, FeConstants.null_string);
+ row.add(createTime);
+
row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
+ row.add(catalog.getComment());
+ rows.add(row);
// sort by catalog name
rows.sort((x, y) -> {
@@ -414,18 +412,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
if (!Strings.isNullOrEmpty(catalog.getResource())) {
rows.add(Arrays.asList("resource", catalog.getResource()));
}
- // use tree map to maintain display order, making it easier to
view properties
- Map<String, String> sortedMap = new
TreeMap<>(catalog.getProperties()).descendingMap();
- for (Map.Entry<String, String> elem : sortedMap.entrySet()) {
- if (PrintableMap.HIDDEN_KEY.contains(elem.getKey())) {
- continue;
- }
- if (PrintableMap.SENSITIVE_KEY.contains(elem.getKey())) {
- rows.add(Arrays.asList(elem.getKey(),
PrintableMap.PASSWORD_MASK));
- } else {
- rows.add(Arrays.asList(elem.getKey(),
elem.getValue()));
- }
- }
+ Map<String, String> sortedMap =
getCatalogPropertiesWithPrintable(catalog);
+ sortedMap.forEach((k, v) -> rows.add(Arrays.asList(k, v)));
}
} finally {
readUnlock();
@@ -434,6 +422,25 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
return new ShowResultSet(showStmt.getMetaData(), rows);
}
+ public static Map<String, String>
getCatalogPropertiesWithPrintable(CatalogIf<?> catalog) {
+ // use tree map to maintain display order, making it easier to view
properties
+ Map<String, String> sortedMap = new TreeMap<>();
+ catalog.getProperties().forEach(
+ (key, value) -> {
+ if (PrintableMap.HIDDEN_KEY.contains(key)) {
+ return;
+ }
+ if (PrintableMap.SENSITIVE_KEY.contains(key)) {
+ sortedMap.put(key, PrintableMap.PASSWORD_MASK);
+ } else {
+ sortedMap.put(key, value);
+ }
+ }
+ );
+ return sortedMap;
+ }
+
+
public ShowResultSet showCreateCatalog(ShowCreateCatalogStmt showStmt)
throws AnalysisException {
List<List<String>> rows = Lists.newArrayList();
readLock();
@@ -538,6 +545,14 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
return nameToCatalog.values().stream().collect(Collectors.toList());
}
+ public List<CatalogIf> listCatalogsWithCheckPriv(UserIdentity
userIdentity) {
+ return nameToCatalog.values().stream().filter(
+ catalog -> Env.getCurrentEnv().getAccessManager()
+ .checkCtlPriv(userIdentity, catalog.getName(),
PrivPredicate.SHOW)
+ ).collect(Collectors.toList());
+ }
+
+
/**
* Reply for alter catalog props event.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 466d324c369..a1cac535995 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -47,6 +47,7 @@ import org.apache.doris.common.util.NetUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.InternalCatalog;
@@ -513,17 +514,17 @@ public class MetadataGenerator {
private static TFetchSchemaTableDataResult
catalogsMetadataResult(TMetadataTableRequestParams params) {
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
- List<CatalogIf> info =
Env.getCurrentEnv().getCatalogMgr().listCatalogs();
- List<TRow> dataBatch = Lists.newArrayList();
+ UserIdentity currentUserIdentity =
UserIdentity.fromThrift(params.getCurrentUserIdent());
+ List<CatalogIf> info =
Env.getCurrentEnv().getCatalogMgr().listCatalogsWithCheckPriv(currentUserIdentity);
+ List<TRow> dataBatch = Lists.newArrayList();
for (CatalogIf catalog : info) {
TRow trow = new TRow();
trow.addToColumnValue(new TCell().setLongVal(catalog.getId()));
trow.addToColumnValue(new TCell().setStringVal(catalog.getName()));
trow.addToColumnValue(new TCell().setStringVal(catalog.getType()));
- Map<String, String> properties = catalog.getProperties();
-
+ Map<String, String> properties =
CatalogMgr.getCatalogPropertiesWithPrintable(catalog);
for (Map.Entry<String, String> entry : properties.entrySet()) {
TRow subTrow = new TRow(trow);
subTrow.addToColumnValue(new
TCell().setStringVal(entry.getKey()));
diff --git a/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
b/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
index 310da69d766..7f97c337999 100644
--- a/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
+++ b/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
@@ -8,3 +8,43 @@ catalog_test_hive00 hms type hms
-- !create --
catalog_test_es00 es type es
+-- !test_10 --
+catalog_tvf_test_dlf hms dlf.catalog.id 987654321
+
+-- !test_11 --
+catalog_tvf_test_dlf hms dlf.secret_key *XXX
+
+-- !test_12 --
+catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA
+
+-- !test_13 --
+catalog_tvf_test_dlf hms dlf.uid 123456789
+
+-- !test_14 --
+catalog_tvf_test_dlf hms type hms
+
+-- !test_15 --
+
+-- !test_16 --
+internal internal NULL NULL
+
+-- !test_17 --
+catalog_tvf_test_dlf hms dlf.secret_key *XXX
+
+-- !test_18 --
+catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA
+
+-- !test_19 --
+catalog_tvf_test_dlf hms dlf.uid 123456789
+
+-- !test_20 --
+catalog_tvf_test_dlf hms type hms
+
+-- !test_21 --
+
+-- !test_22 --
+
+-- !test_23 --
+
+-- !test_24 --
+
diff --git
a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
index a59953cf567..748a7e49d14 100644
--- a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy
@@ -18,12 +18,12 @@
suite("test_catalogs_tvf","p0,external,tvf,external_docker") {
List<List<Object>> table = sql """ select * from catalogs(); """
assertTrue(table.size() > 0)
- assertEquals(5, table[0].size)
+ assertEquals(5, table[0].size())
table = sql """ select CatalogId,CatalogName from catalogs();"""
assertTrue(table.size() > 0)
- assertTrue(table[0].size == 2)
+ assertTrue(table[0].size() == 2)
table = sql """ select * from catalogs() where CatalogId=0;"""
@@ -76,4 +76,68 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker")
{
// check exception
exception "catalogs table-valued-function does not support any params"
}
+
+ sql """ drop catalog if exists catalog_tvf_test_dlf """
+
+ sql """
+ CREATE CATALOG catalog_tvf_test_dlf PROPERTIES (
+ "type"="hms",
+ "hive.metastore.type" = "dlf",
+ "dlf.proxy.mode" = "DLF_ONLY",
+ "dlf.endpoint" = "dlf-vpc.cn-beijing.aliyuncs.com",
+ "dlf.region" = "cn-beijing",
+ "dlf.uid" = "123456789",
+ "dlf.catalog.id" = "987654321",
+ "dlf.access_key" = "AAAAAAAAAAAAAAAAAAAAAA",
+ "dlf.secret_key" = "BBBBBBBBBBBBBBBBBBBBBB"
+ );"""
+
+ order_qt_test_10 """ select CatalogName,CatalogType,Property,Value from
catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.catalog.id" """
+ order_qt_test_11 """ select CatalogName,CatalogType,Property,Value from
catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.secret_key" """
+ order_qt_test_12 """ select CatalogName,CatalogType,Property,Value from
catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.access_key" """
+ order_qt_test_13 """ select CatalogName,CatalogType,Property,Value from
catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid"
"""
+ order_qt_test_14 """ select CatalogName,CatalogType,Property,Value from
catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type"
"""
+
+
+ def user = 'catalog_user_test'
+ def pwd = 'C123_567p'
+ try_sql("DROP USER ${user}")
+
+ sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
+ sql """GRANT SELECT_PRIV on `internal`.``.`` to '${user}'"""
+
+
+
+ connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+ sql """ switch internal """
+ order_qt_test_15 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"type" """
+ order_qt_test_16 """ select CatalogName,CatalogType,Property,Value
from catalogs() """
+ }
+
+ sql """GRANT SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` to '${user}'"""
+
+
+ connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+ sql """ switch internal """
+
+ order_qt_test_17 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.secret_key" """
+ order_qt_test_18 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.access_key" """
+ order_qt_test_19 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.uid" """
+ order_qt_test_20 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"type" """
+ }
+
+ sql """REVOKE SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` FROM '${user}'"""
+
+
+ connect(user=user, password="${pwd}", url=context.config.jdbcUrl) {
+ sql """ switch internal """
+
+ order_qt_test_21 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.secret_key" """
+ order_qt_test_22 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.access_key" """
+ order_qt_test_23 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"dlf.uid" """
+ order_qt_test_24 """ select CatalogName,CatalogType,Property,Value
from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property=
"type" """
+ }
+
+
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
