This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 09cfca191e1 [feat](Nereids) support refresh catalog command (#43449)
09cfca191e1 is described below
commit 09cfca191e18c5ae6d71e2130b7c3cd76b9c1031
Author: Petrichor <31833513+vinle...@users.noreply.github.com>
AuthorDate: Tue Nov 19 16:56:31 2024 +0800
[feat](Nereids) support refresh catalog command (#43449)
Issue Number: close # https://github.com/apache/doris/issues/42580
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 6 +-
.../org/apache/doris/catalog/RefreshManager.java | 6 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 14 +++
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../commands/refresh/RefreshCatalogCommand.java | 128 +++++++++++++++++++++
.../trees/plans/visitor/CommandVisitor.java | 5 +
.../test_nereids_refresh_catalog.out | 63 ++++++++++
.../test_nereids_refresh_catalog.groovy | 70 +++++++++++
8 files changed, 289 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 734be9fa51d..c9958e68641 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -55,6 +55,7 @@ statementBase
| supportedDropStatement #supportedDropStatementAlias
| supportedSetStatement #supportedSetStatementAlias
| supportedUnsetStatement #supportedUnsetStatementAlias
+ | supportedRefreshStatement #supportedRefreshStatementAlias
| supportedShowStatement #supportedShowStatementAlias
| unsupportedStatement #unsupported
| supportedRecoverStatement #supportedRecoverStatementAlias
@@ -418,10 +419,13 @@ channelDescription
partitionSpec? columnList=identifierList?
;
+supportedRefreshStatement
+ : REFRESH CATALOG name=identifier propertyClause?
#refreshCatalog
+ ;
+
unsupportedRefreshStatement
: REFRESH TABLE name=multipartIdentifier
#refreshTable
| REFRESH DATABASE name=multipartIdentifier propertyClause?
#refreshDatabase
- | REFRESH CATALOG name=identifier propertyClause?
#refreshCatalog
| REFRESH LDAP (ALL | (FOR user=identifierOrText))
#refreshLdap
;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 52694e5a5bd..e421cc1475c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -32,8 +32,8 @@ import org.apache.doris.datasource.ExternalObjectLog;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
+import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.persist.OperationType;
-import org.apache.doris.qe.DdlExecutor;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
@@ -284,9 +284,9 @@ public class RefreshManager {
* {@link
org.apache.doris.analysis.RefreshCatalogStmt#analyze(Analyzer)} is ok,
* because the default value of invalidCache is true.
* */
- RefreshCatalogStmt refreshCatalogStmt = new
RefreshCatalogStmt(catalogName, null);
+ RefreshCatalogCommand refreshCatalogCommand = new
RefreshCatalogCommand(catalogName, null);
try {
- DdlExecutor.execute(Env.getCurrentEnv(),
refreshCatalogStmt);
+ refreshCatalogCommand.handleRefreshCatalog();
} catch (Exception e) {
LOG.warn("failed to refresh catalog {}",
catalogName, e);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 064e9568ac2..6fc011d6926 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -163,6 +163,7 @@ import org.apache.doris.nereids.DorisParser.QueryContext;
import org.apache.doris.nereids.DorisParser.QueryOrganizationContext;
import org.apache.doris.nereids.DorisParser.QueryTermContext;
import org.apache.doris.nereids.DorisParser.RecoverDatabaseContext;
+import org.apache.doris.nereids.DorisParser.RefreshCatalogContext;
import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
@@ -513,6 +514,7 @@ import
org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
import
org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
@@ -4050,6 +4052,17 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
}
@Override
+
+ public Object visitRefreshCatalog(RefreshCatalogContext ctx) {
+ if (ctx.name != null) {
+ String catalogName = ctx.name.getText();
+ Map<String, String> properties = ctx.propertyClause() != null
+ ?
Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+ return new RefreshCatalogCommand(catalogName, properties);
+ }
+ throw new AnalysisException("catalog name can not be null");
+ }
+
public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
return new ShowLastInsertCommand();
}
@@ -4061,6 +4074,7 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
partitionId = Long.parseLong(ctx.partitionId.getText());
}
return new ShowPartitionIdCommand(partitionId);
+
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 55c73c11a81..e255ac32621 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -174,6 +174,7 @@ public enum PlanType {
SET_TRANSACTION_COMMAND,
SET_USER_PROPERTIES_COMMAND,
SET_DEFAULT_STORAGE_VAULT_COMMAND,
+ REFRESH_CATALOG_COMMAND,
PREPARED_COMMAND,
EXECUTE_COMMAND,
SHOW_BACKENDS_COMMAND,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java
new file mode 100644
index 00000000000..7b643303819
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshCatalogCommand.java
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.refresh;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.CatalogLog;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.persist.OperationType;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Map;
+
+/**
+ * Manually refresh the catalog metadata.
+ */
+public class RefreshCatalogCommand extends Command implements ForwardWithSync {
+ private static final Logger LOG =
LogManager.getLogger(RefreshCatalogCommand.class);
+ private static final String INVALID_CACHE = "invalid_cache";
+ private final String catalogName;
+ private Map<String, String> properties;
+ private boolean invalidCache = true;
+
+ public RefreshCatalogCommand(String catalogName, Map<String, String>
properties) {
+ super(PlanType.REFRESH_CATALOG_COMMAND);
+ this.catalogName = catalogName;
+ this.properties = properties;
+ }
+
+ private void validate() throws AnalysisException {
+ Util.checkCatalogAllRules(catalogName);
+ if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ throw new AnalysisException("Internal catalog name can't be
refresh.");
+ }
+
+ if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(
+ ConnectContext.get(), catalogName, PrivPredicate.SHOW)) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED_ERROR,
+ PrivPredicate.SHOW.getPrivs().toString(), catalogName);
+ }
+
+ if (properties != null) {
+ // Set to false only if user set the property
"invalid_cache"="false"
+ invalidCache = !(properties.get(INVALID_CACHE) != null &&
properties.get(INVALID_CACHE)
+ .equalsIgnoreCase("false"));
+ }
+
+ }
+
+ private void refreshCatalogInternal(CatalogIf catalog) {
+ if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ ((ExternalCatalog) catalog).onRefreshCache(invalidCache);
+ LOG.info("refresh catalog {} with invalidCache {}", catalogName,
invalidCache);
+ }
+ }
+
+ /**
+ * refresh catalog
+ */
+ public void handleRefreshCatalog() throws AnalysisException {
+ CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalogName);
+ CatalogLog log = new CatalogLog();
+ log.setCatalogId(catalog.getId());
+ log.setInvalidCache(invalidCache);
+ refreshCatalogInternal(catalog);
+
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_REFRESH_CATALOG,
log);
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
+ validate();
+ handleRefreshCatalog();
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitRefreshCatalogCommand(this, context);
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
+
+ /**
+ * return sql expression of this command
+ *
+ * @return sql command
+ */
+ public String toSql() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("REFRESH CATALOG ")
+ .append("`")
+ .append(catalogName)
+ .append("`");
+ return stringBuilder.toString();
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 2ffdcc767ab..117c42b7bfc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -77,6 +77,7 @@ import
org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
import
org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
/** CommandVisitor. */
public interface CommandVisitor<R, C> {
@@ -255,6 +256,10 @@ public interface CommandVisitor<R, C> {
return visitCommand(setDefaultStorageVaultCommand, context);
}
+ default R visitRefreshCatalogCommand(RefreshCatalogCommand
refreshCatalogCommand, C context) {
+ return visitCommand(refreshCatalogCommand, context);
+ }
+
default R visitShowLastInsertCommand(ShowLastInsertCommand
showLastInsertCommand, C context) {
return visitCommand(showLastInsertCommand, context);
}
diff --git
a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
new file mode 100644
index 00000000000..89b47bbe4eb
--- /dev/null
+++
b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
@@ -0,0 +1,63 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+DORIS
+Doris
+doris
+doris_test
+information_schema
+init_db
+mysql
+show_test_do_not_modify
+
+-- !ex_tb0_where --
+111
+
+-- !ex_tb0 --
+111 abc
+112 abd
+113 abe
+114 abf
+115 abg
+
+-- !sql --
+DORIS
+Doris
+doris
+doris_test
+information_schema
+init_db
+mysql
+show_test_do_not_modify
+
+-- !sql --
+DORIS
+Doris
+doris
+doris_test
+information_schema
+init_db
+mysql
+new_mysql_db
+show_test_do_not_modify
+
+-- !sql --
+DORIS
+Doris
+doris
+doris_test
+information_schema
+init_db
+mysql
+new_mysql_db
+show_test_do_not_modify
+
+-- !sql --
+DORIS
+Doris
+doris
+doris_test
+information_schema
+init_db
+mysql
+show_test_do_not_modify
+
diff --git
a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
new file mode 100644
index 00000000000..f74f78a3709
--- /dev/null
+++
b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nereids_refresh_catalog",
"p0,external,mysql,external_docker,external_docker_mysql") {
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar";
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String catalog_name = "refresh_catalog_mysql_catalog_nereids";
+ String internal_db_name = "regression_test_jdbc_catalog_p0";
+ String ex_db_name = "doris_test";
+ String mysql_port = context.config.otherConfigs.get("mysql_57_port");
+ String ex_tb0 = "ex_tb0";
+ String new_mysql_db = "new_mysql_db";
+
+ sql """drop catalog if exists ${catalog_name} """
+
+ sql """set enable_nereids_planner=true;"""
+ sql """set enable_fallback_to_original_planner=false;"""
+
+ sql """create catalog if not exists ${catalog_name} properties(
+ "type"="jdbc",
+ "user"="root",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "com.mysql.cj.jdbc.Driver"
+ );"""
+
+ sql """switch ${catalog_name}"""
+ sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists
${new_mysql_db}");"""
+
+ qt_sql """show databases;"""
+ sql """ use ${ex_db_name}"""
+
+ qt_ex_tb0_where """select id from ${ex_tb0} where id = 111;"""
+ order_qt_ex_tb0 """ select id, name from ${ex_tb0} order by id; """
+ // create database in mysql
+ sql """CALL EXECUTE_STMT("${catalog_name}", "create database
${new_mysql_db} ;");"""
+ qt_sql """show databases;"""
+ checkNereidsExecute("refresh catalog ${catalog_name} ;")
+ qt_sql """show databases;"""
+
+ checkNereidsExecute("refresh catalog ${catalog_name} properties
('invalid_cache'='true');")
+
+ sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists
${new_mysql_db} ;");"""
+ qt_sql """show databases;"""
+
+ checkNereidsExecute("refresh catalog ${catalog_name} properties
('invalid_cache'='true');")
+ qt_sql """show databases;"""
+
+ sql """ drop catalog if exists ${catalog_name} ;"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
