This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new eb15bf552ac branch-3.0: [feat](sql-convertor) support
sql_convertor_config & enable_sql_convertor_features for sql convertor #50959
#50707 (#51080)
eb15bf552ac is described below
commit eb15bf552ac7298860e238e4b6f599941adf981a
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon May 26 10:25:36 2025 +0800
branch-3.0: [feat](sql-convertor) support sql_convertor_config &
enable_sql_convertor_features for sql convertor #50959 #50707 (#51080)
bp #50959 #50707
---
.../plugin/dialect/HttpDialectConverterPlugin.java | 3 +-
.../doris/plugin/dialect/HttpDialectUtils.java | 11 ++++--
.../java/org/apache/doris/qe/SessionVariable.java | 42 ++++++++++++++++++++++
.../apache/doris/plugin/HttpDialectUtilsTest.java | 12 +++----
.../java/org/apache/doris/qe/VariableMgrTest.java | 32 +++++++++++++++++
5 files changed, 90 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
index 29dca027c6a..ff97a19d876 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
@@ -103,7 +103,8 @@ public class HttpDialectConverterPlugin extends Plugin
implements DialectConvert
if (Strings.isNullOrEmpty(targetURL)) {
return null;
}
- return HttpDialectUtils.convertSql(targetURL, originSql,
sessionVariable.getSqlDialect());
+ return HttpDialectUtils.convertSql(targetURL, originSql,
sessionVariable.getSqlDialect(),
+ sessionVariable.getSqlConvertorFeatures(),
sessionVariable.getSqlConvertorConfig());
}
// no need to override parseSqlWithDialect, just return null
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
index 3cb6244d515..89acd66658d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
@@ -38,8 +38,9 @@ import java.nio.charset.StandardCharsets;
public class HttpDialectUtils {
private static final Logger LOG =
LogManager.getLogger(HttpDialectUtils.class);
- public static String convertSql(String targetURL, String originStmt,
String dialect) {
- ConvertRequest convertRequest = new ConvertRequest(originStmt,
dialect);
+ public static String convertSql(String targetURL, String originStmt,
String dialect,
+ String[] features, String config) {
+ ConvertRequest convertRequest = new ConvertRequest(originStmt,
dialect, features, config);
HttpURLConnection connection = null;
try {
@@ -110,14 +111,18 @@ public class HttpDialectUtils {
private String to; // CHECKSTYLE IGNORE THIS LINE
private String source; // CHECKSTYLE IGNORE THIS LINE
private String case_sensitive; // CHECKSTYLE IGNORE THIS LINE
+ private String[] enable_sql_convertor_features; // CHECKSTYLE IGNORE
THIS LINE
+ private String config; // CHECKSTYLE IGNORE THIS LINE
- public ConvertRequest(String originStmt, String dialect) {
+ public ConvertRequest(String originStmt, String dialect, String[]
features, String config) {
this.version = "v1";
this.sql_query = originStmt;
this.from = dialect;
this.to = "doris";
this.source = "text";
this.case_sensitive = "0";
+ this.enable_sql_convertor_features = features;
+ this.config = config;
}
public String toJson() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index fad889f78a9..9391e5c76d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -710,6 +710,10 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_TEXT_VALIDATE_UTF8 =
"enable_text_validate_utf8";
+ public static final String ENABLE_SQL_CONVERTOR_FEATURES =
"enable_sql_convertor_features";
+
+ public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config";
+
/**
* If set false, user couldn't submit analyze SQL and FE won't allocate
any related resources.
*/
@@ -2419,6 +2423,21 @@ public class SessionVariable implements Serializable,
Writable {
| VariableMgr.READ_ONLY)
public boolean newIsIpAddressInRange = true;
+ @VariableMgr.VarAttr(name = ENABLE_SQL_CONVERTOR_FEATURES, needForward =
true,
+ checker = "checkSqlConvertorFeatures",
+ description = {
+ "开启 SQL 转换器的指定功能。多个功能使用逗号分隔",
+ "enable SQL convertor features. Multiple features are
separated by commas"
+ })
+ public String enableSqlConvertorFeatures = "";
+
+ @VariableMgr.VarAttr(name = SQL_CONVERTOR_CONFIG, needForward = true,
+ description = {
+ "SQL 转换器的相关配置，使用 Json 格式。以 {} 为根元素。",
+ "SQL convertor config, use Json format. The root element
is {}"
+ })
+ public String sqlConvertorConfig = "{}";
+
public void setEnableEsParallelScroll(boolean enableESParallelScroll) {
this.enableESParallelScroll = enableESParallelScroll;
}
@@ -3428,6 +3447,14 @@ public class SessionVariable implements Serializable,
Writable {
return waitFullBlockScheduleTimes;
}
+ public String[] getSqlConvertorFeatures() {
+ return enableSqlConvertorFeatures.split(",");
+ }
+
+ public String getSqlConvertorConfig() {
+ return sqlConvertorConfig;
+ }
+
public Dialect getSqlParseDialect() {
return Dialect.getByName(sqlDialect);
}
@@ -4692,4 +4719,19 @@ public class SessionVariable implements Serializable,
Writable {
public boolean getDisableInvertedIndexV1ForVaraint() {
return disableInvertedIndexV1ForVaraint;
}
+
+ public void checkSqlConvertorFeatures(String features) {
+ if (Strings.isNullOrEmpty(features)) {
+ return;
+ }
+ String[] featureArray = features.split(",");
+ for (String feature : featureArray) {
+ if (!feature.equalsIgnoreCase("ctas")
+ && !feature.equalsIgnoreCase("delete_all_comment")) {
+ throw new UnsupportedOperationException("Unknown sql convertor
feature: " + feature
+ + ", current support: ctas, delete_all_comment");
+ }
+ }
+ }
}
+
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
index ac40c353769..de359f79475 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
@@ -52,22 +52,22 @@ public class HttpDialectUtilsTest {
public void testSqlConvert() {
String originSql = "select * from t1 where \"k1\" = 1";
String expectedSql = "select * from t1 where `k1` = 1";
-
+ String[] features = new String[] {"ctas"};
String targetURL = "http://127.0.0.1:"; + port + "/api/v1/convert";
- String res = HttpDialectUtils.convertSql(targetURL, originSql,
"presto");
+ String res = HttpDialectUtils.convertSql(targetURL, originSql,
"presto", features, "{}");
Assert.assertEquals(originSql, res);
// test presto
server.setResponse("{\"version\": \"v1\", \"data\": \"" + expectedSql
+ "\", \"code\": 0, \"message\": \"\"}");
- res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
+ res = HttpDialectUtils.convertSql(targetURL, originSql, "presto",
features, "{}");
Assert.assertEquals(expectedSql, res);
// test response version error
server.setResponse("{\"version\": \"v2\", \"data\": \"" + expectedSql
+ "\", \"code\": 0, \"message\": \"\"}");
- res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
+ res = HttpDialectUtils.convertSql(targetURL, originSql, "presto",
features, "{}");
Assert.assertEquals(originSql, res);
- // 7. test response code error
+ // test response code error
server.setResponse(
"{\"version\": \"v1\", \"data\": \"" + expectedSql + "\",
\"code\": 400, \"message\": \"\"}");
- res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
+ res = HttpDialectUtils.convertSql(targetURL, originSql, "presto",
features, "{}");
Assert.assertEquals(originSql, res);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
index b3e5f286108..39f76b86c64 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -292,4 +292,36 @@ public class VariableMgrTest {
Literal l = VariableMgr.getLiteral(sv, name, setType);
Assert.assertEquals(BigIntType.INSTANCE, l.getDataType());
}
+
+ @Test
+ public void testCheckSqlConvertorFeatures() throws DdlException {
+ // set wrong var
+ SetVar setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral("wrong"));
+ SessionVariable var = new SessionVariable();
+ try {
+ VariableMgr.setVar(var, setVar);
+ } catch (DdlException e) {
+ Assert.assertTrue(e.getMessage().contains("Unknown sql convertor
feature: wrong"));
+ }
+
+ // set one var
+ Assert.assertEquals(new String[] {""}, var.getSqlConvertorFeatures());
+ setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral("ctas"));
+ VariableMgr.setVar(var, setVar);
+ Assert.assertEquals(new String[] {"ctas"},
var.getSqlConvertorFeatures());
+
+ // set multiple var
+ setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral("ctas,delete_all_comment"));
+ VariableMgr.setVar(var, setVar);
+ Assert.assertEquals(new String[] {"ctas", "delete_all_comment"},
var.getSqlConvertorFeatures());
+
+ // set to empty
+ setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral(""));
+ VariableMgr.setVar(var, setVar);
+ Assert.assertEquals(new String[] {""}, var.getSqlConvertorFeatures());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
