This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4b4b3b92c8f [feat](sql-convertor) support enable sql convertor's
feature by session variable (#50707)
4b4b3b92c8f is described below
commit 4b4b3b92c8f1376b107712fc666263e6ac802989
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed May 14 14:04:24 2025 +0800
[feat](sql-convertor) support enable sql convertor's feature by session
variable (#50707)
### What problem does this PR solve?
Problem Summary:
Support new session variable `enable_sql_convertor_features`.
Some features of sql convertor is disable by default,
User can enable certain features of sql convertor by this variable.
eg:
```
// enable ctas statement conversion.
set enable_sql_convertor_features="ctas"
// enable ctas statement conversion, and will delete all comment in
statement
set enable_sql_convertor_features="ctas,delete_all_comment"
```
---
.../plugin/dialect/HttpDialectConverterPlugin.java | 3 +-
.../doris/plugin/dialect/HttpDialectUtils.java | 9 ++++--
.../java/org/apache/doris/qe/SessionVariable.java | 29 ++++++++++++++++++++
.../apache/doris/plugin/HttpDialectUtilsTest.java | 10 +++----
.../java/org/apache/doris/qe/VariableMgrTest.java | 32 ++++++++++++++++++++++
5 files changed, 74 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
index 518981e9b95..d2096cd992b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java
@@ -103,7 +103,8 @@ public class HttpDialectConverterPlugin extends Plugin
implements DialectConvert
if (Strings.isNullOrEmpty(targetURL)) {
return null;
}
- return HttpDialectUtils.convertSql(targetURL, originSql,
sessionVariable.getSqlDialect());
+ return HttpDialectUtils.convertSql(targetURL, originSql,
sessionVariable.getSqlDialect(),
+ sessionVariable.getSqlConvertorFeatures());
}
// no need to override parseSqlWithDialect, just return null
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
index 3cb6244d515..87b06c76125 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java
@@ -38,8 +38,9 @@ import java.nio.charset.StandardCharsets;
public class HttpDialectUtils {
private static final Logger LOG =
LogManager.getLogger(HttpDialectUtils.class);
- public static String convertSql(String targetURL, String originStmt,
String dialect) {
- ConvertRequest convertRequest = new ConvertRequest(originStmt,
dialect);
+ public static String convertSql(String targetURL, String originStmt,
String dialect,
+ String[] features) {
+ ConvertRequest convertRequest = new ConvertRequest(originStmt,
dialect, features);
HttpURLConnection connection = null;
try {
@@ -110,14 +111,16 @@ public class HttpDialectUtils {
private String to; // CHECKSTYLE IGNORE THIS LINE
private String source; // CHECKSTYLE IGNORE THIS LINE
private String case_sensitive; // CHECKSTYLE IGNORE THIS LINE
+ private String[] enable_sql_convertor_features; // CHECKSTYLE IGNORE
THIS LINE
- public ConvertRequest(String originStmt, String dialect) {
+ public ConvertRequest(String originStmt, String dialect, String[]
features) {
this.version = "v1";
this.sql_query = originStmt;
this.from = dialect;
this.to = "doris";
this.source = "text";
this.case_sensitive = "0";
+ this.enable_sql_convertor_features = features;
}
public String toJson() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index e3bae3f4c70..410cabf3727 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -732,6 +732,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_TEXT_VALIDATE_UTF8 =
"enable_text_validate_utf8";
+ public static final String ENABLE_SQL_CONVERTOR_FEATURES =
"enable_sql_convertor_features";
+
/**
* If set false, user couldn't submit analyze SQL and FE won't allocate
any related resources.
*/
@@ -2575,6 +2577,14 @@ public class SessionVariable implements Serializable,
Writable {
})
public boolean skipCheckingAcidVersionFile = false;
+ @VariableMgr.VarAttr(name = ENABLE_SQL_CONVERTOR_FEATURES, needForward =
true,
+ checker = "checkSqlConvertorFeatures",
+ description = {
+ "开启 SQL 转换器的指定功能。多个功能使用逗号分隔",
+ "enable SQL convertor features. Multiple features are
separated by commas"
+ })
+ public String enableSqlConvertorFeatures = "";
+
public void setEnableEsParallelScroll(boolean enableESParallelScroll) {
this.enableESParallelScroll = enableESParallelScroll;
}
@@ -3520,6 +3530,10 @@ public class SessionVariable implements Serializable,
Writable {
return sqlDialect;
}
+ public String[] getSqlConvertorFeatures() {
+ return enableSqlConvertorFeatures.split(",");
+ }
+
public Dialect getSqlParseDialect() {
return Dialect.getByName(sqlDialect);
}
@@ -4794,6 +4808,20 @@ public class SessionVariable implements Serializable,
Writable {
}
}
+ public void checkSqlConvertorFeatures(String features) {
+ if (Strings.isNullOrEmpty(features)) {
+ return;
+ }
+ String[] featureArray = features.split(",");
+ for (String feature : featureArray) {
+ if (!feature.equalsIgnoreCase("ctas")
+ && !feature.equalsIgnoreCase("delete_all_comment")) {
+ throw new UnsupportedOperationException("Unknown sql convertor
feature: " + feature
+ + ", current support: ctas, delete_all_comment");
+ }
+ }
+ }
+
public boolean getEnableLocalMergeSort() {
return enableLocalMergeSort;
}
@@ -4805,4 +4833,5 @@ public class SessionVariable implements Serializable,
Writable {
public boolean showSplitProfileInfo() {
return enableProfile() && getProfileLevel() > 1;
}
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
index ac40c353769..a25b595a0fc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java
@@ -52,22 +52,22 @@ public class HttpDialectUtilsTest {
public void testSqlConvert() {
String originSql = "select * from t1 where \"k1\" = 1";
String expectedSql = "select * from t1 where `k1` = 1";
-
+ String[] features = new String[] {"ctas"};
String targetURL = "http://127.0.0.1:"; + port + "/api/v1/convert";
- String res = HttpDialectUtils.convertSql(targetURL, originSql,
"presto");
+ String res = HttpDialectUtils.convertSql(targetURL, originSql,
"presto", features);
Assert.assertEquals(originSql, res);
// test presto
server.setResponse("{\"version\": \"v1\", \"data\": \"" + expectedSql
+ "\", \"code\": 0, \"message\": \"\"}");
- res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
+ res = HttpDialectUtils.convertSql(targetURL, originSql, "presto",
features);
Assert.assertEquals(expectedSql, res);
// test response version error
server.setResponse("{\"version\": \"v2\", \"data\": \"" + expectedSql
+ "\", \"code\": 0, \"message\": \"\"}");
- res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
+ res = HttpDialectUtils.convertSql(targetURL, originSql, "presto",
features);
Assert.assertEquals(originSql, res);
// 7. test response code error
server.setResponse(
"{\"version\": \"v1\", \"data\": \"" + expectedSql + "\",
\"code\": 400, \"message\": \"\"}");
- res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
+ res = HttpDialectUtils.convertSql(targetURL, originSql, "presto",
features);
Assert.assertEquals(originSql, res);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
index b3e5f286108..39f76b86c64 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -292,4 +292,36 @@ public class VariableMgrTest {
Literal l = VariableMgr.getLiteral(sv, name, setType);
Assert.assertEquals(BigIntType.INSTANCE, l.getDataType());
}
+
+ @Test
+ public void testCheckSqlConvertorFeatures() throws DdlException {
+ // set wrong var
+ SetVar setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral("wrong"));
+ SessionVariable var = new SessionVariable();
+ try {
+ VariableMgr.setVar(var, setVar);
+ } catch (DdlException e) {
+ Assert.assertTrue(e.getMessage().contains("Unknown sql convertor
feature: wrong"));
+ }
+
+ // set one var
+ Assert.assertEquals(new String[] {""}, var.getSqlConvertorFeatures());
+ setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral("ctas"));
+ VariableMgr.setVar(var, setVar);
+ Assert.assertEquals(new String[] {"ctas"},
var.getSqlConvertorFeatures());
+
+ // set multiple var
+ setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral("ctas,delete_all_comment"));
+ VariableMgr.setVar(var, setVar);
+ Assert.assertEquals(new String[] {"ctas", "delete_all_comment"},
var.getSqlConvertorFeatures());
+
+ // set to empty
+ setVar = new SetVar(SetType.SESSION,
SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
+ new StringLiteral(""));
+ VariableMgr.setVar(var, setVar);
+ Assert.assertEquals(new String[] {""}, var.getSqlConvertorFeatures());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
