This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1034d0a2d65 [fix](statistics) Quote identifiers in statistics queries
(#67703)
1034d0a2d65 is described below
commit 1034d0a2d655427a3cfab1cfc23e584895a54537
Author: morrySnow <[email protected]>
AuthorDate: Fri Sep 11 23:18:05 2026 +0800
[fix](statistics) Quote identifiers in statistics queries (#67703)
## Problem
Statistics collection builds internal SQL from catalog, database, table,
index, partition, and column names. Names containing an embedded
backtick could terminate a template-owned quoted identifier. Since
statistics statements execute through an internal privileged context,
the remaining name text could be interpreted as SQL and access objects
unavailable to the initiating user.
## Root cause
The SQL templates owned the surrounding backticks while callers
substituted raw names, or only partially escaped selected column names.
This split ownership was inconsistent across full, sampled, partition,
external-table, plugin-driven, and histogram collection paths. It also
made it easy for a newly added template expression to consume an
unescaped name.
## How to reproduce
1. Create a low-privilege user that can create and analyze tables in one
database but cannot read a table in another database.
2. In the permitted database, create a table whose quoted name contains
an embedded backtick followed by a crafted SQL fragment such as a
`UNION` query against the inaccessible table.
3. Trigger synchronous statistics collection for that table.
4. Before this change, the embedded backtick closes the template's
identifier and the remaining text is parsed in the privileged statistics
context. Values from the inaccessible table can consequently appear in
the attacker's statistics rows.
## Fix
- Build every object-name fragment with `SqlUtils.getIdentSql`, so
embedded backticks are doubled and the complete value remains one
identifier.
- Make templates consume already quoted identifier fragments instead of
adding their own backticks.
- Apply the same model to OLAP, external-table, plugin-driven sample,
partition, index, and histogram SQL generation.
- Keep metadata values used as string literals on a separate SQL-literal
escaping path.
- Add unit coverage for catalog, database, table, index, partition, and
column names containing embedded backticks, including a column name that
also contains an apostrophe.
## Tests
- `OlapAnalysisTaskTest`: 25 tests passed.
- `HistogramTaskTest`: 2 tests passed.
- `AnalyzeTest`: 2 tests passed.
- `BaseAnalysisTaskTest`: 3 tests passed.
- Re-triggered FE UT build: passed.
---
.../statistics/analysis/BaseAnalysisTask.java | 73 ++++++++++----------
.../statistics/analysis/ExternalAnalysisTask.java | 14 ++--
.../doris/statistics/analysis/HistogramTask.java | 17 ++---
.../statistics/analysis/OlapAnalysisTask.java | 25 +++----
.../analysis/PluginDrivenSampleAnalysisTask.java | 4 +-
.../doris/statistics/analysis/AnalyzeTest.java | 3 +
.../statistics/analysis/BaseAnalysisTaskTest.java | 6 +-
.../statistics/analysis/OlapAnalysisTaskTest.java | 79 ++++++++++++++++------
8 files changed, 134 insertions(+), 87 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java
index 52a1077aa8e..5bbb96e1134 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java
@@ -28,6 +28,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.qe.AuditLogHelper;
@@ -91,8 +92,8 @@ public abstract class BaseAnalysisTask {
public static final String ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER =
"ANALYZE_SKIP_LONG_STRING_COLUMN";
protected static final String FULL_ANALYZE_TEMPLATE = "WITH cte1 AS ("
- + "SELECT `${colName}`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}), "
+ + "SELECT ${colName}${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}), "
+ "cte2 AS ("
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS
`id`, "
+ "${catalogId} AS `catalog_id`, "
@@ -102,10 +103,10 @@ public abstract class BaseAnalysisTask {
+ "'${colId}' AS `col_id`, "
+ "NULL AS `part_id`, "
+ "COUNT(1) AS `row_count`, "
- + "NDV(`${colName}`) AS `ndv`, "
- + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS
`min`, "
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS
`max`, "
+ + "NDV(${colName}) AS `ndv`, "
+ + "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS
`min`, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS
`max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() "
+ "FROM cte1), "
@@ -116,9 +117,9 @@ public abstract class BaseAnalysisTask {
+ "as `hot_value` "
+ "FROM ("
+ "SELECT ${subStringColName} as `hash_value`, "
- + "MAX(`${colName}`) as `column_key`, "
+ + "MAX(${colName}) as `column_key`, "
+ "COUNT(1) AS `count` "
- + "FROM cte1 WHERE `${colName}` IS NOT NULL "
+ + "FROM cte1 WHERE ${colName} IS NOT NULL "
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT
${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";
@@ -131,19 +132,19 @@ public abstract class BaseAnalysisTask {
+ "'${colId}' AS `col_id`, "
+ "NULL AS `part_id`, "
+ "COUNT(1) AS `row_count`, "
- + "NDV(`${colName}`) AS `ndv`, "
- + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS
`min`, "
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS
`max`, "
+ + "NDV(${colName}) AS `ndv`, "
+ + "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS
`min`, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS
`max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() AS `update_time`, "
+ "null as `hot_value` "
- + "FROM (SELECT `${colName}`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index})
__lc_t";
+ + "FROM (SELECT ${colName}${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}) __lc_t";
protected static final String LINEAR_ANALYZE_TEMPLATE = "WITH cte1 AS ("
- + "SELECT `${colName}`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${sampleHints} ${limit} ${preAggHint}), "
+ + "SELECT ${colName}${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}
${sampleHints} ${limit} ${preAggHint}), "
+ "cte2 AS ("
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS
`id`, "
+ "${catalogId} AS `catalog_id`, "
@@ -154,7 +155,7 @@ public abstract class BaseAnalysisTask {
+ "NULL AS `part_id`, "
+ "${rowCount} AS `row_count`, "
+ "${ndvFunction} as `ndv`, "
- + "ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END)
* ${scaleFactor}) AS `null_count`, "
+ + "ROUND(SUM(CASE WHEN ${colName} IS NULL THEN 1 ELSE 0 END) *
${scaleFactor}) AS `null_count`, "
+ "SUBSTRING(CAST(${min} AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(${max} AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} * ${scaleFactor} AS `data_size`, "
@@ -166,9 +167,9 @@ public abstract class BaseAnalysisTask {
+ "as `hot_value` "
+ "FROM ("
+ "SELECT ${subStringColName} as `hash_value`, "
- + "MAX(`${colName}`) as `column_key`, "
+ + "MAX(${colName}) as `column_key`, "
+ "COUNT(1) AS `count` "
- + "FROM cte1 WHERE `${colName}` IS NOT NULL "
+ + "FROM cte1 WHERE ${colName} IS NOT NULL "
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT
${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";
@@ -177,9 +178,9 @@ public abstract class BaseAnalysisTask {
+ "FROM "
+ "(SELECT "
+ "${subStringColName} AS `hash_value`, "
- + "`${colName}` AS `col_value`, "
- + "LENGTH(`${colName}`) as `len`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${sampleHints} ${limit}) as `t0` "
+ + "${colName} AS `col_value`, "
+ + "LENGTH(${colName}) as `len`${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}
${sampleHints} ${limit}) as `t0` "
+ "${preAggHint} GROUP BY `t0`.`hash_value`), "
+ "cte2 AS ( "
+ "SELECT CONCAT('${tblId}', '-', '${idxId}', '-', '${colId}')
AS `id`, "
@@ -241,13 +242,13 @@ public abstract class BaseAnalysisTask {
+ "${partId} AS `part_id`, "
+ "'${colId}' AS `col_id`, "
+ "COUNT(1) AS `row_count`, "
- + "HLL_UNION(HLL_HASH(`${colName}`)) as ndv, "
- + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`,
"
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`,
"
+ + "HLL_UNION(HLL_HASH(${colName})) as ndv, "
+ + "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() AS `update_time` "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${partitionInfo}";
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}
${partitionInfo}";
protected static final String MERGE_PARTITION_TEMPLATE =
"SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
@@ -418,7 +419,7 @@ public abstract class BaseAnalysisTask {
}
} else {
if (column.getType().isStringType()) {
- return "SUM(LENGTH(`${colName}`))";
+ return "SUM(LENGTH(${colName}))";
} else {
return "COUNT(1) * " + column.getType().getSlotSize();
}
@@ -427,15 +428,15 @@ public abstract class BaseAnalysisTask {
protected String getStringTypeColName(Column column) {
if (column.getType().isStringType()) {
- return "xxhash_64(SUBSTRING(CAST(`${colName}` AS STRING), 1,
1024))";
+ return "xxhash_64(SUBSTRING(CAST(${colName} AS STRING), 1, 1024))";
} else {
- return "`${colName}`";
+ return "${colName}";
}
}
protected String getMinFunction() {
if (tableSample == null) {
- return "CAST(MIN(`${colName}`) as ${type}) ";
+ return "CAST(MIN(${colName}) as ${type}) ";
} else {
// Min value is not accurate while sample, so set it to NULL to
avoid optimizer generate bad plan.
return "NULL";
@@ -456,7 +457,7 @@ public abstract class BaseAnalysisTask {
// Max value is not accurate while sample, so set it to NULL to avoid
optimizer generate bad plan.
protected String getMaxFunction() {
if (tableSample == null) {
- return "CAST(MAX(`${colName}`) as ${type}) ";
+ return "CAST(MAX(${colName}) as ${type}) ";
} else {
return "NULL";
}
@@ -562,7 +563,7 @@ public abstract class BaseAnalysisTask {
}
}
}
- params.put("partName", "'" + StatisticsUtil.escapeColumnName(part)
+ "'");
+ params.put("partName",
StatisticsUtil.quote(StatisticsUtil.escapeSQL(part)));
params.put("partitionInfo", getPartitionInfo(part));
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
sqls.add(stringSubstitutor.replace(PARTITION_ANALYZE_TEMPLATE));
@@ -646,11 +647,11 @@ public abstract class BaseAnalysisTask {
protected void addLengthAssertParam(Map<String, String> params) {
long maxLen =
org.apache.doris.common.Config.statistics_max_string_column_length;
if (col != null && col.getType().isStringType() && maxLen > 0) {
- String escapedColName =
StatisticsUtil.escapeColumnName(String.valueOf(info.colName));
+ String quotedColName =
SqlUtils.getIdentSql(String.valueOf(info.colName));
// The StringSubstitutor used by callers already has ${colName}
populated,
- // so we inline the escaped column name directly here.
+ // so we inline the quoted column name directly here.
params.put("lengthAssert",
- ", assert_true(`" + escapedColName + "` IS NULL OR
LENGTH(`" + escapedColName + "`) <= "
+ ", assert_true(" + quotedColName + " IS NULL OR LENGTH(" +
quotedColName + ") <= "
+ maxLen + ", '" +
ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER + "') AS `__lc`");
} else {
params.put("lengthAssert", "");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java
index d9e4ce9237f..d21deaff687 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java
@@ -20,9 +20,11 @@ package org.apache.doris.statistics.analysis;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.NotImplementedException;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.StatisticConstants;
+import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.commons.text.StringSubstitutor;
@@ -68,7 +70,7 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
if (shouldCollectHotValue()) {
params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
params.put("subStringColName", getStringTypeColName(col));
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
template = FULL_ANALYZE_TEMPLATE;
} else {
template = FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE;
@@ -87,11 +89,11 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
params.put("idxId", "-1");
- params.put("colName", info.colName);
- params.put("colId", info.colName);
- params.put("catalogName", catalog.getName());
- params.put("dbName", db.getFullName());
- params.put("tblName", tbl.getName());
+ params.put("colName", SqlUtils.getIdentSql(info.colName));
+ params.put("colId", StatisticsUtil.escapeSQL(info.colName));
+ params.put("catalogName", SqlUtils.getIdentSql(catalog.getName()));
+ params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
+ params.put("tblName", SqlUtils.getIdentSql(tbl.getName()));
params.put("sampleHints", getSampleHint());
params.put("limit", "");
params.put("scaleFactor", "1");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java
index ed843c05cc9..46cdc0d3225 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java
@@ -20,6 +20,7 @@ package org.apache.doris.statistics.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.statistics.analysis.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.util.StatisticsUtil;
@@ -44,10 +45,10 @@ public class HistogramTask extends BaseAnalysisTask {
+ " ${idxId} AS idx_id, "
+ " '${colId}' AS col_id, "
+ " ${sampleRate} AS sample_rate, "
- + " HISTOGRAM(`${colName}`, ${maxBucketNum}) AS buckets, "
+ + " HISTOGRAM(${colName}, ${maxBucketNum}) AS buckets, "
+ " NOW() AS create_time "
+ "FROM "
- + " `${dbName}`.`${tblName}`";
+ + " ${dbName}.${tblName}";
public HistogramTask(AnalysisInfo info) {
super(info);
@@ -56,16 +57,16 @@ public class HistogramTask extends BaseAnalysisTask {
@Override
public void doExecute() throws Exception {
Map<String, String> params = new HashMap<>();
- params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
- params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME);
+ params.put("internalDB",
SqlUtils.getIdentSql(FeConstants.INTERNAL_DB_NAME));
+ params.put("histogramStatTbl",
SqlUtils.getIdentSql(StatisticConstants.HISTOGRAM_TBL_NAME));
params.put("catalogId", String.valueOf(catalog.getId()));
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
params.put("idxId", String.valueOf(info.indexId));
- params.put("colId", String.valueOf(info.colName));
- params.put("dbName", db.getFullName());
- params.put("tblName", tbl.getName());
- params.put("colName", String.valueOf(info.colName));
+ params.put("colId",
StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
+ params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
+ params.put("tblName", SqlUtils.getIdentSql(tbl.getName()));
+ params.put("colName",
SqlUtils.getIdentSql(String.valueOf(info.colName)));
params.put("sampleRate", getSampleRateFunction());
params.put("maxBucketNum", String.valueOf(info.maxBucketNum));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java
index 9644aadfb00..606a2424d10 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java
@@ -34,6 +34,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
@@ -67,9 +68,9 @@ import java.util.stream.Collectors;
public class OlapAnalysisTask extends BaseAnalysisTask {
private static final String BASIC_STATS_TEMPLATE = "SELECT "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) as min, "
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max "
- + "FROM `${dbName}`.`${tblName}` ${index}";
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) as min, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) as max "
+ + "FROM ${dbName}.${tblName} ${index}";
private boolean keyColumnSampleTooManyRows = false;
private boolean partitionColumnSampleTooManyRows = false;
@@ -365,12 +366,12 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("dataSizeFunction", getDataSizeFunction(col, true));
params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)");
} else {
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
// For single unique key, use count as ndv.
if (isSingleUniqueKey()) {
params.put("ndvFunction", String.valueOf(tableRowCount));
} else {
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
+ params.put("ndvFunction", "ROUND(NDV(${colName}) *
${scaleFactor})");
}
}
}
@@ -387,7 +388,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
if (shouldCollectHotValue()) {
params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
params.put("subStringColName", getStringTypeColName(col));
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE));
} else {
runQuery(stringSubstitutor.replace(FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE));
@@ -446,7 +447,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
@Override
protected String getPartitionInfo(String partitionName) {
- return "partition " + partitionName;
+ return "partition " + SqlUtils.getIdentSql(partitionName);
}
@Override
@@ -460,10 +461,10 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("idxId", String.valueOf(info.indexId));
params.put("colId",
StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
params.put("dataSizeFunction", getDataSizeFunction(col, false));
- params.put("catalogName", catalog.getName());
- params.put("dbName", db.getFullName());
- params.put("colName",
StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
- params.put("tblName", String.valueOf(tbl.getName()));
+ params.put("catalogName", SqlUtils.getIdentSql(catalog.getName()));
+ params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
+ params.put("colName",
SqlUtils.getIdentSql(String.valueOf(info.colName)));
+ params.put("tblName",
SqlUtils.getIdentSql(String.valueOf(tbl.getName())));
params.put("index", getIndex());
params.put("preAggHint", "");
addLengthAssertParam(params);
@@ -475,7 +476,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
return "";
} else {
OlapTable olapTable = (OlapTable) this.tbl;
- return "index `" + olapTable.getIndexNameById(info.indexId) + "`";
+ return "index " +
SqlUtils.getIdentSql(olapTable.getIndexNameById(info.indexId));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java
index 6f55158189f..91690df1eb0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java
@@ -89,9 +89,9 @@ public class PluginDrivenSampleAnalysisTask extends
ExternalAnalysisTask {
if (distributionColumns.size() == 1 &&
distributionColumns.contains(col.getName().toLowerCase())) {
bucketFlag = true;
sb.append(LINEAR_ANALYZE_TEMPLATE);
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
+ params.put("ndvFunction", "ROUND(NDV(${colName}) *
${scaleFactor})");
params.put("rowCount", "ROUND(COUNT(1) * ${scaleFactor})");
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
} else {
sb.append(DUJ1_ANALYZE_TEMPLATE);
params.put("subStringColName", getStringTypeColName(col));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java
index 4e9658d4f53..0b82a21bd0f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java
@@ -91,6 +91,9 @@ public class AnalyzeTest extends TestWithFeService {
Database database = Mockito.mock(Database.class);
OlapTable olapTable = Mockito.mock(OlapTable.class);
+ Mockito.when(catalog.getName()).thenReturn("internal");
+ Mockito.when(database.getFullName()).thenReturn("analysis_job_test");
+ Mockito.when(olapTable.getName()).thenReturn("t1");
Mockito.when(olapTable.getColumn(Mockito.anyString()))
.thenReturn(new Column("col1", PrimitiveType.INT));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java
index ad23fb6155b..597c23da2d5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java
@@ -39,7 +39,7 @@ public class BaseAnalysisTaskTest {
String dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column,
true);
Assertions.assertEquals("SUM(`column_length`)", dataSizeFunction);
dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, false);
- Assertions.assertEquals("SUM(LENGTH(`${colName}`))", dataSizeFunction);
+ Assertions.assertEquals("SUM(LENGTH(${colName}))", dataSizeFunction);
column = new Column("int_column", PrimitiveType.INT);
dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, false);
@@ -48,14 +48,14 @@ public class BaseAnalysisTaskTest {
Assertions.assertEquals("SUM(t1.count) * 4", dataSizeFunction);
String minFunction = olapAnalysisTask.getMinFunction();
- Assertions.assertEquals("CAST(MIN(`${colName}`) as ${type}) ",
minFunction);
+ Assertions.assertEquals("CAST(MIN(${colName}) as ${type}) ",
minFunction);
olapAnalysisTask.tableSample = new TableSample(true, 20L);
minFunction = olapAnalysisTask.getMinFunction();
Assertions.assertEquals("NULL", minFunction);
olapAnalysisTask.tableSample = null;
String maxFunction = olapAnalysisTask.getMaxFunction();
- Assertions.assertEquals("CAST(MAX(`${colName}`) as ${type}) ",
maxFunction);
+ Assertions.assertEquals("CAST(MAX(${colName}) as ${type}) ",
maxFunction);
olapAnalysisTask.tableSample = new TableSample(true, 20L);
maxFunction = olapAnalysisTask.getMaxFunction();
Assertions.assertEquals("NULL", maxFunction);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java
index c4c387e033e..b6976a5d41f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java
@@ -39,6 +39,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugPointUtil;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.SessionVariable;
@@ -142,6 +143,7 @@ public class OlapAnalysisTaskTest {
Mockito.when(catalogIf.getId()).thenReturn(10001L);
Mockito.when(catalogIf.getName()).thenReturn("catalogName");
Mockito.when(databaseIf.getId()).thenReturn(20001L);
+ Mockito.when(databaseIf.getFullName()).thenReturn("dbName");
OlapAnalysisTask olapAnalysisTask = Mockito.spy(new
OlapAnalysisTask());
Mockito.doReturn(new ResultRow(Lists.newArrayList("1",
"2"))).when(olapAnalysisTask).collectMinMax();
@@ -153,7 +155,7 @@ public class OlapAnalysisTaskTest {
Mockito.doReturn(true).when(olapAnalysisTask).useLinearAnalyzeTemplate();
Mockito.doAnswer(inv -> {
String sql = inv.getArgument(0);
- Assertions.assertEquals("WITH cte1 AS (SELECT `null` FROM
`catalogName`.`${dbName}`.`null` "
+ Assertions.assertEquals("WITH cte1 AS (SELECT `null` FROM
`catalogName`.`dbName`.`null` "
+ "${sampleHints} ${limit} ), cte2 AS (SELECT
CONCAT(30001, '-', -1, '-', 'null') AS `id`, "
+ "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS
`tbl_id`, -1 AS `idx_id`, "
+ "'null' AS `col_id`, NULL AS `part_id`, ${rowCount} AS
`row_count`, ${ndvFunction} as `ndv`, "
@@ -185,7 +187,7 @@ public class OlapAnalysisTaskTest {
String sql = inv.getArgument(0);
Assertions.assertEquals("WITH cte1 AS (SELECT MAX(t0.`col_value`)
as `col_value`, COUNT(1) as `count`,"
+ " SUM(`len`) as `column_length` FROM (SELECT
${subStringColName} AS `hash_value`, "
- + "`null` AS `col_value`, LENGTH(`null`) as `len` FROM
`catalogName`.`${dbName}`.`null` "
+ + "`null` AS `col_value`, LENGTH(`null`) as `len` FROM
`catalogName`.`dbName`.`null` "
+ "${sampleHints} ${limit}) as `t0` GROUP BY
`t0`.`hash_value`), "
+ "cte2 AS ( SELECT CONCAT('30001', '-', '-1', '-',
'null') AS `id`, 10001 AS `catalog_id`, "
+ "20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`,
'null' AS `col_id`, NULL AS `part_id`, "
@@ -320,9 +322,9 @@ public class OlapAnalysisTaskTest {
new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null));
Assertions.assertEquals("1", params.get("scaleFactor"));
Assertions.assertEquals("", params.get("sampleHints"));
- Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE ${colName}
IS NOT NULL)",
params.get("rowCount2"));
- Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})",
params.get("ndvFunction"));
Assertions.assertNull(params.get("preAggHint"));
Assertions.assertEquals("COUNT(1)", params.get("rowCount"));
params.clear();
@@ -332,9 +334,9 @@ public class OlapAnalysisTaskTest {
new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
- Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE ${colName}
IS NOT NULL)",
params.get("rowCount2"));
- Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})",
params.get("ndvFunction"));
params.clear();
// DUJ1 algorithm with sample tablets: rowCount2 and ndvFunction must
reference the cte1
@@ -366,7 +368,7 @@ public class OlapAnalysisTaskTest {
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("SUM(`t1`.`count`) * COUNT(`t1`.`col_value`) /
(SUM(`t1`.`count`) - SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null,
1, 0)) + SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null, 1, 0)) *
SUM(`t1`.`count`) / 1000)", params.get("ndvFunction"));
Assertions.assertEquals("SUM(t1.count) * 4",
params.get("dataSizeFunction"));
- Assertions.assertEquals("`${colName}`",
params.get("subStringColName"));
+ Assertions.assertEquals("${colName}", params.get("subStringColName"));
Assertions.assertEquals("/*+PREAGGOPEN*/", params.get("preAggHint"));
params.clear();
@@ -417,7 +419,7 @@ public class OlapAnalysisTaskTest {
Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
- Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})",
params.get("ndvFunction"));
params.clear();
task = Mockito.spy(new OlapAnalysisTask());
@@ -774,6 +776,43 @@ public class OlapAnalysisTaskTest {
}
}
+ @Test
+ public void testBuildSqlParamsQuotesIdentifiers() {
+ CatalogIf catalog = Mockito.mock(CatalogIf.class);
+ DatabaseIf database = Mockito.mock(DatabaseIf.class);
+ OlapTable table = Mockito.mock(OlapTable.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ Mockito.when(catalog.getName()).thenReturn("cat`alog");
+ Mockito.when(database.getId()).thenReturn(2L);
+ Mockito.when(database.getFullName()).thenReturn("db`name");
+ Mockito.when(table.getId()).thenReturn(3L);
+ Mockito.when(table.getName()).thenReturn("table`name");
+ Mockito.when(table.getIndexNameById(4L)).thenReturn("index`name");
+
+ OlapAnalysisTask task = new OlapAnalysisTask();
+ task.catalog = catalog;
+ task.db = database;
+ task.tbl = table;
+ task.col = new Column("col`'name", PrimitiveType.INT);
+ task.info = new AnalysisInfoBuilder()
+ .setIndexId(4L)
+ .setColName("col`'name")
+ .build();
+
+ Map<String, String> params = task.buildSqlParams();
+ Assertions.assertEquals("`cat``alog`", params.get("catalogName"));
+ Assertions.assertEquals("`db``name`", params.get("dbName"));
+ Assertions.assertEquals("`table``name`", params.get("tblName"));
+ Assertions.assertEquals("`col``'name`", params.get("colName"));
+ Assertions.assertEquals("col`''name", params.get("colId"));
+ Assertions.assertEquals("index `index``name`", params.get("index"));
+ Assertions.assertEquals("partition `part``name`",
task.getPartitionInfo("part`name"));
+
+ String sql = new
StringSubstitutor(params).replace(BaseAnalysisTask.FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE);
+ Assertions.assertTrue(sql.contains("FROM (SELECT `col``'name` FROM "
+ + "`cat``alog`.`db``name`.`table``name` index `index``name`)
__lc_t"), sql);
+ }
+
@Test
public void testFullAnalyzeTemplateRendersLengthAssert() {
// Confirm the rendered FULL_ANALYZE_TEMPLATE wraps the base table in
a subquery
@@ -787,10 +826,10 @@ public class OlapAnalysisTaskTest {
params.put("idxId", "3");
params.put("colId", "s");
params.put("dataSizeFunction", "100");
- params.put("catalogName", "internal");
- params.put("dbName", "db1");
- params.put("colName", "s");
- params.put("tblName", "tbl1");
+ params.put("catalogName", SqlUtils.getIdentSql("internal"));
+ params.put("dbName", SqlUtils.getIdentSql("db1"));
+ params.put("colName", SqlUtils.getIdentSql("s"));
+ params.put("tblName", SqlUtils.getIdentSql("tbl1"));
params.put("index", "");
params.put("lengthAssert",
", assert_true(`s` IS NULL OR LENGTH(`s`) <= 1024, '"
@@ -814,10 +853,10 @@ public class OlapAnalysisTaskTest {
params.put("idxId", "3");
params.put("colId", "id");
params.put("dataSizeFunction", "100");
- params.put("catalogName", "internal");
- params.put("dbName", "db1");
- params.put("colName", "id");
- params.put("tblName", "tbl1");
+ params.put("catalogName", SqlUtils.getIdentSql("internal"));
+ params.put("dbName", SqlUtils.getIdentSql("db1"));
+ params.put("colName", SqlUtils.getIdentSql("id"));
+ params.put("tblName", SqlUtils.getIdentSql("tbl1"));
params.put("index", "");
params.put("lengthAssert", "");
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
@@ -836,11 +875,11 @@ public class OlapAnalysisTaskTest {
params.put("tblId", "2");
params.put("idxId", "3");
params.put("colId", "col1");
- params.put("colName", "col1");
+ params.put("colName", SqlUtils.getIdentSql("col1"));
params.put("dataSizeFunction", "SUM(LENGTH(`col1`))");
- params.put("catalogName", "internal");
- params.put("dbName", "db1");
- params.put("tblName", "tbl1");
+ params.put("catalogName", SqlUtils.getIdentSql("internal"));
+ params.put("dbName", SqlUtils.getIdentSql("db1"));
+ params.put("tblName", SqlUtils.getIdentSql("tbl1"));
params.put("index", "");
params.put("hotValueCollectCount", "10");
params.put("subStringColName", "`col1`");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
