This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 19be696120a [Refactor](statistics)Refactor of statistics
buildConnectContext. (#41553) (#41814)
19be696120a is described below
commit 19be696120ab1e16df1120717251fadb6a64f030
Author: Jibing-Li <[email protected]>
AuthorDate: Mon Oct 14 22:09:57 2024 +0800
[Refactor](statistics)Refactor of statistics buildConnectContext. (#41553)
(#41814)
backport: https://github.com/apache/doris/pull/41553
---
.../doris/datasource/jdbc/JdbcExternalTable.java | 2 +-
.../java/org/apache/doris/qe/AuditLogHelper.java | 18 ++-
.../java/org/apache/doris/qe/ConnectProcessor.java | 2 +-
.../apache/doris/statistics/BaseAnalysisTask.java | 3 +-
.../apache/doris/statistics/OlapAnalysisTask.java | 140 ++++++++++-----------
.../doris/statistics/util/StatisticsUtil.java | 4 +-
.../doris/statistics/OlapAnalysisTaskTest.java | 6 +-
7 files changed, 90 insertions(+), 85 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
index 07ce183a589..20520d7c542 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -119,7 +119,7 @@ public class JdbcExternalTable extends ExternalTable {
params.put("tblName", name);
switch (((JdbcExternalCatalog) catalog).getDatabaseTypeName()) {
case JdbcResource.MYSQL:
- try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext()) {
+ try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext(false)) {
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
String sql =
stringSubstitutor.replace(MYSQL_ROW_COUNT_SQL);
StmtExecutor stmtExecutor = new
StmtExecutor(r.connectContext, sql);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
index ed532a83303..4f221fe42d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
@@ -86,18 +86,24 @@ public class AuditLogHelper {
.setFuzzyVariables(!printFuzzyVariables ? "" :
ctx.getSessionVariable().printFuzzyVariables());
if (ctx.getState().isQuery()) {
- MetricRepo.COUNTER_QUERY_ALL.increase(1L);
-
MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ if (!ctx.getSessionVariable().internalSession) {
+ MetricRepo.COUNTER_QUERY_ALL.increase(1L);
+
MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ }
if (ctx.getState().getStateType() == MysqlStateType.ERR
&& ctx.getState().getErrType() !=
QueryState.ErrType.ANALYSIS_ERR) {
// err query
- MetricRepo.COUNTER_QUERY_ERR.increase(1L);
-
MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ if (!ctx.getSessionVariable().internalSession) {
+ MetricRepo.COUNTER_QUERY_ERR.increase(1L);
+
MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd(ctx.getQualifiedUser()).increase(1L);
+ }
} else if (ctx.getState().getStateType() == MysqlStateType.OK
|| ctx.getState().getStateType() == MysqlStateType.EOF) {
// ok query
- MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
-
MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd(ctx.getQualifiedUser()).update(elapseMs);
+ if (!ctx.getSessionVariable().internalSession) {
+ MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
+
MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd(ctx.getQualifiedUser()).update(elapseMs);
+ }
if (elapseMs > Config.qe_slow_log_ms) {
String sqlDigest = DigestUtils.md5Hex(((Queriable)
parsedStmt).toDigest());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 3a5b35e00d0..d1538284b22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -222,7 +222,7 @@ public abstract class ConnectProcessor {
}
public void executeQuery(MysqlCommand mysqlCommand, String originStmt)
throws Exception {
- if (MetricRepo.isInit) {
+ if (MetricRepo.isInit && !ctx.getSessionVariable().internalSession) {
MetricRepo.COUNTER_REQUEST_ALL.increase(1L);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index a7aaaf2c037..31fae23284b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -290,7 +290,7 @@ public abstract class BaseAnalysisTask {
protected void runQuery(String sql) {
long startTime = System.currentTimeMillis();
String queryId = "";
- try (AutoCloseConnectContext a =
StatisticsUtil.buildConnectContext()) {
+ try (AutoCloseConnectContext a =
StatisticsUtil.buildConnectContext(info.jobType.equals(JobType.SYSTEM))) {
stmtExecutor = new StmtExecutor(a.connectContext, sql);
ColStatsData colStatsData = new
ColStatsData(stmtExecutor.executeInternalQuery().get(0));
// Update index row count after analyze.
@@ -316,5 +316,4 @@ public abstract class BaseAnalysisTask {
stmtExecutor = null;
}
}
-
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 5e5d6ac4740..298450b878a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -30,7 +30,6 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.StmtExecutor;
-import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
@@ -113,75 +112,73 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
String tabletStr = tabletIds.stream()
.map(Object::toString)
.collect(Collectors.joining(", "));
- try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext(info.jobType.equals(JobType.SYSTEM))) {
- // Get basic stats, including min and max.
- ResultRow basicStats = collectBasicStat(r);
- String min = StatisticsUtil.escapeSQL(basicStats != null &&
basicStats.getValues().size() > 0
- ? basicStats.get(0) : null);
- String max = StatisticsUtil.escapeSQL(basicStats != null &&
basicStats.getValues().size() > 1
- ? basicStats.get(1) : null);
+ // Get basic stats, including min and max.
+ ResultRow basicStats = collectBasicStat();
+ String min = StatisticsUtil.escapeSQL(basicStats != null &&
basicStats.getValues().size() > 0
+ ? basicStats.get(0) : null);
+ String max = StatisticsUtil.escapeSQL(basicStats != null &&
basicStats.getValues().size() > 1
+ ? basicStats.get(1) : null);
- boolean limitFlag = false;
- long rowsToSample = pair.second;
- Map<String, String> params = new HashMap<>();
- params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
- params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
- params.put("catalogId", String.valueOf(catalog.getId()));
- params.put("catalogName", catalog.getName());
- params.put("dbId", String.valueOf(db.getId()));
- params.put("tblId", String.valueOf(tbl.getId()));
- params.put("idxId", String.valueOf(info.indexId));
- params.put("colId",
StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
- params.put("dataSizeFunction", getDataSizeFunction(col, false));
- params.put("dbName", db.getFullName());
- params.put("colName",
StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
- params.put("tblName", String.valueOf(tbl.getName()));
- params.put("scaleFactor", String.valueOf(scaleFactor));
- params.put("sampleHints", tabletStr.isEmpty() ? "" :
String.format("TABLET(%s)", tabletStr));
- params.put("ndvFunction",
getNdvFunction(String.valueOf(totalRowCount)));
- params.put("min", StatisticsUtil.quote(min));
- params.put("max", StatisticsUtil.quote(max));
- params.put("rowCount", String.valueOf(totalRowCount));
- params.put("type", col.getType().toString());
- params.put("limit", "");
- params.put("index", getIndex());
- if (needLimit()) {
- // If the tablets to be sampled are too large, use limit to
control the rows to read, and re-calculate
- // the scaleFactor.
- rowsToSample = Math.min(getSampleRows(), pair.second);
- // Empty table doesn't need to limit.
- if (rowsToSample > 0) {
- limitFlag = true;
- params.put("limit", "limit " + rowsToSample);
- params.put("scaleFactor", String.valueOf(scaleFactor *
(double) pair.second / rowsToSample));
- }
+ boolean limitFlag = false;
+ long rowsToSample = pair.second;
+ Map<String, String> params = new HashMap<>();
+ params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
+ params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
+ params.put("catalogId", String.valueOf(catalog.getId()));
+ params.put("catalogName", catalog.getName());
+ params.put("dbId", String.valueOf(db.getId()));
+ params.put("tblId", String.valueOf(tbl.getId()));
+ params.put("idxId", String.valueOf(info.indexId));
+ params.put("colId",
StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
+ params.put("dataSizeFunction", getDataSizeFunction(col, false));
+ params.put("dbName", db.getFullName());
+ params.put("colName",
StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
+ params.put("tblName", String.valueOf(tbl.getName()));
+ params.put("scaleFactor", String.valueOf(scaleFactor));
+ params.put("sampleHints", tabletStr.isEmpty() ? "" :
String.format("TABLET(%s)", tabletStr));
+ params.put("ndvFunction",
getNdvFunction(String.valueOf(totalRowCount)));
+ params.put("min", StatisticsUtil.quote(min));
+ params.put("max", StatisticsUtil.quote(max));
+ params.put("rowCount", String.valueOf(totalRowCount));
+ params.put("type", col.getType().toString());
+ params.put("limit", "");
+ params.put("index", getIndex());
+ if (needLimit()) {
+ // If the tablets to be sampled are too large, use limit to
control the rows to read, and re-calculate
+ // the scaleFactor.
+ rowsToSample = Math.min(getSampleRows(), pair.second);
+ // Empty table doesn't need to limit.
+ if (rowsToSample > 0) {
+ limitFlag = true;
+ params.put("limit", "limit " + rowsToSample);
+ params.put("scaleFactor", String.valueOf(scaleFactor *
(double) pair.second / rowsToSample));
}
- StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
- String sql;
- if (useLinearAnalyzeTemplate()) {
- // For single unique key, use count as ndv.
- if (isSingleUniqueKey()) {
- params.put("ndvFunction", String.valueOf(totalRowCount));
- } else {
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
- }
- sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
+ }
+ StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
+ String sql;
+ if (useLinearAnalyzeTemplate()) {
+ // For single unique key, use count as ndv.
+ if (isSingleUniqueKey()) {
+ params.put("ndvFunction", String.valueOf(totalRowCount));
} else {
- params.put("dataSizeFunction", getDataSizeFunction(col, true));
- params.put("subStringColName", getStringTypeColName(col));
- sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
+ params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
}
- LOG.info("Sample for column [{}]. Total rows [{}], rows to sample
[{}], scale factor [{}], "
- + "limited [{}], distribute column [{}], partition column
[{}], key column [{}], "
- + "is single unique key [{}]",
- col.getName(), params.get("rowCount"), rowsToSample,
params.get("scaleFactor"),
- limitFlag, tbl.isDistributionColumn(col.getName()),
- tbl.isPartitionColumn(col.getName()), col.isKey(),
isSingleUniqueKey());
- runQuery(sql);
+ sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
+ } else {
+ params.put("dataSizeFunction", getDataSizeFunction(col, true));
+ params.put("subStringColName", getStringTypeColName(col));
+ sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
}
+ LOG.info("Sample for column [{}]. Total rows [{}], rows to sample
[{}], scale factor [{}], "
+ + "limited [{}], distribute column [{}], partition column
[{}], key column [{}], "
+ + "is single unique key [{}]",
+ col.getName(), params.get("rowCount"), rowsToSample,
params.get("scaleFactor"),
+ limitFlag, tbl.isDistributionColumn(col.getName()),
+ tbl.isPartitionColumn(col.getName()), col.isKey(),
isSingleUniqueKey());
+ runQuery(sql);
}
- protected ResultRow collectBasicStat(AutoCloseConnectContext context) {
+ protected ResultRow collectBasicStat() {
// Agg table value columns has no zone map.
// For these columns, skip collecting min and max value to avoid scan
whole table.
if (((OlapTable) tbl).getKeysType().equals(KeysType.AGG_KEYS) &&
!col.isKey()) {
@@ -197,14 +194,17 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("index", getIndex());
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql = stringSubstitutor.replace(BASIC_STATS_TEMPLATE);
- stmtExecutor = new StmtExecutor(context.connectContext, sql);
- ResultRow resultRow = stmtExecutor.executeInternalQuery().get(0);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Cost time in millisec: " + (System.currentTimeMillis()
- startTime)
- + " Min max SQL: " + sql + " QueryId: " +
DebugUtil.printId(stmtExecutor.getContext().queryId()));
+ ResultRow resultRow = null;
+ try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext(false)) {
+ stmtExecutor = new StmtExecutor(r.connectContext, sql);
+ resultRow = stmtExecutor.executeInternalQuery().get(0);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cost time in millisec: " +
(System.currentTimeMillis() - startTime) + " Min max SQL: "
+ + sql + " QueryId: " +
DebugUtil.printId(stmtExecutor.getContext().queryId()));
+ }
+ // Release the reference to stmtExecutor, reduce memory usage.
+ stmtExecutor = null;
}
- // Release the reference to stmtExecutor, reduce memory usage.
- stmtExecutor = null;
return resultRow;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index dc1dba5d981..25563bf6c39 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -142,7 +142,7 @@ public class StatisticsUtil {
public static QueryState execUpdate(String sql) throws Exception {
StmtExecutor stmtExecutor = null;
- AutoCloseConnectContext r = StatisticsUtil.buildConnectContext();
+ AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false);
try {
r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
stmtExecutor = new StmtExecutor(r.connectContext, sql);
@@ -209,7 +209,7 @@ public class StatisticsUtil {
}
public static void analyze(StatementBase statementBase) throws
UserException {
- try (AutoCloseConnectContext r = buildConnectContext()) {
+ try (AutoCloseConnectContext r = buildConnectContext(false)) {
Analyzer analyzer = new Analyzer(Env.getCurrentEnv(),
r.connectContext);
statementBase.analyze(analyzer);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index 5bb0920e433..a7283e351de 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -141,7 +141,7 @@ public class OlapAnalysisTaskTest {
}
@Mock
- public ResultRow collectBasicStat(AutoCloseConnectContext context)
{
+ public ResultRow collectBasicStat() {
List<String> values = Lists.newArrayList();
values.add("1");
values.add("2");
@@ -221,7 +221,7 @@ public class OlapAnalysisTaskTest {
}
@Mock
- public ResultRow collectBasicStat(AutoCloseConnectContext context)
{
+ public ResultRow collectBasicStat() {
List<String> values = Lists.newArrayList();
values.add("1");
values.add("2");
@@ -303,7 +303,7 @@ public class OlapAnalysisTaskTest {
}
@Mock
- public ResultRow collectBasicStat(AutoCloseConnectContext context)
{
+ public ResultRow collectBasicStat() {
List<String> values = Lists.newArrayList();
values.add("1");
values.add("2");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
