This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 50be2de4389 [feature](statistics)Support analyze and drop single
partition stats. (#36031)
50be2de4389 is described below
commit 50be2de4389b5a4a128639f4220b0df7cee9f9bf
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Thu Jun 13 13:28:25 2024 +0800
[feature](statistics)Support analyze and drop single partition stats.
(#36031)
Support analyze a single partition and drop stats of a single partition.
```
analyze table table_name partition(p1,p2...)
drop stats table_name partition(p1, p2...)
```
---
.../org/apache/doris/analysis/AnalyzeTblStmt.java | 10 +--
.../org/apache/doris/analysis/DropStatsStmt.java | 38 ++++++----
.../apache/doris/service/FrontendServiceImpl.java | 12 ++-
.../org/apache/doris/statistics/AnalysisJob.java | 6 ++
.../apache/doris/statistics/AnalysisManager.java | 62 +++++++++++----
.../apache/doris/statistics/BaseAnalysisTask.java | 21 ++++--
.../doris/statistics/InvalidateStatsTarget.java | 6 +-
.../doris/statistics/StatisticsAutoCollector.java | 2 +
.../apache/doris/statistics/StatisticsCache.java | 20 +++--
.../doris/statistics/StatisticsRepository.java | 73 ++++++++++--------
.../apache/doris/statistics/TableStatsMeta.java | 2 -
.../suites/statistics/test_partition_stats.groovy | 88 ++++++++++++++++++----
12 files changed, 239 insertions(+), 101 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
index 083af1dad09..270c41d71bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
@@ -30,7 +30,6 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -242,13 +241,8 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
}
public Set<String> getPartitionNames() {
- if (partitionNames == null || partitionNames.getPartitionNames() ==
null) {
- if (table instanceof ExternalTable) {
- // External table couldn't return all partitions when
partitionNames is not set.
- // Because Analyze Table command for external table could
specify partition names.
- return Collections.emptySet();
- }
- return table.getPartitionNames();
+ if (partitionNames == null || partitionNames.getPartitionNames() ==
null || partitionNames.isStar()) {
+ return Collections.emptySet();
}
Set<String> partitions = Sets.newHashSet();
partitions.addAll(partitionNames.getPartitionNames());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
index 480dff2d94c..aa83a1de186 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
@@ -25,7 +25,6 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -45,13 +44,12 @@ import java.util.Set;
*/
public class DropStatsStmt extends DdlStmt {
+ public static final int MAX_IN_ELEMENT_TO_DELETE = 100;
public final boolean dropExpired;
private final TableName tableName;
private Set<String> columnNames;
private PartitionNames partitionNames;
- // Flag to drop external table row count in table_statistics.
- private boolean dropTableRowCount;
private boolean isAllColumns;
private long catalogId;
@@ -63,7 +61,6 @@ public class DropStatsStmt extends DdlStmt {
this.tableName = null;
this.columnNames = null;
this.partitionNames = null;
- this.dropTableRowCount = false;
}
public DropStatsStmt(TableName tableName,
@@ -72,11 +69,6 @@ public class DropStatsStmt extends DdlStmt {
this.partitionNames = partitionNames;
if (columnNames != null) {
this.columnNames = new HashSet<>(columnNames);
- this.dropTableRowCount = false;
- } else {
- // columnNames == null means drop all columns, in this case,
- // external table need to drop the table row count as well.
- dropTableRowCount = true;
}
dropExpired = false;
}
@@ -91,12 +83,11 @@ public class DropStatsStmt extends DdlStmt {
if (dropExpired) {
return;
}
+ if (tableName == null) {
+ throw new UserException("Should specify a valid table name.");
+ }
tableName.analyze(analyzer);
String catalogName = tableName.getCtl();
- if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(catalogName)) {
- // Internal table doesn't need to drop table row count.
- dropTableRowCount = false;
- }
String dbName = tableName.getDb();
String tblName = tableName.getTbl();
CatalogIf catalog = analyzer.getEnv().getCatalogMgr()
@@ -110,6 +101,9 @@ public class DropStatsStmt extends DdlStmt {
checkAnalyzePriv(catalogName, db.getFullName(), table.getName());
// check columnNames
if (columnNames != null) {
+ if (columnNames.size() > MAX_IN_ELEMENT_TO_DELETE) {
+ throw new UserException("Can't delete more that " +
MAX_IN_ELEMENT_TO_DELETE + " columns at one time.");
+ }
isAllColumns = false;
for (String cName : columnNames) {
if (table.getColumn(cName) == null) {
@@ -124,6 +118,10 @@ public class DropStatsStmt extends DdlStmt {
} else {
isAllColumns = true;
}
+ if (partitionNames != null && partitionNames.getPartitionNames() !=
null
+ && partitionNames.getPartitionNames().size() >
MAX_IN_ELEMENT_TO_DELETE) {
+ throw new UserException("Can't delete more that " +
MAX_IN_ELEMENT_TO_DELETE + " partitions at one time");
+ }
}
public long getTblId() {
@@ -146,8 +144,8 @@ public class DropStatsStmt extends DdlStmt {
return isAllColumns;
}
- public boolean dropTableRowCount() {
- return dropTableRowCount;
+ public PartitionNames getPartitionNames() {
+ return partitionNames;
}
@Override
@@ -165,6 +163,16 @@ public class DropStatsStmt extends DdlStmt {
sb.append(")");
}
+ if (partitionNames != null) {
+ sb.append(" PARTITION(");
+ if (partitionNames.isStar()) {
+ sb.append("*");
+ } else {
+ sb.append(StringUtils.join(partitionNames.getPartitionNames(),
","));
+ }
+ sb.append(")");
+ }
+
return sb.toString();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 29ff3ead06d..e05b126ca3b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.AddPartitionClause;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.LabelName;
import org.apache.doris.analysis.PartitionExprUtil;
+import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.RestoreStmt;
import org.apache.doris.analysis.SetType;
import org.apache.doris.analysis.TableName;
@@ -3177,8 +3178,8 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
ColStatsData data = GsonUtils.GSON.fromJson(request.colStatsData,
ColStatsData.class);
ColumnStatistic c = data.toColumnStatistic();
if (c == ColumnStatistic.UNKNOWN) {
- Env.getCurrentEnv().getStatisticsCache().invalidate(k.catalogId,
k.dbId, k.tableId,
- k.idxId, null, k.colName);
+
Env.getCurrentEnv().getStatisticsCache().invalidateColumnStatsCache(k.catalogId,
k.dbId, k.tableId,
+ k.idxId, k.colName);
} else {
Env.getCurrentEnv().getStatisticsCache().updateColStatsCache(
k.catalogId, k.dbId, k.tableId, k.idxId, k.colName, c);
@@ -3195,7 +3196,12 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
if (tableStats == null) {
return new TStatus(TStatusCode.OK);
}
- analysisManager.invalidateLocalStats(target.catalogId, target.dbId,
target.tableId, target.columns, tableStats);
+ PartitionNames partitionNames = null;
+ if (target.partitions != null) {
+ partitionNames = new PartitionNames(false, new
ArrayList<>(target.partitions));
+ }
+ analysisManager.invalidateLocalStats(target.catalogId, target.dbId,
target.tableId,
+ target.columns, tableStats, partitionNames);
return new TStatus(TStatusCode.OK);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
index d3734cf66dc..a2a094cbeca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
@@ -68,6 +68,12 @@ public class AnalysisJob {
this.analysisManager = Env.getCurrentEnv().getAnalysisManager();
}
+ public synchronized void taskDoneWithoutData(BaseAnalysisTask task) {
+ queryingTask.remove(task);
+ queryFinished.add(task);
+ markOneTaskDone();
+ }
+
public synchronized void appendBuf(BaseAnalysisTask task,
List<ColStatsData> statsData) {
queryingTask.remove(task);
buf.addAll(statsData);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 7e1123e422d..7ae4474c411 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.DropAnalyzeJobStmt;
import org.apache.doris.analysis.DropStatsStmt;
import org.apache.doris.analysis.KillAnalysisJobStmt;
+import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.ShowAnalyzeStmt;
import org.apache.doris.analysis.ShowAutoAnalyzeJobsStmt;
import org.apache.doris.analysis.TableName;
@@ -92,6 +93,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
@@ -320,7 +322,6 @@ public class AnalysisManager implements Writable {
long jobId = Env.getCurrentEnv().getNextId();
TableIf table = stmt.getTable();
Set<String> columnNames = stmt.getColumnNames();
- Set<String> partitionNames = stmt.getPartitionNames();
boolean partitionOnly = stmt.isPartitionOnly();
boolean isSamplingPartition = stmt.isSamplingPartition();
boolean isAllPartition = stmt.isStarPartition();
@@ -337,7 +338,7 @@ public class AnalysisManager implements Writable {
infoBuilder.setCatalogId(stmt.getCatalogId());
infoBuilder.setDBId(stmt.getDbId());
infoBuilder.setTblId(stmt.getTable().getId());
- infoBuilder.setPartitionNames(partitionNames);
+ infoBuilder.setPartitionNames(stmt.getPartitionNames());
infoBuilder.setPartitionOnly(partitionOnly);
infoBuilder.setSamplingPartition(isSamplingPartition);
infoBuilder.setAllPartition(isAllPartition);
@@ -648,6 +649,7 @@ public class AnalysisManager implements Writable {
}
Set<String> cols = dropStatsStmt.getColumnNames();
+ PartitionNames partitionNames = dropStatsStmt.getPartitionNames();
long catalogId = dropStatsStmt.getCatalogIdId();
long dbId = dropStatsStmt.getDbId();
long tblId = dropStatsStmt.getTblId();
@@ -655,10 +657,14 @@ public class AnalysisManager implements Writable {
if (tableStats == null) {
return;
}
- invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats);
+ invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats,
partitionNames);
// Drop stats ddl is master only operation.
- invalidateRemoteStats(catalogId, dbId, tblId, cols);
- StatisticsRepository.dropStatistics(catalogId, dbId, tblId, cols);
+ Set<String> partitions = null;
+ if (partitionNames != null && !partitionNames.isStar() &&
partitionNames.getPartitionNames() != null) {
+ partitions = new HashSet<>(partitionNames.getPartitionNames());
+ }
+ invalidateRemoteStats(catalogId, dbId, tblId, cols, partitions);
+ StatisticsRepository.dropStatistics(catalogId, dbId, tblId, cols,
partitions);
}
public void dropStats(TableIf table) throws DdlException {
@@ -669,24 +675,37 @@ public class AnalysisManager implements Writable {
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
- invalidateLocalStats(catalogId, dbId, tableId, null, tableStats);
+ invalidateLocalStats(catalogId, dbId, tableId, null, tableStats, null);
// Drop stats ddl is master only operation.
- invalidateRemoteStats(catalogId, dbId, tableId, null);
- StatisticsRepository.dropStatistics(catalogId, dbId, table.getId(),
null);
+ invalidateRemoteStats(catalogId, dbId, tableId, null, null);
+ StatisticsRepository.dropStatistics(catalogId, dbId, table.getId(),
null, null);
}
- public void invalidateLocalStats(long catalogId, long dbId, long tableId,
- Set<String> columns, TableStatsMeta
tableStats) {
+ public void invalidateLocalStats(long catalogId, long dbId, long tableId,
Set<String> columns,
+ TableStatsMeta tableStats, PartitionNames
partitionNames) {
if (tableStats == null) {
return;
}
TableIf table = StatisticsUtil.findTable(catalogId, dbId, tableId);
- StatisticsCache statisticsCache =
Env.getCurrentEnv().getStatisticsCache();
+ StatisticsCache statsCache = Env.getCurrentEnv().getStatisticsCache();
if (columns == null) {
columns = table.getSchemaAllIndexes(false)
.stream().map(Column::getName).collect(Collectors.toSet());
}
+ Set<String> partNames = new HashSet<>();
+ boolean allPartition = false;
+ if (table.isPartitionedTable()) {
+ if (partitionNames == null || partitionNames.isStar() ||
partitionNames.getPartitionNames() == null) {
+ partNames = table.getPartitionNames();
+ allPartition = true;
+ } else {
+ partNames = new HashSet<>(partitionNames.getPartitionNames());
+ }
+ } else {
+ allPartition = true;
+ }
+
for (String column : columns) {
List<Long> indexIds = Lists.newArrayList();
if (table instanceof OlapTable) {
@@ -704,8 +723,20 @@ public class AnalysisManager implements Writable {
indexName = olapTable.getIndexNameById(indexId);
}
}
- tableStats.removeColumn(indexName, column);
- statisticsCache.invalidate(catalogId, dbId, tableId, indexId,
null, column);
+ statsCache.invalidateColumnStatsCache(catalogId, dbId,
tableId, indexId, column);
+ if (allPartition) {
+ tableStats.removeColumn(indexName, column);
+ }
+ ColStatsMeta columnStatsMeta =
tableStats.findColumnStatsMeta(indexName, column);
+ for (String part : partNames) {
+ statsCache.invalidatePartitionColumnStatsCache(catalogId,
dbId, tableId, indexId, part, column);
+ if (columnStatsMeta != null &&
columnStatsMeta.partitionUpdateRows != null) {
+ Partition partition = table.getPartition(part);
+ if (partition != null) {
+
columnStatsMeta.partitionUpdateRows.remove(partition.getId());
+ }
+ }
+ }
}
}
tableStats.updatedTime = 0;
@@ -713,8 +744,9 @@ public class AnalysisManager implements Writable {
tableStats.rowCount = table.getRowCount();
}
- public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
Set<String> columns) {
- InvalidateStatsTarget target = new InvalidateStatsTarget(catalogId,
dbId, tableId, columns);
+ public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
+ Set<String> columns, Set<String>
partitions) {
+ InvalidateStatsTarget target = new InvalidateStatsTarget(catalogId,
dbId, tableId, columns, partitions);
TInvalidateFollowerStatsCacheRequest request = new
TInvalidateFollowerStatsCacheRequest();
request.key = GsonUtils.GSON.toJson(target);
StatisticsCache statisticsCache =
Env.getCurrentEnv().getStatisticsCache();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 830beac6d67..0f076013b03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -361,7 +361,8 @@ public abstract class BaseAnalysisTask {
deleteNotExistPartitionStats();
Map<String, String> params = buildSqlParams();
params.put("dataSizeFunction", getDataSizeFunction(col, false));
- Set<String> partitionNames = tbl.getPartitionNames();
+ boolean isAllPartitions = info.partitionNames.isEmpty();
+ Set<String> partitionNames = isAllPartitions ? tbl.getPartitionNames()
: info.partitionNames;
List<String> sqls = Lists.newArrayList();
int count = 0;
AnalysisManager analysisManager =
Env.getServingEnv().getAnalysisManager();
@@ -376,6 +377,7 @@ public abstract class BaseAnalysisTask {
// For sync job, get jobInfo from job.jobInfo.
boolean isSync = jobInfo == null;
jobInfo = isSync ? job.jobInfo : jobInfo;
+ StatisticsCache cache = Env.getCurrentEnv().getStatisticsCache();
for (String part : partitionNames) {
// External table partition is null.
Partition partition = tbl.getPartition(part);
@@ -409,6 +411,9 @@ public abstract class BaseAnalysisTask {
params.put("partitionInfo", getPartitionInfo(part));
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
sqls.add(stringSubstitutor.replace(PARTITION_ANALYZE_TEMPLATE));
+ // TODO: invalidate remote FE's cache.
+ cache.invalidatePartitionColumnStatsCache(
+ info.catalogId, info.dbId, info.tblId, info.indexId, part,
col.getName());
count++;
if (count == PARTITION_BATCH_SIZE) {
String sql = "INSERT INTO " +
StatisticConstants.FULL_QUALIFIED_PARTITION_STATS_TBL_NAME
@@ -434,11 +439,15 @@ public abstract class BaseAnalysisTask {
doSample();
}
} else {
- params = buildSqlParams();
- params.put("min", castToNumeric("min"));
- params.put("max", castToNumeric("max"));
- StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
- runQuery(stringSubstitutor.replace(MERGE_PARTITION_TEMPLATE));
+ if (isAllPartitions) {
+ params = buildSqlParams();
+ params.put("min", castToNumeric("min"));
+ params.put("max", castToNumeric("max"));
+ StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
+ runQuery(stringSubstitutor.replace(MERGE_PARTITION_TEMPLATE));
+ } else {
+ job.taskDoneWithoutData(this);
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
index 0108edd8228..c45db7553dc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
@@ -35,10 +35,14 @@ public class InvalidateStatsTarget {
@SerializedName("columns")
public final Set<String> columns;
- public InvalidateStatsTarget(long catalogId, long dbId, long tableId,
Set<String> columns) {
+ @SerializedName("partitions")
+ public final Set<String> partitions;
+
+ public InvalidateStatsTarget(long catalogId, long dbId, long tableId,
Set<String> columns, Set<String> partitions) {
this.catalogId = catalogId;
this.dbId = dbId;
this.tableId = tableId;
this.columns = columns;
+ this.partitions = partitions;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 9e237fe8040..5b02c1357cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -39,6 +39,7 @@ import org.apache.logging.log4j.Logger;
import java.time.LocalTime;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -215,6 +216,7 @@ public class StatisticsAutoCollector extends MasterDaemon {
.setAnalysisType(AnalysisInfo.AnalysisType.FUNDAMENTALS)
.setAnalysisMode(AnalysisInfo.AnalysisMode.INCREMENTAL)
.setAnalysisMethod(analysisMethod)
+ .setPartitionNames(Collections.emptySet())
.setSampleRows(analysisMethod.equals(AnalysisMethod.SAMPLE)
? StatisticsUtil.getHugeTableSampleRows() : -1)
.setScheduleType(ScheduleType.AUTOMATIC)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index 2680e29c683..9d5c15eff15 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -142,13 +142,21 @@ public class StatisticsCache {
return Optional.empty();
}
- public void invalidate(long ctlId, long dbId, long tblId, long idxId,
String partId, String colName) {
+ public void invalidateColumnStatsCache(long ctlId, long dbId, long tblId,
long idxId, String colName) {
+ columnStatisticsCache.synchronous().invalidate(new
StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName));
+ }
+
+ public void invalidatePartitionColumnStatsCache(long ctlId, long dbId,
long tblId, long idxId,
+ String partId, String
colName) {
if (partId == null) {
- columnStatisticsCache.synchronous().invalidate(new
StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName));
- } else {
- partitionColumnStatisticCache.synchronous().invalidate(
- new PartitionColumnStatisticCacheKey(ctlId, dbId, tblId,
idxId, partId, colName));
+ return;
}
+ partitionColumnStatisticCache.synchronous().invalidate(
+ new PartitionColumnStatisticCacheKey(ctlId, dbId, tblId, idxId,
partId, colName));
+ }
+
+ public void invalidateAllPartitionStatsCache() {
+ partitionColumnStatisticCache.synchronous().invalidateAll();
}
public void updateColStatsCache(long ctlId, long dbId, long tblId, long
idxId, String colName,
@@ -229,7 +237,7 @@ public class StatisticsCache {
statsId.idxId, statsId.colId);
ColumnStatistic columnStatistic = data.toColumnStatistic();
if (columnStatistic == ColumnStatistic.UNKNOWN) {
- invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, null,
k.colName);
+ invalidateColumnStatsCache(k.catalogId, k.dbId, k.tableId,
k.idxId, k.colName);
} else {
putCache(k, columnStatistic);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index 02b22504d3d..f84f0181d44 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -206,11 +206,11 @@ public class StatisticsRepository {
}
public static void dropStatistics(
- long ctlId, long dbId, long tblId, Set<String> colNames) throws
DdlException {
- if (colNames == null) {
+ long ctlId, long dbId, long tblId, Set<String> colNames,
Set<String> partNames) throws DdlException {
+ if (colNames == null && partNames == null) {
executeDropStatisticsAllColumnSql(ctlId, dbId, tblId);
} else {
- dropStatisticsByColName(ctlId, dbId, tblId, colNames);
+ dropStatisticsByColAndPartitionName(ctlId, dbId, tblId, colNames,
partNames);
}
}
@@ -229,43 +229,56 @@ public class StatisticsRepository {
}
}
- private static void dropStatisticsByColName(long ctlId, long dbId, long
tblId, Set<String> colNames)
+ private static void dropStatisticsByColAndPartitionName(long ctlId, long
dbId, long tblId,
+ Set<String> colNames,
Set<String> partNames)
throws DdlException {
Map<String, String> params = new HashMap<>();
- Iterator<String> iterator = colNames.iterator();
- int columnCount = 0;
- StringBuilder inPredicate = new StringBuilder();
- while (iterator.hasNext()) {
- inPredicate.append("'");
- inPredicate.append(iterator.next());
- inPredicate.append("'");
- inPredicate.append(",");
- columnCount++;
- if (columnCount == Config.max_allowed_in_element_num_of_delete) {
- executeDropStatisticsByColumnSql(inPredicate, ctlId, dbId,
tblId, params);
- columnCount = 0;
- inPredicate.setLength(0);
+ String columnCondition = "";
+ String partitionCondition = "";
+ if (colNames != null) {
+ Iterator<String> iterator = colNames.iterator();
+ StringBuilder inPredicate = new StringBuilder();
+ while (iterator.hasNext()) {
+ inPredicate.append("'");
+ inPredicate.append(iterator.next());
+ inPredicate.append("'");
+ inPredicate.append(",");
}
+ if (inPredicate.length() > 0) {
+ inPredicate.delete(inPredicate.length() - 1,
inPredicate.length());
+ }
+ columnCondition = String.format("AND %s IN (%s)", "col_id",
inPredicate);
}
- if (inPredicate.length() > 0) {
- executeDropStatisticsByColumnSql(inPredicate, ctlId, dbId, tblId,
params);
+ if (partNames != null) {
+ Iterator<String> iterator = partNames.iterator();
+ StringBuilder inPredicate = new StringBuilder();
+ while (iterator.hasNext()) {
+ inPredicate.append("'");
+ inPredicate.append(iterator.next());
+ inPredicate.append("'");
+ inPredicate.append(",");
+ }
+ if (inPredicate.length() > 0) {
+ inPredicate.delete(inPredicate.length() - 1,
inPredicate.length());
+ }
+ partitionCondition = String.format("AND %s IN (%s)", "part_name",
inPredicate);
}
+ executeDropStatisticsByColumnAndPartitionSql(
+ columnCondition, partitionCondition, ctlId, dbId, tblId,
params, partNames == null);
+
}
- private static void executeDropStatisticsByColumnSql(
- StringBuilder inPredicate, long ctlId, long dbId, long tblId,
Map<String, String> params)
- throws DdlException {
+ private static void executeDropStatisticsByColumnAndPartitionSql(String
columnCondition, String partitionCondition,
+ long ctlId, long dbId, long tblId, Map<String, String> params,
boolean tableLevel) throws DdlException {
generateCtlDbIdParams(ctlId, dbId, params);
params.put("tblId", String.valueOf(tblId));
- if (inPredicate.length() > 0) {
- inPredicate.delete(inPredicate.length() - 1, inPredicate.length());
- }
- String predicate = String.format("AND %s IN (%s)", "col_id",
inPredicate);
- params.put("columnCondition", predicate);
- params.put("partitionCondition", "");
+ params.put("columnCondition", columnCondition);
+ params.put("partitionCondition", partitionCondition);
try {
- StatisticsUtil.execUpdate(
- new
StringSubstitutor(params).replace(DELETE_TABLE_STATISTICS_BY_COLUMN_TEMPLATE));
+ if (tableLevel) {
+ StatisticsUtil.execUpdate(
+ new
StringSubstitutor(params).replace(DELETE_TABLE_STATISTICS_BY_COLUMN_TEMPLATE));
+ }
StatisticsUtil.execUpdate(
new
StringSubstitutor(params).replace(DELETE_PARTITION_STATISTICS_TEMPLATE));
} catch (Exception e) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index ba425352f6f..08f2f3e71f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -147,8 +147,6 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
colStatsMeta.rowCount = analyzedJob.rowCount;
if (colStatsMeta.partitionUpdateRows == null) {
colStatsMeta.partitionUpdateRows = new
ConcurrentHashMap<>();
- } else {
- colStatsMeta.partitionUpdateRows.clear();
}
colStatsMeta.partitionUpdateRows.putAll(analyzedJob.partitionUpdateRows);
}
diff --git a/regression-test/suites/statistics/test_partition_stats.groovy
b/regression-test/suites/statistics/test_partition_stats.groovy
index 7f57a7e17f4..b9239fd4d51 100644
--- a/regression-test/suites/statistics/test_partition_stats.groovy
+++ b/regression-test/suites/statistics/test_partition_stats.groovy
@@ -17,6 +17,7 @@
suite("test_partition_stats") {
+
def wait_row_count_reported = { db, table, row, column, expected ->
def result = sql """show frontends;"""
logger.info("show frontends result origin: " + result)
@@ -48,15 +49,15 @@ suite("test_partition_stats") {
}
- sql """drop database if exists test_partition_stats"""
- sql """create database test_partition_stats"""
- sql """use test_partition_stats"""
def enable = sql """show variables like "%enable_partition_analyze%" """
if (enable[0][1].equalsIgnoreCase("false")) {
logger.info("partition analyze disabled. " + enable)
return;
}
+ sql """drop database if exists test_partition_stats"""
+ sql """create database test_partition_stats"""
+ sql """use test_partition_stats"""
sql """CREATE TABLE `part` (
`id` INT NULL,
`colint` INT NULL,
@@ -89,11 +90,12 @@ suite("test_partition_stats") {
assertEquals(1, result.size())
assertEquals("18", result[0][0])
+ // Test show cached partition stats.
sql """analyze table part with sync;"""
result = sql """show column cached stats part(id) partition(p1)"""
assertEquals(1, result.size())
Thread.sleep(1000)
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 20; i++) {
result = sql """show column cached stats part(id) partition(p1)"""
if (result[0][3] == "6.0") {
logger.info("cache is ready.")
@@ -252,7 +254,7 @@ suite("test_partition_stats") {
assertEquals("12.0", result[7][2])
assertEquals("12.0", result[8][2])
- // Test skip big partitions and fallback to sample analyze
+ // Test analyze and drop single partition
sql """CREATE TABLE `part3` (
`id` INT NULL,
`colint` INT NULL,
@@ -277,15 +279,71 @@ suite("test_partition_stats") {
"replication_allocation" = "tag.location.default: 1"
)
"""
- sql """Insert into part3 values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2,
2, 2, 2, 2, 2.2, 2.2, 2.2), (3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4,
4, 4.4, 4.4, 4.4),(5, 5, 5, 5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6,
6.6, 6.6),(1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2, 2, 2, 2, 2, 2.2, 2.2, 2.2),
(3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4, 4, 4.4, 4.4, 4.4),(5, 5, 5,
5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6, 6.6, 6.6),(10001, 10001, 10001,
10001, 10001, 10001, 1 [...]
- wait_row_count_reported("test_partition_stats", "part3", 0, 4, "24")
- result = sql """show tablets from part3"""
+ sql """Insert into part3 values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2,
2, 2, 2, 2, 2.2, 2.2, 2.2), (3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4,
4, 4.4, 4.4, 4.4),(5, 5, 5, 5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6,
6.6, 6.6),(10001, 10001, 10001, 10001, 10001, 10001, 10001.10001, 10001.10001,
10001.10001),(10002, 10002, 10002, 10002, 10002, 10002, 10002.10002,
10002.10002, 10002.10002),(10003, 10003, 10003, 10003, 10003, 10003,
10003.10003, 10003.10003, 10003.10003),(100 [...]
+ sql """analyze table part3 partition(p1) with sync;"""
+ result = sql """show column stats part3 partition(*)"""
+ assertEquals(9, result.size())
+ assertEquals("p1", result[0][1])
+ assertEquals("p1", result[1][1])
+ assertEquals("p1", result[2][1])
+ assertEquals("p1", result[3][1])
+ assertEquals("p1", result[4][1])
+ assertEquals("p1", result[5][1])
+ assertEquals("p1", result[6][1])
+ assertEquals("p1", result[7][1])
+ assertEquals("p1", result[8][1])
+ result = sql """show column stats part3"""
+ assertEquals(0, result.size())
+ sql """analyze table part3 partition(*) with sync;"""
+ result = sql """show column stats part3 partition(*)"""
+ assertEquals(27, result.size())
+ result = sql """show column stats part3"""
+ assertEquals(9, result.size())
+ result = sql """drop stats part3 partition(p1)"""
+ result = sql """show column stats part3 partition(*)"""
+ assertEquals(18, result.size())
+ for (int i = 0; i < 18; i++) {
+ assertNotEquals('p1', result[i][1])
+ }
+ sql """drop stats part3 partition(*)"""
+ result = sql """show column stats part3 partition(*)"""
+ assertEquals(0, result.size())
+ result = sql """show column stats part3"""
+ assertEquals(0, result.size())
+
+ // Test skip big partitions and fallback to sample analyze
+ sql """CREATE TABLE `part4` (
+ `id` INT NULL,
+ `colint` INT NULL,
+ `coltinyint` tinyint NULL,
+ `colsmallint` smallINT NULL,
+ `colbigint` bigINT NULL,
+ `collargeint` largeINT NULL,
+ `colfloat` float NULL,
+ `coldouble` double NULL,
+ `coldecimal` decimal(27, 9) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE(`id`)
+ (
+ PARTITION p1 VALUES [("-2147483648"), ("10000")),
+ PARTITION p2 VALUES [("10000"), ("20000")),
+ PARTITION p3 VALUES [("20000"), ("30000"))
+ )
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ )"""
+ sql """Insert into part4 values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2,
2, 2, 2, 2, 2.2, 2.2, 2.2), (3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4,
4, 4.4, 4.4, 4.4),(5, 5, 5, 5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6,
6.6, 6.6),(1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1), (2, 2, 2, 2, 2, 2, 2.2, 2.2, 2.2),
(3, 3, 3, 3, 3, 3, 3.3, 3.3, 3.3),(4, 4, 4, 4, 4, 4, 4.4, 4.4, 4.4),(5, 5, 5,
5, 5, 5, 5.5, 5.5, 5.5),(6, 6, 6, 6, 6, 6, 6.6, 6.6, 6.6),(10001, 10001, 10001,
10001, 10001, 10001, 1 [...]
+ wait_row_count_reported("test_partition_stats", "part4", 0, 4, "24")
+ result = sql """show tablets from part4"""
logger.info("tablets: " + result)
sql """set global huge_partition_lower_bound_rows = 10"""
result = sql """ show variables like \"huge_partition_lower_bound_rows\""""
logger.info("huge partition bound: " + result)
- sql """analyze table part3 with sync;"""
- result = sql """show column stats part3"""
+ sql """analyze table part4 with sync;"""
+ result = sql """show column stats part4"""
logger.info("column result" + result)
assertEquals(9, result.size())
assertEquals("24.0", result[0][2])
@@ -297,17 +355,17 @@ suite("test_partition_stats") {
assertEquals("24.0", result[6][2])
assertEquals("24.0", result[7][2])
assertEquals("24.0", result[8][2])
- result = sql """show column stats part3 partition(*)"""
+ result = sql """show column stats part4 partition(*)"""
logger.info("partition result" + result)
assertEquals(18, result.size())
- result = sql """show column stats part3 partition(p1)"""
+ result = sql """show column stats part4 partition(p1)"""
assertEquals(0, result.size())
sql """set global huge_partition_lower_bound_rows = 100000000"""
- sql """analyze table part3 with sync;"""
- result = sql """show column stats part3 partition(*)"""
+ sql """analyze table part4 with sync;"""
+ result = sql """show column stats part4 partition(*)"""
assertEquals(27, result.size())
- result = sql """show column stats part3 partition(p1)"""
+ result = sql """show column stats part4 partition(p1)"""
assertEquals(9, result.size())
sql """drop database test_partition_stats"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
