This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f1fd1b432fb [improvement](fe) Scan index change jobs once in the light
DROP INDEX pre-check (#69441)
f1fd1b432fb is described below
commit f1fd1b432fb4df3089c31e84cdfb6b4cccfe1999
Author: Jack <[email protected]>
AuthorDate: Sat Oct 24 16:34:31 2026 +0800
[improvement](fe) Scan index change jobs once in the light DROP INDEX
pre-check (#69441)
### What problem does this PR solve?
Issue Number: None
Related PR: #68674
Problem Summary:
The light `DROP INDEX` pre-check added in #68674
(`SchemaChangeHandler#checkNoIndexChangeJobOnPartitions`) calls
`hasIndexChangeJobOnPartition` once per partition, and every call copies
and scans the whole retained `indexChangeJobs` history (finished jobs
are kept for days). For a table with P partitions and a history of H
jobs that is O(P * H) work while the table write lock is held, on top of
the existing per-partition check in `buildOrDeleteTableInvertedIndices`.
A recent `BUILD INDEX` on a table with many partitions leaves at least
one record per partition, which makes it quadratic in the number of
partitions.
The pre-check also reported a conflict with the `BUILD INDEX` wording
`partition xxx has been built specified index. please check your build
stmt.`, which is misleading for a `DROP INDEX`.
Fix: the pre-check takes one snapshot of the job history and scans it
once, matching jobs on any of the table's partitions. The match
condition is extracted into `isConflictingIndexChangeJob` and shared
with `hasIndexChangeJobOnPartition`, so both checks keep the same rule.
The pre-check reports `partition xxx of table t has an unfinished index
change job, please retry after it finishes.` The `BUILD INDEX` / `DROP
INDEX ... PARTITION` check and its message are unchanged.
---
.../apache/doris/alter/SchemaChangeHandler.java | 35 ++++++++++++++--------
.../doris/alter/SchemaChangeHandlerTest.java | 4 +--
2 files changed, 24 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 5d8f17fbef4..7538e0a4f9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -3960,11 +3960,14 @@ public class SchemaChangeHandler extends AlterHandler {
// Throws if any of the partitions still has an unfinished index change
job on the base index.
private void checkNoIndexChangeJobOnPartitions(Database db, OlapTable
olapTable, List<Index> alterIndexes,
Set<String> partitionNames) throws DdlException {
- for (String partitionName : partitionNames) {
- if (hasIndexChangeJobOnPartition(olapTable.getBaseIndexId(),
db.getId(), olapTable.getId(),
- partitionName, alterIndexes, true)) {
- throw new DdlException("partition " + partitionName + " has
been built specified index."
- + " please check your build stmt.");
+ // Scan the retained job history once instead of once per partition,
the table write lock is held here.
+ for (IndexChangeJob indexChangeJob :
ImmutableList.copyOf(indexChangeJobs.values())) {
+ // Cheap partition check first, the predicate compares index lists.
+ if (partitionNames.contains(indexChangeJob.getPartitionName())
+ && isConflictingIndexChangeJob(indexChangeJob,
olapTable.getBaseIndexId(), db.getId(),
+ olapTable.getId(), alterIndexes, true)) {
+ throw new DdlException("partition " +
indexChangeJob.getPartitionName() + " of table "
+ + olapTable.getName() + " has an unfinished index
change job, please retry after it finishes.");
}
}
}
@@ -3974,20 +3977,26 @@ public class SchemaChangeHandler extends AlterHandler {
List<Index> alterIndexes, boolean isDrop) {
// TODO: this is temporary methods
for (IndexChangeJob indexChangeJob :
ImmutableList.copyOf(indexChangeJobs.values())) {
- if (indexChangeJob.getOriginIndexId() == originIndexId
- && indexChangeJob.getDbId() == dbId
- && indexChangeJob.getTableId() == tableId
- && indexChangeJob.getPartitionName().equals(partitionName)
- && ((Config.isNotCloudMode() &&
indexChangeJob.hasSameAlterInvertedIndex(isDrop, alterIndexes))
- || Config.isCloudMode())
- && !indexChangeJob.isDone()) {
- // if JobState is done (CANCELLED or FINISHED), also allow
user to create job again
+ if (indexChangeJob.getPartitionName().equals(partitionName)
+ && isConflictingIndexChangeJob(indexChangeJob,
originIndexId, dbId, tableId, alterIndexes,
+ isDrop)) {
return true;
}
}
return false;
}
+ private static boolean isConflictingIndexChangeJob(IndexChangeJob
indexChangeJob, long originIndexId,
+ long dbId, long tableId, List<Index> alterIndexes, boolean isDrop)
{
+ // if JobState is done (CANCELLED or FINISHED), also allow user to
create job again
+ return indexChangeJob.getOriginIndexId() == originIndexId
+ && indexChangeJob.getDbId() == dbId
+ && indexChangeJob.getTableId() == tableId
+ && ((Config.isNotCloudMode() &&
indexChangeJob.hasSameAlterInvertedIndex(isDrop, alterIndexes))
+ || Config.isCloudMode())
+ && !indexChangeJob.isDone();
+ }
+
// schema change job is not allowed when index change job is running
boolean hasIndexChangeJobOnTable(long dbId, long tableId) {
for (IndexChangeJob indexChangeJob :
ImmutableList.copyOf(indexChangeJobs.values())) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
index 1189f5a5903..b3ac191c658 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
@@ -1328,14 +1328,14 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
try {
String dropIndexStmt = "ALTER TABLE test." + tableName + " DROP
INDEX idx_v1";
Exception e = Assertions.assertThrows(Exception.class, () ->
alterTable(dropIndexStmt, connectContext));
- Assertions.assertTrue(e.getMessage().contains("has been built
specified index"), e.getMessage());
+ Assertions.assertTrue(e.getMessage().contains("has an unfinished
index change job"), e.getMessage());
Assertions.assertTrue(tbl.getIndexes().stream().anyMatch(idx ->
idx.getIndexName().equals("idx_v1")));
Assertions.assertEquals(schemaVersion,
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()).getSchemaVersion());
// The check must run before DROP COLUMN updates the bloom filter
columns in place.
String dropColumnAndIndexStmt = "ALTER TABLE test." + tableName +
" DROP COLUMN v2, DROP INDEX idx_v1";
e = Assertions.assertThrows(Exception.class, () ->
alterTable(dropColumnAndIndexStmt, connectContext));
- Assertions.assertTrue(e.getMessage().contains("has been built
specified index"), e.getMessage());
+ Assertions.assertTrue(e.getMessage().contains("has an unfinished
index change job"), e.getMessage());
Assertions.assertEquals(Sets.newHashSet("v2"),
tbl.getCopiedBfColumns());
Assertions.assertNotNull(tbl.getColumn("v2"));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
