This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 1330a0785d1 branch-4.1: [fix](fe) Finish the load job of an INSERT
whose source is planned empty #70649 (#70710)
1330a0785d1 is described below
commit 1330a0785d1e2149d354dfc82040a6aec7077df3
Author: Bei Zhang <[email protected]>
AuthorDate: Wed Nov 18 23:31:25 2026 -0800
branch-4.1: [fix](fe) Finish the load job of an INSERT whose source is
planned empty #70649 (#70710)
Cherry-pick of #70649 to branch-4.1. Two adaptations:
- `InsertIntoTableCommand.runInternal` on this branch still returns
early through
`isEmptyInsert()` / `shouldExecuteEmptyInsert()` instead of master's
`requiresTransaction()`, so
`onSkipExecution()` is called before that return.
- `OlapInsertExecutorTest` keeps this branch's `createExecutor(ctx)`
helper and only adds the
`needRegister` overload the new tests use.
### What problem does this PR solve?
Issue Number: #70632
Related PR: #70649
Problem Summary: An OLAP `INSERT INTO ... SELECT` whose source the
planner reduces to an empty set (for
example a LIST partition that does not exist, or `WHERE 1 = 0`)
registers an `InsertLoadJob` and then
returns before `afterExec` / `onFail`, so the job stays PENDING in the
master FE until it restarts. These
jobs never expire, leak heap, and once more than `label_num_threshold`
are stuck, every label-cleaner run
removes all finished load jobs. The fix records the registered job as
FINISHED with 0 rows on that
early-return path.
### Release note
Fix `INSERT INTO ... SELECT` statements whose source is planned empty
leaving PENDING load jobs in the
master FE, which leaked memory until restart.
### Check List (For Author)
- Test: Unit Test (OlapInsertExecutorTest) and Regression test
(load_p0/insert/test_insert_statistic),
carried over from #70649; not run locally on this branch
- Behavior changed: Yes. Such an INSERT now shows up in SHOW LOAD as
FINISHED instead of PENDING
- Does this need documentation: No
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---
.../commands/insert/AbstractInsertExecutor.java | 9 +++++
.../commands/insert/InsertIntoTableCommand.java | 1 +
.../plans/commands/insert/OlapInsertExecutor.java | 7 ++++
.../commands/insert/OlapInsertExecutorTest.java | 47 +++++++++++++++++++++-
.../load_p0/insert/test_insert_statistic.groovy | 32 +++++++++++++++
5 files changed, 95 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
index 3710358e699..1e8a24932e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
@@ -72,6 +72,7 @@ public abstract class AbstractInsertExecutor {
protected String errMsg = "";
protected Optional<InsertCommandContext> insertCtx;
protected final boolean emptyInsert;
+ protected final boolean loadJobRegistered;
protected long txnId = INVALID_TXN_ID;
/**
@@ -128,6 +129,7 @@ public abstract class AbstractInsertExecutor {
this.insertCtx = insertCtx;
this.emptyInsert = emptyInsert;
this.jobId = jobId;
+ this.loadJobRegistered = needRegister;
}
public void registerListener(InsertExecutorListener listener) {
@@ -188,6 +190,13 @@ public abstract class AbstractInsertExecutor {
*/
protected abstract void afterExec(StmtExecutor executor);
+ /**
+ * Called instead of executeSingleInsert when the insert needs no
transaction, i.e. the planner proved the
+ * source empty. Must close any bookkeeping opened in the constructor,
such as a registered load job.
+ */
+ public void onSkipExecution() {
+ }
+
protected final void execImpl(StmtExecutor executor) throws Exception {
String queryId = DebugUtil.printId(ctx.queryId());
coordinator.setLoadZeroTolerance(ctx.getSessionVariable().getEnableInsertStrict());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
index 2bd2ce18c4a..a7eb4d41a3f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
@@ -680,6 +680,7 @@ public class InsertIntoTableCommand extends Command
implements NeedAuditEncrypti
// if the insert stmt data source is empty, directly return, no need
to be executed.
if (insertExecutor.isEmptyInsert()) {
if (!insertExecutor.shouldExecuteEmptyInsert()) {
+ insertExecutor.onSkipExecution();
return;
}
if (insertExecutorListener != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
index 6cbc9d40c88..204ec7efa8d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
@@ -355,6 +355,13 @@ public class OlapInsertExecutor extends
AbstractInsertExecutor {
setReturnInfo();
}
+ @Override
+ public void onSkipExecution() {
+ if (loadJobRegistered) {
+ recordLoadJob(ctx.getCurrentUserIdentity());
+ }
+ }
+
protected void setReturnInfo() {
// {'label':'my_label1', 'status':'visible', 'txnId':'123'}
// {'label':'my_label1', 'status':'visible', 'txnId':'123'
'err':'error messages'}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java
index ea5318eedcb..4beeda778b3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutorTest.java
@@ -114,6 +114,47 @@ class OlapInsertExecutorTest {
}
}
+ @Test
+ void testSkipExecutionFinishesRegisteredLoadJob() throws Exception {
+ ConnectContext ctx = createExecutorContext();
+ LoadManager loadManager = Mockito.mock(LoadManager.class);
+ Env currentEnv = createCurrentEnv(loadManager);
+
+ try (MockedStatic<EnvFactory> envFactoryMock =
Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> envMock = Mockito.mockStatic(Env.class)) {
+ prepareFactoryMocks(envFactoryMock, envMock, createCoordinator(),
+ Mockito.mock(GlobalTransactionMgrIface.class),
Mockito.mock(TransactionState.class), currentEnv);
+ ctx.setEnv(currentEnv);
+
+ OlapInsertExecutor executor = createExecutor(ctx, true, true);
+ Mockito.verify(loadManager).addLoadJob(Mockito.any());
+ executor.onSkipExecution();
+
+
Mockito.verify(loadManager).recordFinishedLoadJob(Mockito.eq("label_test"),
Mockito.eq(-1L),
+ Mockito.eq("test_db"), Mockito.eq(2L),
Mockito.eq(EtlJobType.INSERT), Mockito.anyLong(),
+ Mockito.eq(""), Mockito.isNull(), Mockito.isNull(),
Mockito.eq(UserIdentity.ROOT),
+ Mockito.anyLong());
+ }
+ }
+
+ @Test
+ void testSkipExecutionRecordsNothingForUnregisteredLoadJob() throws
Exception {
+ ConnectContext ctx = createExecutorContext();
+ LoadManager loadManager = Mockito.mock(LoadManager.class);
+ Env currentEnv = createCurrentEnv(loadManager);
+
+ try (MockedStatic<EnvFactory> envFactoryMock =
Mockito.mockStatic(EnvFactory.class);
+ MockedStatic<Env> envMock = Mockito.mockStatic(Env.class)) {
+ prepareFactoryMocks(envFactoryMock, envMock, createCoordinator(),
+ Mockito.mock(GlobalTransactionMgrIface.class),
Mockito.mock(TransactionState.class), currentEnv);
+ ctx.setEnv(currentEnv);
+
+ createExecutor(ctx, true, false).onSkipExecution();
+
+ Mockito.verifyNoInteractions(loadManager);
+ }
+ }
+
@Test
void testPublishTimeoutCommittedModeReturnsOk() throws Exception {
ConnectContext ctx = createExecutorContext();
@@ -243,6 +284,10 @@ class OlapInsertExecutorTest {
// Create an executor with mocked table metadata because this test only
validates timeout result handling.
private OlapInsertExecutor createExecutor(ConnectContext ctx) {
+ return createExecutor(ctx, false, false);
+ }
+
+ private OlapInsertExecutor createExecutor(ConnectContext ctx, boolean
emptyInsert, boolean needRegister) {
Database database = Mockito.mock(Database.class);
Mockito.when(database.getFullName()).thenReturn("test_db");
Mockito.when(database.getId()).thenReturn(1L);
@@ -254,7 +299,7 @@ class OlapInsertExecutorTest {
Mockito.when(table.getId()).thenReturn(2L);
return new OlapInsertExecutor(ctx, table, "label_test",
Mockito.mock(NereidsPlanner.class),
- Optional.empty(), false, 0L);
+ Optional.empty(), emptyInsert, 0L, needRegister);
}
// Redirect coordinator creation and transaction access to mocks so the
test stays deterministic.
diff --git a/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
b/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
index ffc1263b5f4..4c152c16017 100644
--- a/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
+++ b/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
@@ -109,6 +109,38 @@ suite("test_insert_statistic", "p0") {
def cancelledJob = result.find { it[2] == "CANCELLED" }
assertNotNull(cancelledJob, "Expected a CANCELLED load job for failed
insert")
+ // insert into select whose source is planned as an empty set → job should
be FINISHED, not left PENDING
+ sql """ DROP TABLE IF EXISTS ${insert_tbl}_empty_src"""
+ sql """
+ CREATE TABLE ${insert_tbl}_empty_src (
+ `k1` varchar(16) NOT NULL,
+ `k2` int NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ AUTO PARTITION BY LIST (`k1`) ()
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES ("replication_num"="1");
+ """
+ sql """ INSERT INTO ${insert_tbl}_empty_src VALUES ('a', 1) """
+ explain {
+ sql """ SELECT * FROM ${insert_tbl}_empty_src WHERE k1 = 'b' """
+ contains "VEMPTYSET"
+ }
+ def emptyLabels = []
+ ["k1 = 'b'", "1 = 0"].each { predicate ->
+ def emptyLabel = "label_insert_empty_select_" + emptyLabels.size() +
"_" + System.currentTimeMillis()
+ sql """ INSERT INTO ${insert_tbl}_2 WITH LABEL ${emptyLabel}
+ SELECT k2, k2, k2, k2 FROM ${insert_tbl}_empty_src WHERE
${predicate} """
+ emptyLabels.add(emptyLabel)
+ }
+ emptyLabels.each { emptyLabel ->
+ result = sql "SHOW LOAD FROM ${dbName} WHERE LABEL = '${emptyLabel}'"
+ assertEquals(1, result.size())
+ assertEquals("FINISHED", result[0][2])
+ }
+ result = sql "SHOW LOAD FROM ${dbName}"
+ assertNull(result.find { it[2] == "PENDING" }, "No insert load job should
be left PENDING")
+
// insert into s3 tvf
String ak = getS3AK()
String sk = getS3SK()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]