This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b17e271f329 [fix](join) Handle oversized nested loop join blocks
(#65327)
b17e271f329 is described below
commit b17e271f329486387d1e850efd6c4e104fd139a2
Author: Mryange <[email protected]>
AuthorDate: Tue Jul 14 17:37:32 2026 +0800
[fix](join) Handle oversized nested loop join blocks (#65327)
### What problem does this PR solve?
Nested loop join may receive a single probe or build block whose row
count is larger than the runtime batch size. Some probe-side generation
loops checked whether the next appended rows would still fit in
`state->batch_size()` before entering the loop. When the input block
itself was oversized, those loops skipped the body entirely and did not
advance probe/build progress.
Root cause: the batch-size guard was used as a loop entry condition, so
oversized but otherwise valid blocks prevented the operator from
processing at least one chunk.
This change converts the affected nested loop join generation loops to
`do while`, so each call can make progress even when one input block is
larger than batch size. It also removes the hard debug check that
required `_join_block.rows()` to stay within batch size, because
oversized input can now legitimately produce an oversized output block
for that processing step.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../operator/nested_loop_join_probe_operator.cpp | 27 +++++++---------------
1 file changed, 8 insertions(+), 19 deletions(-)
diff --git a/be/src/exec/operator/nested_loop_join_probe_operator.cpp
b/be/src/exec/operator/nested_loop_join_probe_operator.cpp
index fd80d5471ce..03222de8f65 100644
--- a/be/src/exec/operator/nested_loop_join_probe_operator.cpp
+++ b/be/src/exec/operator/nested_loop_join_probe_operator.cpp
@@ -526,7 +526,8 @@ void
NestedLoopJoinProbeLocalState::_append_lazy_build_eval_columns(
bool
NestedLoopJoinProbeLocalState::_should_delay_lazy_probe_build_block(size_t
candidate_rows,
size_t batch_size) const {
- return _lazy_should_output_matched_rows() && _join_block.rows() +
candidate_rows > batch_size;
+ return _lazy_should_output_matched_rows() && _join_block.rows() > 0 &&
+ _join_block.rows() + candidate_rows > batch_size;
}
bool NestedLoopJoinProbeLocalState::_lazy_should_output_matched_rows() const {
@@ -694,7 +695,7 @@ Status
NestedLoopJoinProbeLocalState::_generate_lazy_block_base_build(RuntimeSta
}
size_t processed_rows = 0;
- while (processed_rows + probe_rows <= state->batch_size()) {
+ do {
if (_probe_block_pos == probe_rows) {
_current_build_row_pos++;
_probe_block_pos = 0;
@@ -738,7 +739,7 @@ Status
NestedLoopJoinProbeLocalState::_generate_lazy_block_base_build(RuntimeSta
*probe_block, build_block));
}
_probe_block_pos = probe_rows;
- }
+ } while (processed_rows + probe_rows <= state->batch_size());
return Status::OK();
}
@@ -758,7 +759,7 @@ void
NestedLoopJoinProbeLocalState::_generate_block_base_probe(RuntimeState* sta
return build_blocks[_current_build_pos].rows();
};
- while (_join_block.rows() + add_rows() <= state->batch_size()) {
+ do {
while (_current_build_pos == _shared_state->build_blocks.size() ||
_probe_block_pos == probe_block->rows()) {
// if probe block is empty(), do not need disprocess the probe
block rows
@@ -794,13 +795,7 @@ void
NestedLoopJoinProbeLocalState::_generate_block_base_probe(RuntimeState* sta
SCOPED_TIMER(_output_temp_blocks_timer);
process_probe_block(_probe_block_pos, _join_block, *probe_block,
p._num_probe_side_columns,
now_process_build_block,
p._num_build_side_columns);
- }
-
- DCHECK_LE(_join_block.rows(), state->batch_size())
- << "join block rows:" << _join_block.rows()
- << ", state batch size:" << state->batch_size()
- << "probe_block rows:" << probe_block->rows()
- << " build blocks size:" << _shared_state->build_blocks.size();
+ } while (_join_block.rows() + add_rows() <= state->batch_size());
}
// When the build side is small, generate data based on the build side.
@@ -833,7 +828,7 @@ void
NestedLoopJoinProbeLocalState::_generate_block_base_build(RuntimeState* sta
return;
}
- while (_join_block.rows() + probe_rows <= state->batch_size()) {
+ do {
// The current build row has processed the entire probe block; move to
the next build row
if (_probe_block_pos == probe_rows) {
// Move to the next build row and reset the probe position
@@ -865,13 +860,7 @@ void
NestedLoopJoinProbeLocalState::_generate_block_base_build(RuntimeState* sta
// Mark the current probe block as processed; the next loop will move
to the next build row
_probe_block_pos = probe_rows;
- }
-
- DCHECK_LE(_join_block.rows(), state->batch_size())
- << "join block rows:" << _join_block.rows()
- << ", state batch size:" << state->batch_size()
- << "probe_block rows:" << probe_block->rows()
- << "build block rows:" << build_block.rows();
+ } while (_join_block.rows() + probe_rows <= state->batch_size());
}
// for inner join only
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
