[doris] branch branch-1.1-lts updated: [Fix](Schema Change) fix schema change fail with memory alloc error when sorting #16867

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
     new ff30b13550 [Fix](Schema Change) fix schema change fail with memory 
alloc error when sorting #16867
ff30b13550 is described below

commit ff30b135505942644429fabc374c6851c160222a
Author: GoGoWen <82132356+gogo...@users.noreply.github.com>
AuthorDate: Fri Feb 17 15:29:59 2023 +0800

    [Fix](Schema Change) fix schema change fail with memory alloc error when 
sorting #16867
    
    lease memory check in sorting step to make schema change succeed. 
currently, the schema change may fail in sorting step as memory tracking is not 
accurate.
    Co-authored-by: yangbowen22 <yangbowe...@jd.com>
---
 be/src/olap/schema_change.cpp | 15 ++++++++-------
 be/src/olap/schema_change.h   |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 799a0ccdf7..a57b270f20 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -699,7 +699,7 @@ bool RowBlockSorter::sort(RowBlock** row_block) {
             _swap_row_block = nullptr;
         }
 
-        if (_row_block_allocator->allocate(&_swap_row_block, row_num, 
null_supported) !=
+        if (_row_block_allocator->allocate(&_swap_row_block, row_num, 
null_supported, true) !=
             OLAP_SUCCESS) {
             LOG(WARNING) << "fail to allocate memory.";
             return false;
@@ -764,15 +764,16 @@ RowBlockAllocator::~RowBlockAllocator() {
     }
 }
 
-OLAPStatus RowBlockAllocator::allocate(RowBlock** row_block, size_t num_rows, 
bool null_supported) {
+OLAPStatus RowBlockAllocator::allocate(RowBlock** row_block, size_t num_rows, 
bool null_supported, bool allow_overflow) {
     size_t row_block_size = _row_len * num_rows;
 
-    if (_memory_limitation > 0 &&
-        _mem_tracker->consumption() + row_block_size > _memory_limitation) {
-        *row_block = nullptr;
-        return OLAP_ERR_FETCH_MEMORY_EXCEEDED;
+    if (!allow_overflow) {
+        if (_memory_limitation > 0 &&
+            _mem_tracker->consumption() + row_block_size > _memory_limitation) 
{
+            *row_block = nullptr;
+            return OLAP_ERR_FETCH_MEMORY_EXCEEDED;
+        }
     }
-
     // TODO(lijiao) : Why abandon the original m_row_block_buffer
     *row_block = new (nothrow) RowBlock(&_tablet_schema);
 
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 1916574f5d..a294c7128e 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -80,7 +80,7 @@ public:
                       size_t memory_limitation);
     virtual ~RowBlockAllocator();
 
-    OLAPStatus allocate(RowBlock** row_block, size_t num_rows, bool 
null_supported);
+    OLAPStatus allocate(RowBlock** row_block, size_t num_rows, bool 
null_supported, bool allow_overflow=false);
     void release(RowBlock* row_block);
     bool is_memory_enough_for_sorting(size_t num_rows, size_t allocated_rows);
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org