date:20240925

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776202806


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;
+
+public static class BlackListInfo {
+public BlackListInfo() {}
+
+private Lock lock = new ReentrantLock();
+private String reasonForBlackList = "";
+private Long firstRecordBlackTimestampMs = 0L;
+private Long lastRecordBlackTimestampMs = 0L;
+private Long lastBlackTimestampMs = 0L;
+private Long recordBlackListCount = 0L;
+private Long backendID = 0L;
+
+public void tryAddBlackList(String reason) {
+lock.lock();
+try {
+recordAddBlackList(reason);
+if (shuoldBeBlackListed()) {
+doAddBlackList();
+LOG.info("Backend is added to black list.\n{}", 
getDebugInfo());
+}
+} finally {
+lock.unlock();
+}
+}
+
+private void recordAddBlackList(String reason) {
+if (firstRecordBlackTimestampMs <= 0) {
+firstRecordBlackTimestampMs = System.currentTimeMillis();
+}
+
+lastRecordBlackTimestampMs = System.currentTimeMillis();
+recordBlackListCount++;
+
+if (Strings.isNullOrEmpty(reasonForBlackList) && 
!Strings.isNullOrEmpty(reason)) {
+reasonForBlackList = reason;
+}
+}
+
+private boolean shuoldBeBlackListed() {
+if (lastRecordBlackTimestampMs <= 0 || firstRecordBlackTimestampMs 
<= 0) {
+return false;
+}
+if (recordBlackListCount < RecordBlackListThreshold) {
+return false;
+}
+
+if (lastRecordBlackTimestampMs - firstRecordBlackTimestampMs
+>= Config.do_add_backend_black_list_threshold_secs * 1000) 
{
+return false;
+}
+
+return true;
+}
+
+private void doAddBlackList() {
+lastBlackTimestampMs = System.currentTimeMillis();
+Exception e = new Exception();
+String stack = 
org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(e);
+LOG.warn("Add backend {} to black list, reason: {}. Stack {}", 
backendID, reasonForBlackList, stack);
+}
+
+public boolean shuoldBeRemoved() {
+lock.lock();
+try {
+if (lastBlackTimestampMs <= 0) {
+return false;
+}
+
+Long currentTimeStamp = System.currentTimeMillis();
+// If this backend has not been recorded as black for more 
than 10 secs, then regard it as normal
+if (currentTimeStamp - lastBlackTimestampMs
+>= Config.stay_in_backend_black_list_threshold_secs * 
1000) {
+return true;
+} else {
+return false;
+}
+} finally {
+lock.unlock();
+}
+}
+
+public boolean isBlacked() {
+lock.lock();
+try {
+return lastBlackTimestampMs > 0;
+} finally {
+lock.unlock();
+}
+}
+
+public String getReasonForBlackList() {
+lock.lock();
+try {
+return reasonForBlackList;
+} finally {
+lock.unlock();
+}
+}
+
+public String getDebugInfo() {

Review Comment:
   maybe toString?



-- 
This is an automated message from th

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776202648


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;
+
+public static class BlackListInfo {
+public BlackListInfo() {}
+
+private Lock lock = new ReentrantLock();
+private String reasonForBlackList = "";
+private Long firstRecordBlackTimestampMs = 0L;
+private Long lastRecordBlackTimestampMs = 0L;
+private Long lastBlackTimestampMs = 0L;
+private Long recordBlackListCount = 0L;
+private Long backendID = 0L;
+
+public void tryAddBlackList(String reason) {
+lock.lock();
+try {
+recordAddBlackList(reason);
+if (shuoldBeBlackListed()) {
+doAddBlackList();
+LOG.info("Backend is added to black list.\n{}", 
getDebugInfo());
+}
+} finally {
+lock.unlock();
+}
+}
+
+private void recordAddBlackList(String reason) {
+if (firstRecordBlackTimestampMs <= 0) {
+firstRecordBlackTimestampMs = System.currentTimeMillis();
+}
+
+lastRecordBlackTimestampMs = System.currentTimeMillis();
+recordBlackListCount++;
+
+if (Strings.isNullOrEmpty(reasonForBlackList) && 
!Strings.isNullOrEmpty(reason)) {
+reasonForBlackList = reason;
+}
+}
+
+private boolean shuoldBeBlackListed() {
+if (lastRecordBlackTimestampMs <= 0 || firstRecordBlackTimestampMs 
<= 0) {
+return false;
+}
+if (recordBlackListCount < RecordBlackListThreshold) {
+return false;
+}
+
+if (lastRecordBlackTimestampMs - firstRecordBlackTimestampMs
+>= Config.do_add_backend_black_list_threshold_secs * 1000) 
{
+return false;
+}
+
+return true;
+}
+
+private void doAddBlackList() {
+lastBlackTimestampMs = System.currentTimeMillis();
+Exception e = new Exception();
+String stack = 
org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(e);
+LOG.warn("Add backend {} to black list, reason: {}. Stack {}", 
backendID, reasonForBlackList, stack);
+}
+
+public boolean shuoldBeRemoved() {

Review Comment:
   typo error



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776202163


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;

Review Comment:
   java 里的final 字段不是使用的驼峰，是全部大写，中间使用下划线。



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](window function) Fix illegal frame range (#41147) [doris]

2024-09-25 Thread via GitHub



Gabriel39 merged PR #41305:
URL: https://github.com/apache/doris/pull/41305


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch branch-2.1 updated: [fix](window function) Fix illegal frame range (#41147) (#41305)

2024-09-25 Thread gabriellee

This is an automated email from the ASF dual-hosted git repository.

gabriellee pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.1 by this push:
 new a11fd620433 [fix](window function) Fix illegal frame range (#41147) 
(#41305)
a11fd620433 is described below

commit a11fd6204337d0a263c7beceb94146a3539f32e6
Author: Gabriel 
AuthorDate: Thu Sep 26 09:55:33 2024 +0800

[fix](window function) Fix illegal frame range (#41147) (#41305)

pick #41147

0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at

/home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421
 1# 0x7F591D573520 in /lib/x86_64-linux-gnu/libc.so.6
 2# pthread_kill at ./nptl/pthread_kill.c:89
 3# raise at ../sysdeps/posix/raise.c:27
 4# abort at ./stdlib/abort.c:81
 5# _nl_load_domain at ./intl/loadmsgcat.c:1177
 6# 0x7F591D56AE96 in /lib/x86_64-linux-gnu/libc.so.6
7# doris::vectorized::PODArray, 16ul, 15ul>::operator[](long)
const at

/home/zcp/repo_center/doris_master/doris/be/src/vec/common/pod_array.h:365
8# doris::vectorized::ColumnNullable::is_null_at(unsigned long) const at

/home/zcp/repo_center/doris_master/doris/be/src/vec/columns/column_nullable.h:158
9#


doris::vectorized::ReaderFirstAndLastData,
true, true, false>::insert_result_into(doris::vectorized::IColumn&)
const at


/home/zcp/repo_center/doris_master/doris/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h:125
10# doris::pipeline::AnalyticLocalState::_insert_result_info(long) in
/mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
11# std::_Function_handler))(long)> >::_M_invoke(std::_Any_data const&,
long&&) at


/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291
12# std::function::operator()(long) const at

/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
13# doris::pipeline::AnalyticLocalState::_get_next_for_rows(unsigned
long) in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be 14#
std::enable_if,
doris::Status>::type std::__invoke_r(doris::Status
(doris::pipeline::AnalyticLocalState::*&)(unsigned long),
doris::pipeline::AnalyticLocalState*&, unsigned long&&) at

/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:114
15# doris::Status std::_Bind_result))(unsigned long)>::__call(std::tuple&&, std::_Index_tuple<0ul,
1ul>) at


/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:570
16# std::_Function_handler))(unsigned long)> >::_M_invoke(std::_Any_data
const&, unsigned long&&) at


/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291
17# std::function::operator()(unsigned
long) const at


/var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
18#

doris::pipeline::AnalyticSourceOperatorX::get_block(doris::RuntimeState*,
doris::vectorized::Block*, bool*) in
/mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
19#


doris::pipeline::OperatorXBase::get_block_after_projects(doris::RuntimeState*,
doris::vectorized::Block*, bool*) at


/home/zcp/repo_center/doris_master/doris/be/src/pipeline/exec/operator.cpp:322
20# doris::pipeline::PipelineTask::execute(bool*) in
/mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
21# doris::pipeline::TaskScheduler::_do_work(unsigned long) at

/home/zcp/repo_center/doris_master/doris/be/src/pipeline/task_scheduler.cpp:138
22# doris::ThreadPool::dispatch_thread() in
/mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
23# doris::Thread::supervise_thread(void*) at
/home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:499 24#
start_thread at ./nptl/pthread_create.c:442
25# 0x7F591D657850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83
---
 be/src/pipeline/exec/analytic_source_operator.cpp  |   4 +-
 .../aggregate_function_window.h|   6 +-
 .../data/nereids_syntax_p0/window_function.out | 202 +
 .../nereids_syntax_p0/window_function.groovy   |  39 
 4 files changed, 247 insertions(+), 4 deletions(-)

diff --git a/be/src/pipeline/exec/analytic_source_operator.cpp 
b/be/src/pipeline/exec/analytic_source_operator.cpp
index f169cfe8208..9b638b87832 100644
--- a/be/src/pipeline/exec/analytic_source_operator.cpp
+++ b/be/src/pipeline/exec/analytic_source_operator.cpp
@@ -361,6 +361,7 @@ Status AnalyticLocalState::_get_next_for_rows(size_t 
current_block_rows) {
 1; //going on calculate,add up data, no

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776202384


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;
+
+public static class BlackListInfo {
+public BlackListInfo() {}
+
+private Lock lock = new ReentrantLock();
+private String reasonForBlackList = "";

Review Comment:
   add comment to explain these field



##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;
+
+public static class BlackListInfo {
+public BlackListInfo() {}
+
+private Lock lock = new ReentrantLock();
+private String reasonForBlackList = "";
+private Long firstRecordBlackTimestampMs = 0L;
+private Long lastRecordBlackTimestampMs = 0L;
+private Long lastBlackTimestampMs = 0L;
+private Long recordBlackListCount = 0L;
+private Long backendID = 0L;
+
+public void tryAddBlackList(String reason) {
+lock.lock();
+try {
+recordAddBlackList(reason);
+if (shuoldBeBlackListed()) {
+doAddBlackList();
+LOG.info("Backend is added to black list.\n{}", 
getDebugInfo());
+}
+} finally {
+lock.unlock();
+}
+}
+
+private void recordAddBlackList(String reason) {
+if (firstRecordBlackTimestampMs <= 0) {
+firstRecordBlackTimestampMs = System.currentTimeMillis();
+}
+
+lastRecordBlackTimestampMs = System.currentTimeMillis();
+recordBlackListCount++;
+
+if (Strings.isNullOrEmpty(reasonForBlackList) && 
!Strings.isNullOrEmpty(reason)) {
+reasonForBlackList = reason;
+}
+}
+
+private boolean shuoldBeBlackListed() {

Review Comment:
   typo error



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](snapshot) Link binlog files according to consistent rowsets [doris]

2024-09-25 Thread via GitHub



w41ter commented on PR #41319:
URL: https://github.com/apache/doris/pull/41319#issuecomment-2375596459

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) 01/01: [fix](snapshot) Link binlog files according to consistent rowsets

2024-09-25 Thread w41ter

This is an automated email from the ASF dual-hosted git repository.

w41ter pushed a commit to branch fix_snapshot_binlog_files
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 4254a77c40d6f3502099637c4a43c9ab9ed53990
Author: w41ter 
AuthorDate: Thu Sep 26 01:50:39 2024 +

[fix](snapshot) Link binlog files according to consistent rowsets

In previous implementations, binlog files would be linked according to
making snapshot request. However, sometimes not all requests can be
executed directly. For example, when a certain version in
missing_version does not exist, it will fallback to full snapshot.
Therefore, it is correct to link binlog files according to consistent
rowsets.
---
 be/src/olap/snapshot_manager.cpp | 21 ++---
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index e0bfaf704dd..2cfa9a8e8b7 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -411,13 +411,13 @@ Status SnapshotManager::_create_snapshot_files(const 
TabletSharedPtr& ref_tablet
 string snapshot_id;
 
RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize(snapshot_id_path, 
&snapshot_id));
 
+std::vector consistent_rowsets;
 do {
 TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
 if (new_tablet_meta == nullptr) {
 res = Status::Error("fail to malloc 
TabletMeta.");
 break;
 }
-std::vector consistent_rowsets;
 DeleteBitmap delete_bitmap_snapshot(new_tablet_meta->tablet_id());
 
 /// If set missing_version, try to get all missing version.
@@ -628,17 +628,16 @@ Status SnapshotManager::_create_snapshot_files(const 
TabletSharedPtr& ref_tablet
 }
 
 RowsetBinlogMetasPB rowset_binlog_metas_pb;
-if (request.__isset.missing_version) {
-res = ref_tablet->get_rowset_binlog_metas(request.missing_version,
-  &rowset_binlog_metas_pb);
-} else {
-std::vector missing_versions;
-res = ref_tablet->get_rowset_binlog_metas(missing_versions, 
&rowset_binlog_metas_pb);
-}
-if (!res.ok()) {
-break;
+for (auto& rs : consistent_rowsets) {
+if (!rs->is_local()) {
+continue;
+}
+res = ref_tablet->get_rowset_binlog_metas(rs->version(), 
&rowset_binlog_metas_pb);
+if (!res.ok()) {
+break;
+}
 }
-if (rowset_binlog_metas_pb.rowset_binlog_metas_size() == 0) {
+if (!res.ok() || rowset_binlog_metas_pb.rowset_binlog_metas_size() == 
0) {
 break;
 }
 


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch fix_snapshot_binlog_files created (now 4254a77c40d)

2024-09-25 Thread w41ter

This is an automated email from the ASF dual-hosted git repository.

w41ter pushed a change to branch fix_snapshot_binlog_files
in repository https://gitbox.apache.org/repos/asf/doris.git


  at 4254a77c40d [fix](snapshot) Link binlog files according to consistent 
rowsets

This branch includes the following new commits:

 new 4254a77c40d [fix](snapshot) Link binlog files according to consistent 
rowsets

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](snapshot) Link binlog files according to consistent rowsets [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41319:
URL: https://github.com/apache/doris/pull/41319#issuecomment-2375595941

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [fix](snapshot) Link binlog files according to consistent rowsets [doris]

2024-09-25 Thread via GitHub



w41ter opened a new pull request, #41319:
URL: https://github.com/apache/doris/pull/41319

   In previous implementations, binlog files would be linked according to 
making snapshot requests. However, sometimes not all requests can be executed 
directly. For example, when a certain version in missing_version does not 
exist, it will fallback to the full snapshot. Therefore, it is correct to link 
binlog files according to consistent rowsets.
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](new_json_reader)fix new_json_reader core [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41290:
URL: https://github.com/apache/doris/pull/41290#issuecomment-2375595333

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](new_json_reader)fix new_json_reader core [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41290:
URL: https://github.com/apache/doris/pull/41290#issuecomment-2375595372

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) fix double free when get_error_log_file_path [doris]

2024-09-25 Thread via GitHub



liaoxin01 commented on PR #41320:
URL: https://github.com/apache/doris/pull/41320#issuecomment-2375601498

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](pipeline) Make all upstream tasks runnable if all tasks finishe… [doris]

2024-09-25 Thread via GitHub



Gabriel39 commented on PR #41292:
URL: https://github.com/apache/doris/pull/41292#issuecomment-2375600284

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776205425


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;

Review Comment:
   把这个也是用config，不要定义变量，策略上的参数都可能会改。



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) fix double free when get_error_log_file_path [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41320:
URL: https://github.com/apache/doris/pull/41320#issuecomment-2375601445

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776206563


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;
+
+public static class BlackListInfo {
+public BlackListInfo() {}
+
+private Lock lock = new ReentrantLock();
+private String reasonForBlackList = "";
+private Long firstRecordBlackTimestampMs = 0L;
+private Long lastRecordBlackTimestampMs = 0L;
+private Long lastBlackTimestampMs = 0L;
+private Long recordBlackListCount = 0L;
+private Long backendID = 0L;
+
+public void tryAddBlackList(String reason) {
+lock.lock();
+try {
+recordAddBlackList(reason);
+if (shuoldBeBlackListed()) {
+doAddBlackList();
+LOG.info("Backend is added to black list.\n{}", 
getDebugInfo());
+}
+} finally {
+lock.unlock();
+}
+}
+
+private void recordAddBlackList(String reason) {
+if (firstRecordBlackTimestampMs <= 0) {
+firstRecordBlackTimestampMs = System.currentTimeMillis();
+}
+
+lastRecordBlackTimestampMs = System.currentTimeMillis();
+recordBlackListCount++;

Review Comment:
   如果当前时间- 之前的last 时间超过了 1min，那么就应该把count 变为0，然后把first 变成当前的时间。



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776207896


##
fe/fe-common/src/main/java/org/apache/doris/common/Config.java:
##
@@ -1954,6 +1954,12 @@ public class Config extends ConfigBase {
 @ConfField(mutable = true, masterOnly = false)
 public static boolean disable_backend_black_list = false;
 
+@ConfField(mutable = true, masterOnly = false)
+public static long do_add_backend_black_list_threshold_secs = 10;

Review Comment:
   change to 30s



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776207473


##
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java:
##
@@ -19,37 +19,169 @@
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.UserException;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TScanRangeLocation;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 public class SimpleScheduler {
 private static final Logger LOG = 
LogManager.getLogger(SimpleScheduler.class);
 
+public static final long RecordBlackListThreshold = 10;
+
+public static class BlackListInfo {
+public BlackListInfo() {}
+
+private Lock lock = new ReentrantLock();
+private String reasonForBlackList = "";
+private Long firstRecordBlackTimestampMs = 0L;
+private Long lastRecordBlackTimestampMs = 0L;
+private Long lastBlackTimestampMs = 0L;
+private Long recordBlackListCount = 0L;
+private Long backendID = 0L;
+
+public void tryAddBlackList(String reason) {
+lock.lock();
+try {
+recordAddBlackList(reason);
+if (shuoldBeBlackListed()) {
+doAddBlackList();
+LOG.info("Backend is added to black list.\n{}", 
getDebugInfo());
+}
+} finally {
+lock.unlock();
+}
+}
+
+private void recordAddBlackList(String reason) {
+if (firstRecordBlackTimestampMs <= 0) {
+firstRecordBlackTimestampMs = System.currentTimeMillis();
+}
+
+lastRecordBlackTimestampMs = System.currentTimeMillis();
+recordBlackListCount++;
+
+if (Strings.isNullOrEmpty(reasonForBlackList) && 
!Strings.isNullOrEmpty(reason)) {
+reasonForBlackList = reason;
+}
+}
+
+private boolean shuoldBeBlackListed() {
+if (lastRecordBlackTimestampMs <= 0 || firstRecordBlackTimestampMs 
<= 0) {
+return false;
+}
+if (recordBlackListCount < RecordBlackListThreshold) {
+return false;
+}
+
+if (lastRecordBlackTimestampMs - firstRecordBlackTimestampMs
+>= Config.do_add_backend_black_list_threshold_secs * 1000) 
{
+return false;
+}
+
+return true;
+}
+
+private void doAddBlackList() {
+lastBlackTimestampMs = System.currentTimeMillis();
+Exception e = new Exception();
+String stack = 
org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(e);

Review Comment:
   不用这么写，LOG.warn(e), 就会直接打印stack



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](load) fix priority queue order in memtable memory limiter [doris]

2024-09-25 Thread via GitHub



liaoxin01 merged PR #41278:
URL: https://github.com/apache/doris/pull/41278


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](pipeline) Make all upstream tasks runnable if all tasks finishe… [doris]

2024-09-25 Thread via GitHub



Gabriel39 commented on PR #41292:
URL: https://github.com/apache/doris/pull/41292#issuecomment-2375603915

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](pipeline) Make all upstream tasks runnable if all tasks finishe… [doris]

2024-09-25 Thread via GitHub



Gabriel39 commented on PR #41292:
URL: https://github.com/apache/doris/pull/41292#issuecomment-2375604760

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](blacklist) Backend should not be added to blacklist easily [doris]

2024-09-25 Thread via GitHub



yiguolei commented on code in PR #41170:
URL: https://github.com/apache/doris/pull/41170#discussion_r1776208051


##
fe/fe-common/src/main/java/org/apache/doris/common/Config.java:
##
@@ -1954,6 +1954,12 @@ public class Config extends ConfigBase {
 @ConfField(mutable = true, masterOnly = false)
 public static boolean disable_backend_black_list = false;
 
+@ConfField(mutable = true, masterOnly = false)
+public static long do_add_backend_black_list_threshold_secs = 10;
+
+@ConfField(mutable = true, masterOnly = false)
+public static long stay_in_backend_black_list_threshold_secs = 10;

Review Comment:
   stay in 继续跟之前一样，是1min，害怕策略变动会出问题。



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [improve](serde) support const column in serialize/deserialize function [doris]

2024-09-25 Thread via GitHub



zhangstar333 commented on PR #41175:
URL: https://github.com/apache/doris/pull/41175#issuecomment-2375604312

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch master updated (16d05dc2f9d -> 519bb399fd5)

2024-09-25 Thread liaoxin

This is an automated email from the ASF dual-hosted git repository.

liaoxin pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


from 16d05dc2f9d [fix](array-funcs)fix array agg func with decimal type 
(#40839)
 add 519bb399fd5 [fix](load) fix priority queue order in memtable memory 
limiter (#41278)

No new revisions were added by this update.

Summary of changes:
 be/src/olap/memtable_memory_limiter.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](profile) Add new config profile_waiting_time_for_spill_s [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #40302:
URL: https://github.com/apache/doris/pull/40302#issuecomment-2375607183

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](profile) Add new config profile_waiting_time_for_spill_s [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #40302:
URL: https://github.com/apache/doris/pull/40302#issuecomment-2375607150

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] refactor logic of revoking and low memory mode in local exchange oper… [doris]

2024-09-25 Thread via GitHub



yiguolei merged PR #41264:
URL: https://github.com/apache/doris/pull/41264


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Enhancement](doris-future) Support REGR_SXX_SXY_SYY aggregation functions [doris]

2024-09-25 Thread via GitHub



zhiqiang- commented on code in PR #39187:
URL: https://github.com/apache/doris/pull/39187#discussion_r1776211458


##
be/src/vec/aggregate_functions/aggregate_function_regr_sxx_.h:
##
@@ -0,0 +1,274 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "olap/olap_common.h"
+#include "runtime/decimalv2_value.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/common/assert_cast.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/io/io_helper.h"
+namespace doris {
+namespace vectorized {
+class Arena;
+class BufferReadable;
+class BufferWritable;
+template 
+class ColumnDecimal;
+template 
+class ColumnVector;
+} // namespace vectorized
+} // namespace doris
+
+namespace doris::vectorized {
+
+template 
+struct AggregateFunctionRegrSxxData {
+UInt64 count = 0;
+double square_of_sum_x {};
+double sum_x {};
+void write(BufferWritable& buf) const {
+write_binary(square_of_sum_x, buf);
+write_binary(sum_x, buf);
+write_binary(count, buf);
+}
+
+void read(BufferReadable& buf) {
+read_binary(square_of_sum_x, buf);
+read_binary(sum_x, buf);
+read_binary(count, buf);
+}
+
+void reset() {
+square_of_sum_x = {};
+sum_x = {};
+count = 0;
+}
+
+double get_regr_sxx_result() const { return square_of_sum_x - (sum_x * 
sum_x) / count; }
+
+void merge(const AggregateFunctionRegrSxxData& rhs) {
+if (rhs.count == 0) {
+return;
+}
+square_of_sum_x += rhs.square_of_sum_x;
+sum_x += rhs.sum_x;
+count += rhs.count;
+}
+
+void add(const IColumn* column_y, const IColumn* column_x, size_t row_num) 
{
+#ifdef __clang__
+#pragma clang fp reassociate(on)
+#endif
+const auto& sources_x =
+assert_cast&, 
TypeCheckOnRelease::DISABLE>(*column_x);
+const auto& value = sources_x.get_data()[row_num];
+sum_x += value;
+square_of_sum_x += value * value;
+count += 1;
+}
+};
+template 
+struct AggregateFunctionRegrSxyData {
+using ResultType = T;
+UInt64 count = 0;
+double sum_x {};
+double sum_y {};
+double sum_of_x_mul_y {};
+void write(BufferWritable& buf) const {
+write_binary(sum_x, buf);
+write_binary(sum_y, buf);
+write_binary(sum_of_x_mul_y, buf);
+write_binary(count, buf);
+}
+
+void read(BufferReadable& buf) {
+read_binary(sum_x, buf);
+read_binary(sum_y, buf);
+read_binary(sum_of_x_mul_y, buf);
+read_binary(count, buf);
+}
+
+void reset() {
+count = 0;
+sum_x = {};
+sum_y = {};
+sum_of_x_mul_y = {};
+}
+
+double get_regr_sxy_result() const { return sum_of_x_mul_y - (sum_x * 
sum_y) / count; }

Review Comment:
   what if count is zero



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch spill_and_reserve updated: refactor logic of revoking and low memory mode in local exchange oper… (#41264)

2024-09-25 Thread yiguolei

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/spill_and_reserve by this push:
 new 41bb04b9e9e refactor logic of revoking and low memory mode in local 
exchange oper… (#41264)
41bb04b9e9e is described below

commit 41bb04b9e9e237fd5680674d924228be7aa3b0e6
Author: Jerry Hu 
AuthorDate: Thu Sep 26 10:12:36 2024 +0800

refactor logic of revoking and low memory mode in local exchange oper… 
(#41264)

…ator

## Proposed changes

Issue Number: close #xxx


---
 be/src/pipeline/dependency.h   | 21 ++-
 be/src/pipeline/exec/operator.h| 37 -
 .../exec/partitioned_aggregation_sink_operator.cpp | 29 --
 .../exec/partitioned_aggregation_sink_operator.h   |  8 ++-
 .../exec/partitioned_hash_join_probe_operator.cpp  |  6 +-
 .../exec/partitioned_hash_join_probe_operator.h|  3 +-
 .../exec/partitioned_hash_join_sink_operator.cpp   | 59 ++--
 .../exec/partitioned_hash_join_sink_operator.h | 11 ++--
 be/src/pipeline/exec/spill_sort_sink_operator.cpp  | 19 ---
 be/src/pipeline/exec/spill_sort_sink_operator.h|  7 ++-
 be/src/pipeline/exec/spill_utils.h | 32 +++
 .../local_exchange_sink_operator.cpp   |  8 ++-
 be/src/pipeline/local_exchange/local_exchanger.cpp | 49 +
 be/src/pipeline/pipeline_task.cpp  |  9 ++-
 be/src/pipeline/pipeline_task.h|  2 +-
 be/src/runtime/query_context.cpp   | 64 ++
 be/src/runtime/query_context.h |  7 ++-
 .../workload_group/workload_group_manager.cpp  | 38 ++---
 18 files changed, 325 insertions(+), 84 deletions(-)

diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index 8cb479ccbb0..5f030dda5d2 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -26,6 +26,7 @@
 #include 
 #include 
 
+#include "common/config.h"
 #include "common/logging.h"
 #include "concurrentqueue.h"
 #include "gutil/integral_types.h"
@@ -832,6 +833,7 @@ public:
 std::unique_ptr exchanger {};
 std::vector mem_trackers;
 std::atomic mem_usage = 0;
+size_t _buffer_mem_limit = config::local_exchange_buffer_mem_limit;
 // We need to make sure to add mem_usage first and then enqueue, otherwise 
sub mem_usage may cause negative mem_usage during concurrent dequeue.
 std::mutex le_lock;
 virtual void create_dependencies(int local_exchange_id) {
@@ -875,7 +877,7 @@ public:
 void sub_mem_usage(int channel_id, size_t delta) { 
mem_trackers[channel_id]->release(delta); }
 
 virtual void add_total_mem_usage(size_t delta, int channel_id) {
-if (mem_usage.fetch_add(delta) + delta > 
config::local_exchange_buffer_mem_limit) {
+if (mem_usage.fetch_add(delta) + delta > _buffer_mem_limit) {
 sink_deps.front()->block();
 }
 }
@@ -884,10 +886,15 @@ public:
 auto prev_usage = mem_usage.fetch_sub(delta);
 DCHECK_GE(prev_usage - delta, 0) << "prev_usage: " << prev_usage << " 
delta: " << delta
  << " channel_id: " << channel_id;
-if (prev_usage - delta <= config::local_exchange_buffer_mem_limit) {
+if (prev_usage - delta <= _buffer_mem_limit) {
 sink_deps.front()->set_ready();
 }
 }
+
+virtual void set_low_memory_mode() {
+_buffer_mem_limit =
+std::min(config::local_exchange_buffer_mem_limit, 10 
* 1024 * 1024);
+}
 };
 
 struct LocalMergeExchangeSharedState : public LocalExchangeSharedState {
@@ -933,6 +940,14 @@ struct LocalMergeExchangeSharedState : public 
LocalExchangeSharedState {
 source_deps[channel_id]->set_ready();
 }
 
+void set_low_memory_mode() override {
+_buffer_mem_limit =
+std::min(config::local_exchange_buffer_mem_limit, 10 
* 1024 * 1024);
+DCHECK(!_queues_mem_usage.empty());
+_each_queue_limit =
+std::max(64 * 1024, _buffer_mem_limit / 
_queues_mem_usage.size());
+}
+
 Dependency* get_sink_dep_by_channel_id(int channel_id) override {
 return sink_deps[channel_id].get();
 }
@@ -943,7 +958,7 @@ struct LocalMergeExchangeSharedState : public 
LocalExchangeSharedState {
 
 private:
 std::vector _queues_mem_usage;
-const int64_t _each_queue_limit;
+int64_t _each_queue_limit;
 };
 
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index 3a644eb4f02..1d2dbcc3592 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -32,6 +32,7 @@
 #include "common/status.h"
 #include "pipeline/dependency.h"
 #include "pipeline/exec/operator.

Re: [PR] [fix](datasource) fix typo in ExternalSchemaCache [doris]

2024-09-25 Thread via GitHub



yiguolei merged PR #41221:
URL: https://github.com/apache/doris/pull/41221


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch master updated: [fix](datasource) fix typo in ExternalSchemaCache (#41221)

2024-09-25 Thread yiguolei

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
 new 3f2f549b961 [fix](datasource) fix typo in ExternalSchemaCache (#41221)
3f2f549b961 is described below

commit 3f2f549b961e611107bb19a9c7889554396ca142
Author: Zijie Lu 
AuthorDate: Thu Sep 26 10:13:31 2024 +0800

[fix](datasource) fix typo in ExternalSchemaCache (#41221)

## Proposed changes
Fix typo in ExternalSchemaCache

Issue Number: close #xxx



Signed-off-by: uid=luzijie01,ou=people,dc=hz,dc=netease,dc=com 

---
 .../main/java/org/apache/doris/datasource/ExternalSchemaCache.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
index ad1c1306e34..a0558766e81 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
@@ -51,13 +51,13 @@ public class ExternalSchemaCache {
 }
 
 private void init(ExecutorService executor) {
-CacheFactory schemaCacheeFactory = new CacheFactory(
+CacheFactory schemaCacheFactory = new CacheFactory(
 OptionalLong.of(86400L),
 
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
 Config.max_external_schema_cache_num,
 false,
 null);
-schemaCache = schemaCacheeFactory.buildCache(key -> loadSchema(key), 
null, executor);
+schemaCache = schemaCacheFactory.buildCache(key -> loadSchema(key), 
null, executor);
 }
 
 private void initMetrics() {


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) fix double free when get_error_log_file_path [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41320:
URL: https://github.com/apache/doris/pull/41320#issuecomment-2375615754

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](snapshot) Link binlog files according to consistent rowsets [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41319:
URL: https://github.com/apache/doris/pull/41319#issuecomment-2375616719

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](snapshot) Link binlog files according to consistent rowsets [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41319:
URL: https://github.com/apache/doris/pull/41319#issuecomment-2375616741

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) fix double free when get_error_log_file_path [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41320:
URL: https://github.com/apache/doris/pull/41320#issuecomment-2375615799

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch master updated: [Performance](func) opt the print_id func (#41302)

2024-09-25 Thread lihaopeng

This is an automated email from the ASF dual-hosted git repository.

lihaopeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
 new 2208cde22e7 [Performance](func) opt the print_id func (#41302)
2208cde22e7 is described below

commit 2208cde22e7d8f85c7b62ee571d6dc7fd514b91f
Author: HappenLee 
AuthorDate: Thu Sep 26 10:20:10 2024 +0800

[Performance](func) opt the print_id func (#41302)

Load Average: 52.48, 35.97, 38.43
--
BenchmarkTime CPU   Iterations
--
old 3390427270 ns   3390354519 ns1
new335514305 ns335513720 ns2
---
 be/src/util/uid_util.cpp | 11 +--
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/be/src/util/uid_util.cpp b/be/src/util/uid_util.cpp
index 6743c05a842..0f93f437ab6 100644
--- a/be/src/util/uid_util.cpp
+++ b/be/src/util/uid_util.cpp
@@ -17,6 +17,7 @@
 
 #include "util/uid_util.h"
 
+#include 
 #include 
 #include 
 #include 
@@ -44,15 +45,13 @@ std::ostream& operator<<(std::ostream& os, const UniqueId& 
uid) {
 }
 
 std::string print_id(const TUniqueId& id) {
-std::stringstream out;
-out << std::hex << id.hi << "-" << id.lo;
-return out.str();
+return fmt::format(FMT_COMPILE("{:x}-{:x}"), static_cast(id.hi),
+   static_cast(id.lo));
 }
 
 std::string print_id(const PUniqueId& id) {
-std::stringstream out;
-out << std::hex << id.hi() << "-" << id.lo();
-return out.str();
+return fmt::format(FMT_COMPILE("{:x}-{:x}"), 
static_cast(id.hi()),
+   static_cast(id.lo()));
 }
 
 bool parse_id(const std::string& s, TUniqueId* id) {


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Performance](func) opt the print_id func [doris]

2024-09-25 Thread via GitHub



HappenLee merged PR #41302:
URL: https://github.com/apache/doris/pull/41302


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [chore](cancel log) Remove some cancel log on FE [doris]

2024-09-25 Thread via GitHub



zhiqiang- commented on PR #41312:
URL: https://github.com/apache/doris/pull/41312#issuecomment-2375618899

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch master updated: [enhancement](mem-tracker) Use thread local mem tracker to track s3 file buffer memory usage (#40597)

2024-09-25 Thread zouxinyi

This is an automated email from the ASF dual-hosted git repository.

zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
 new 62a64d10ae8 [enhancement](mem-tracker) Use thread local mem tracker to 
track s3 file buffer memory usage (#40597)
62a64d10ae8 is described below

commit 62a64d10ae8d1d6431834f09ad296836b8cdedc3
Author: Siyang Tang <82279870+tangsiyang2...@users.noreply.github.com>
AuthorDate: Thu Sep 26 12:32:12 2024 +0800

[enhancement](mem-tracker) Use thread local mem tracker to track s3 file 
buffer memory usage (#40597)

Track s3 file buffer memory usage with thread local tracker, so that
memory usage will be specified to detail.
---
 be/src/io/fs/s3_file_bufferpool.cpp | 25 +++--
 be/src/io/fs/s3_file_bufferpool.h   | 14 +-
 be/src/olap/tablet.cpp  |  6 ++
 be/src/olap/tablet.h|  3 +++
 be/src/runtime/snapshot_loader.cpp  |  5 +
 be/src/runtime/snapshot_loader.h|  2 ++
 6 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/be/src/io/fs/s3_file_bufferpool.cpp 
b/be/src/io/fs/s3_file_bufferpool.cpp
index f1f90ea7f2e..0d59ea0ed88 100644
--- a/be/src/io/fs/s3_file_bufferpool.cpp
+++ b/be/src/io/fs/s3_file_bufferpool.cpp
@@ -31,6 +31,7 @@
 #include "io/cache/file_cache_common.h"
 #include "io/fs/s3_common.h"
 #include "runtime/exec_env.h"
+#include "runtime/memory/mem_tracker_limiter.h"
 #include "runtime/thread_context.h"
 #include "util/defer_op.h"
 #include "util/slice.h"
@@ -77,17 +78,19 @@ Slice FileBuffer::get_slice() const {
 }
 
 FileBuffer::FileBuffer(BufferType type, std::function 
alloc_holder,
-   size_t offset, OperationState state)
+   size_t offset, OperationState state,
+   std::shared_ptr mem_tracker)
 : _type(type),
   _alloc_holder(std::move(alloc_holder)),
   _offset(offset),
   _size(0),
   _state(std::move(state)),
   _inner_data(std::make_unique()),
-  _capacity(_inner_data->size()) {}
+  _capacity(_inner_data->size()),
+  _mem_tracker(std::move(mem_tracker)) {}
 
 FileBuffer::~FileBuffer() {
-
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->s3_file_buffer_tracker());
+SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker);
 _inner_data.reset();
 }
 
@@ -240,13 +243,22 @@ FileBufferBuilder& 
FileBufferBuilder::set_allocate_file_blocks_holder(
 }
 
 Status FileBufferBuilder::build(std::shared_ptr* buf) {
-
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->s3_file_buffer_tracker());
+auto mem_tracker = ExecEnv::GetInstance()->s3_file_buffer_tracker();
+auto* thread_ctx = doris::thread_context(true);
+if (thread_ctx != nullptr) {
+// if thread local mem tracker is set, use it instead.
+auto curr_tracker = 
thread_ctx->thread_mem_tracker_mgr->limiter_mem_tracker();
+if (curr_tracker != ExecEnv::GetInstance()->orphan_mem_tracker()) {
+mem_tracker = std::move(curr_tracker);
+}
+}
+SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(mem_tracker);
 OperationState state(_sync_after_complete_task, _is_cancelled);
 
 if (_type == BufferType::UPLOAD) {
 RETURN_IF_CATCH_EXCEPTION(*buf = std::make_shared(
   std::move(_upload_cb), 
std::move(state), _offset,
-  std::move(_alloc_holder_cb)));
+  std::move(_alloc_holder_cb), 
std::move(mem_tracker)));
 return Status::OK();
 }
 if (_type == BufferType::DOWNLOAD) {
@@ -254,7 +266,8 @@ Status 
FileBufferBuilder::build(std::shared_ptr* buf) {
   std::move(_download),
   
std::move(_write_to_local_file_cache),
   std::move(_write_to_use_buffer), 
std::move(state),
-  _offset, 
std::move(_alloc_holder_cb)));
+  _offset, std::move(_alloc_holder_cb),
+  std::move(mem_tracker)));
 return Status::OK();
 }
 // should never come here
diff --git a/be/src/io/fs/s3_file_bufferpool.h 
b/be/src/io/fs/s3_file_bufferpool.h
index 1b552850ae3..a603c3cb29a 100644
--- a/be/src/io/fs/s3_file_bufferpool.h
+++ b/be/src/io/fs/s3_file_bufferpool.h
@@ -27,6 +27,7 @@
 
 #include "common/status.h"
 #include "io/cache/file_block.h"
+#include "runtime/memory/mem_tracker_limiter.h"
 #include "util/crc32c.h"
 #include "util/slice.h"
 #include "util/threadpool.h"
@@ -77,7 +78,7 @@ struct OperationState {
 
 struct FileBuffer {
 FileBuffer(BufferType type, std::function 
alloc_holder, size_t offset,
-   Operatio

Re: [PR] [enhancement](mem-tracker) Use thread local mem tracker to track s3 file buffer memory usage [doris]

2024-09-25 Thread via GitHub



xinyiZzz merged PR #40597:
URL: https://github.com/apache/doris/pull/40597


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](tools) update tools schema [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41335:
URL: https://github.com/apache/doris/pull/41335#issuecomment-2375849002

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](tools) update tools schema [doris]

2024-09-25 Thread via GitHub



xzj7019 commented on PR #41335:
URL: https://github.com/apache/doris/pull/41335#issuecomment-2375849796

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [opt](tools) update tools schema [doris]

2024-09-25 Thread via GitHub



xzj7019 opened a new pull request, #41335:
URL: https://github.com/apache/doris/pull/41335

   optimize tpcds schema bucket number.
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](pipeline) Make all upstream tasks runnable if all tasks finishe… [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41292:
URL: https://github.com/apache/doris/pull/41292#issuecomment-2375855413

   TeamCity be ut coverage result:
Function Coverage: 37.28% (9623/25816) 
Line Coverage: 28.70% (79675/277609)
Region Coverage: 28.11% (41178/146488)
Branch Coverage: 24.75% (20977/84766)
Coverage Report: 
http://coverage.selectdb-in.cc/coverage/0c6d47cff0cbf3b5382128720777cac466ac36e5_0c6d47cff0cbf3b5382128720777cac466ac36e5/report/index.html


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](tools) update tools schema [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41335:
URL: https://github.com/apache/doris/pull/41335#issuecomment-2375859380

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](tools) update tools schema [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41335:
URL: https://github.com/apache/doris/pull/41335#issuecomment-2375859356

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](memory) Support Memory Profile [doris]

2024-09-25 Thread via GitHub



xinyiZzz commented on PR #41310:
URL: https://github.com/apache/doris/pull/41310#issuecomment-2375855738

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Improvement]Add customStdAllocator for vector/map [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41193:
URL: https://github.com/apache/doris/pull/41193#issuecomment-2375894125

   
   
   ClickBench: Total hot run time: 33.11 s
   
   ```
   machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
   scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
   ClickBench test result on commit 87e606d0bcccd09cbca56f91e480faf7b28bfd34, 
data reload: false
   
   query1   0.050.040.04
   query2   0.060.030.03
   query3   0.230.060.06
   query4   1.680.090.10
   query5   0.510.500.52
   query6   1.130.720.72
   query7   0.020.020.02
   query8   0.050.030.04
   query9   0.560.510.50
   query10  0.550.590.55
   query11  0.130.110.11
   query12  0.130.110.10
   query13  0.610.590.60
   query14  2.712.722.84
   query15  0.910.820.83
   query16  0.390.360.38
   query17  1.030.971.04
   query18  0.200.190.19
   query19  1.941.882.04
   query20  0.020.010.01
   query21  15.35   0.600.57
   query22  2.532.742.13
   query23  17.56   0.830.83
   query24  2.961.291.61
   query25  0.180.120.06
   query26  0.580.140.13
   query27  0.040.040.04
   query28  9.751.111.06
   query29  12.53   3.203.17
   query30  0.240.060.05
   query31  2.890.370.38
   query32  3.290.470.46
   query33  2.972.983.03
   query34  16.78   4.494.44
   query35  4.484.534.47
   query36  0.670.490.48
   query37  0.080.050.06
   query38  0.050.030.04
   query39  0.030.020.02
   query40  0.150.120.12
   query41  0.070.020.02
   query42  0.030.020.02
   query43  0.040.030.03
   Total cold run time: 106.16 s
   Total hot run time: 33.11 s
   ```
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 [doris]

2024-09-25 Thread via GitHub



Jibing-Li commented on PR #40982:
URL: https://github.com/apache/doris/pull/40982#issuecomment-2375893706

   run p0


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Improvement]Add customStdAllocator for vector/map [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41193:
URL: https://github.com/apache/doris/pull/41193#issuecomment-2375877188

   TeamCity be ut coverage result:
Function Coverage: 37.30% (9632/25822) 
Line Coverage: 28.70% (79682/277619)
Region Coverage: 28.12% (41199/146492)
Branch Coverage: 24.76% (20981/84754)
Coverage Report: 
http://coverage.selectdb-in.cc/coverage/87e606d0bcccd09cbca56f91e480faf7b28bfd34_87e606d0bcccd09cbca56f91e480faf7b28bfd34/report/index.html


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Improvement]Add customStdAllocator for vector/map [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41193:
URL: https://github.com/apache/doris/pull/41193#issuecomment-2375876665

   
   
   TPC-H: Total hot run time: 41015 ms
   
   ```
   machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
   scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
   Tpch sf100 test result on commit 87e606d0bcccd09cbca56f91e480faf7b28bfd34, 
data reload: false
   
   -- Round 1 --
   q1   17576   810472577257
   q2   2019288 291 288
   q3   11960   109312251093
   q4   10564   782 730 730
   q5   7785288628562856
   q6   235 149 148 148
   q7   998 619 598 598
   q8   9370197619421942
   q9   6622649763906390
   q10  6947232523352325
   q11  435 253 255 253
   q12  419 223 223 223
   q13  17773   300629762976
   q14  238 208 220 208
   q15  584 530 520 520
   q16  646 598 581 581
   q17  995 569 552 552
   q18  7178662067936620
   q19  1402106410671064
   q20  484 208 209 208
   q21  4105329832143214
   q22  1093969 1016969
   Total cold run time: 109428 ms
   Total hot run time: 41015 ms
   
   - Round 2, with runtime_filter_mode=off -
   q1   7235722672417226
   q2   327 230 232 230
   q3   3066294029722940
   q4   2110192518741874
   q5   5724567757105677
   q6   231 153 148 148
   q7   2202178317771777
   q8   3411353235563532
   q9   8897913489728972
   q10  3581354135663541
   q11  581 512 490 490
   q12  873 627 661 627
   q13  8553316131983161
   q14  310 273 290 273
   q15  573 530 531 530
   q16  702 645 646 645
   q17  1884165315961596
   q18  8325767078737670
   q19  1726164315351535
   q20  2139187718881877
   q21  5541538654135386
   q22  1144104310511043
   Total cold run time: 69135 ms
   Total hot run time: 60750 ms
   ```
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [cherry-pick](branch-21) fix tablet sink shuffle without project not match the output tuple (#40299)(#41293) [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41327:
URL: https://github.com/apache/doris/pull/41327#issuecomment-2375876837

   TeamCity be ut coverage result:
Function Coverage: 36.15% (9345/25853) 
Line Coverage: 27.68% (76787/277454)
Region Coverage: 26.43% (39399/149066)
Branch Coverage: 23.23% (20061/86376)
Coverage Report: 
http://coverage.selectdb-in.cc/coverage/379180b6d748e1e745f77a6a3d34941cf96a0b68_379180b6d748e1e745f77a6a3d34941cf96a0b68/report/index.html


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 [doris]

2024-09-25 Thread via GitHub



Jibing-Li commented on PR #40982:
URL: https://github.com/apache/doris/pull/40982#issuecomment-2375879843

   run p0


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [feat](Nereids) support set var in hint when parse sql [doris]

2024-09-25 Thread via GitHub



LiBinfeng-01 commented on PR #41331:
URL: https://github.com/apache/doris/pull/41331#issuecomment-2375881573

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](window_func) fix bug of agg function used in window function and add many test cases (#40678) [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41328:
URL: https://github.com/apache/doris/pull/41328#issuecomment-2375882054

   TeamCity be ut coverage result:
Function Coverage: 36.14% (9347/25863) 
Line Coverage: 27.66% (76757/277468)
Region Coverage: 26.44% (39412/149063)
Branch Coverage: 23.22% (20057/86368)
Coverage Report: 
http://coverage.selectdb-in.cc/coverage/c2b3cb7903c7dfc29f670944dbd0ff04861bc1bd_c2b3cb7903c7dfc29f670944dbd0ff04861bc1bd/report/index.html


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [regression-test](parquet) add some parquet cases with tvf [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #40379:
URL: https://github.com/apache/doris/pull/40379#issuecomment-2375882689

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [bugfix](iceberg)add prefix for endpoint with s3 client [doris]

2024-09-25 Thread via GitHub



wuwenchi commented on PR #41336:
URL: https://github.com/apache/doris/pull/41336#issuecomment-2375887331

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [bugfix](iceberg)add prefix for endpoint with s3 client [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41336:
URL: https://github.com/apache/doris/pull/41336#issuecomment-2375887273

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [bugfix](iceberg)add prefix for endpoint with s3 client [doris]

2024-09-25 Thread via GitHub



wuwenchi opened a new pull request, #41336:
URL: https://github.com/apache/doris/pull/41336

   ## Proposed changes
   
   add prefix for endpoint with s3 client.
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Improvement]Add customStdAllocator for vector/map [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41193:
URL: https://github.com/apache/doris/pull/41193#issuecomment-2375889121

   
   
   TPC-DS: Total hot run time: 192538 ms
   
   ```
   machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
   scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
   TPC-DS sf100 test result on commit 87e606d0bcccd09cbca56f91e480faf7b28bfd34, 
data reload: false
   
   query1   960 381 377 377
   query2   6356211820202020
   query3   8681199 208 199
   query4   34297   23493   23429   23429
   query5   3376464 477 464
   query6   270 164 160 160
   query7   4193301 302 301
   query8   277 222 221 221
   query9   9593267226702670
   query10  458 277 272 272
   query11  17956   15204   15241   15204
   query12  156 100 99  99
   query13  1533430 410 410
   query14  9638726675597266
   query15  254 166 180 166
   query16  8031473 491 473
   query17  1689605 618 605
   query18  2174337 324 324
   query19  375 148 154 148
   query20  120 111 125 111
   query21  216 119 105 105
   query22  4665445246964452
   query23  35309   34684   34507   34507
   query24  11171   285128432843
   query25  620 406 424 406
   query26  1167180 159 159
   query27  2610304 291 291
   query28  7497244024432440
   query29  847 440 442 440
   query30  262 156 151 151
   query31  993 786 808 786
   query32  93  54  55  54
   query33  747 301 294 294
   query34  907 509 494 494
   query35  877 718 735 718
   query36  1093946 915 915
   query37  146 80  84  80
   query38  3973387938883879
   query39  1516140114221401
   query40  203 100 100 100
   query41  51  49  49  49
   query42  115 94  94  94
   query43  527 488 486 486
   query44  1246822 795 795
   query45  197 169 167 167
   query46  1150714 732 714
   query47  1924183818251825
   query48  469 372 377 372
   query49  902 403 394 394
   query50  841 409 412 409
   query51  7038694670146946
   query52  101 89  89  89
   query53  254 178 203 178
   query54  1105460 478 460
   query55  75  76  75  75
   query56  283 267 251 251
   query57  1236109111161091
   query58  235 229 241 229
   query59  3142287829702878
   query60  288 266 276 266
   query61  103 106 105 105
   query62  841 654 651 651
   query63  226 183 180 180
   query64  4424642 636 636
   query65  3251319231943192
   query66  889 298 305 298
   query67  16042   15664   15613   15613
   query68  5065555 572 555
   query69  602 301 301 301
   query70  1188117911431143
   query71  434 271 274 271
   query72  7608409340984093
   query73  763 348 355 348
   query74  10440   900291739002
   query75  3796268926982689
   query76  3537897 840 840
   query77  519 312 299 299
   query78  10575   959795979597
   query79  2252584 608 584
   query80  2695438 456 438
   query81  580 238 235 235
   query82  708 140 140 140
   query83  297 133 138 133
   query84  282 81  74  74
   query85  1775299 282 282
   query86  471 314 312 312
   query87  4390431343404313
   query88  3806244624192419
   query89  406 282 294 282
   query90  2163185 191 185
   query91  171 163 142 142
   query92  66  47  46  46
   query93  2400545 556 545
   query94  1269306 294 294
   query95  367 253 261 253
   query96  633 283 288 283
   query97  3257310030883088
   query98  222 214 194 194
   query99  1523128712811281
   Total cold run time: 306514 ms
   Total hot run time: 192538 ms
   ```
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to t

Re: [PR] [bugfix](paimon)upgrade paimon to 0.9 [doris]

2024-09-25 Thread via GitHub



wuwenchi commented on PR #41337:
URL: https://github.com/apache/doris/pull/41337#issuecomment-2375977707

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [bugfix](paimon)upgrade paimon to 0.9 [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41337:
URL: https://github.com/apache/doris/pull/41337#issuecomment-2375977608

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [opt](ci) start dockers in parallel [doris]

2024-09-25 Thread via GitHub



suxiaogang223 opened a new pull request, #41338:
URL: https://github.com/apache/doris/pull/41338

   ## Proposed changes
   Start docker in parallel to reduce external pipeline time
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](ci) start dockers in parallel [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41338:
URL: https://github.com/apache/doris/pull/41338#issuecomment-2375982776

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [opt](ci) start dockers in parallel [doris]

2024-09-25 Thread via GitHub



suxiaogang223 commented on PR #41338:
URL: https://github.com/apache/doris/pull/41338#issuecomment-2375982943

   run external


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [I] [Bug] 导出同一个k8s集群的minio使用内部域名的方式会报错 [doris]

2024-09-25 Thread via GitHub



764276020 commented on issue #40129:
URL: https://github.com/apache/doris/issues/40129#issuecomment-2375993883

   大佬，这个问题解决了吗，我这里也遇到了这样的问题


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [Fix](Serde) fix potential mem leak in array serde write_one_cell_to_json [doris]

2024-09-25 Thread via GitHub



eldenmoon opened a new pull request, #41339:
URL: https://github.com/apache/doris/pull/41339

   placement new may lead to mem leak in Field without calling it's desctructor
   
   related PR #40573
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Fix](Serde) fix potential mem leak in array serde write_one_cell_to_json [doris]

2024-09-25 Thread via GitHub



eldenmoon commented on PR #41339:
URL: https://github.com/apache/doris/pull/41339#issuecomment-2375996634

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Fix](Serde) fix potential mem leak in array serde write_one_cell_to_json [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41339:
URL: https://github.com/apache/doris/pull/41339#issuecomment-2375996619

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [Fix](Serde-2.1) fix potential mem leak in array serde write_one_cell_to_json [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41339:
URL: https://github.com/apache/doris/pull/41339#issuecomment-2376004625

   clang-tidy review says "All clean, LGTM! :+1:"


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [refine](time type) remove and deprecated datatypetimev1 [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41334:
URL: https://github.com/apache/doris/pull/41334#issuecomment-2376007560

   TeamCity be ut coverage result:
Function Coverage: 37.31% (9625/25800) 
Line Coverage: 28.70% (79655/277523)
Region Coverage: 28.13% (41198/146451)
Branch Coverage: 24.76% (20982/84738)
Coverage Report: 
http://coverage.selectdb-in.cc/coverage/8f54a6d67af8d1320868f03550e858f087582600_8f54a6d67af8d1320868f03550e858f087582600/report/index.html


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) abortTransaction does not handle response code [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41275:
URL: https://github.com/apache/doris/pull/41275#issuecomment-2376009125

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch branch-2.0 updated: [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 (#40982)

2024-09-25 Thread lijibing

This is an automated email from the ASF dual-hosted git repository.

lijibing pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
 new 297809ed58e [fix](statistics)Fix empty table keep auto analyze bug. 
(#40811) #40982 (#40982)
297809ed58e is described below

commit 297809ed58e02b08a97472737d58ce5dfbfb4972
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Thu Sep 26 14:04:17 2024 +0800

[fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 
(#40982)

backport: https://github.com/apache/doris/pull/40811
---
 .../doris/statistics/AnalysisInfoBuilder.java  |  2 +-
 .../apache/doris/statistics/TableStatsMeta.java| 12 +++---
 .../doris/statistics/util/StatisticsUtil.java  | 35 ++--
 .../statistics/test_drop_stats_and_truncate.groovy | 48 ++
 4 files changed, 68 insertions(+), 29 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index ddef30ee4de..17a32900be3 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -63,7 +63,7 @@ public class AnalysisInfoBuilder {
 private boolean usingSqlForPartitionColumn;
 private long tblUpdateTime;
 private boolean emptyJob;
-private boolean userInject;
+private boolean userInject = false;
 private long rowCount;
 
 public AnalysisInfoBuilder() {
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index a7701f06ef1..d0673998b54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -24,7 +24,6 @@ import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
-import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
 import org.apache.doris.statistics.AnalysisInfo.JobType;
 import org.apache.doris.statistics.util.StatisticsUtil;
 
@@ -167,7 +166,9 @@ public class TableStatsMeta implements Writable, 
GsonPostProcessable {
 
 public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
 updatedTime = analyzedJob.tblUpdateTime;
-userInjected = analyzedJob.userInject;
+if (analyzedJob.userInject) {
+userInjected = true;
+}
 String colNameStr = analyzedJob.colName;
 // colName field AnalyzeJob's format likes: "[col1, col2]", we need to 
remove brackets here
 // TODO: Refactor this later
@@ -195,9 +196,6 @@ public class TableStatsMeta implements Writable, 
GsonPostProcessable {
 indexesRowCount.putAll(analyzedJob.indexesRowCount);
 clearStaleIndexRowCount((OlapTable) tableIf);
 }
-if (analyzedJob.emptyJob && 
AnalysisMethod.SAMPLE.equals(analyzedJob.analysisMethod)) {
-return;
-}
 if (analyzedJob.colToPartitions.keySet()
 .containsAll(tableIf.getBaseSchema().stream()
 .filter(c -> 
!StatisticsUtil.isUnsupportedType(c.getType()))
@@ -205,6 +203,10 @@ public class TableStatsMeta implements Writable, 
GsonPostProcessable {
 updatedRows.set(0);
 newPartitionLoaded.set(false);
 }
+// Set userInject back to false after manual analyze.
+if (JobType.MANUAL.equals(jobType) && !analyzedJob.userInject) {
+userInjected = false;
+}
 }
 }
 
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index e0eef39a217..fe32755be45 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -976,37 +976,26 @@ public class StatisticsUtil {
 }
 
 public static boolean isEmptyTable(TableIf table, 
AnalysisInfo.AnalysisMethod method) {
-int waitRowCountReportedTime = 90;
+int waitRowCountReportedTime = 120;
 if (!(table instanceof OlapTable) || 
method.equals(AnalysisInfo.AnalysisMethod.FULL)) {
 return false;
 }
 OlapTable olapTable = (OlapTable) table;
+long rowCount = 0;
 for (int i = 0; i < waitRowCountReportedTime; i++) {
-if (olapTable.getRowCount() > 0) {
-return false;
-}

(doris) branch master updated: [fix](cloud) abortTransaction does not handle response code (#41275)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
 new f4c4b27c3cb [fix](cloud) abortTransaction does not handle response 
code (#41275)
f4c4b27c3cb is described below

commit f4c4b27c3cb0f6f94fde74d8be15e7ef9df1dc09
Author: meiyi 
AuthorDate: Thu Sep 26 14:06:00 2024 +0800

[fix](cloud) abortTransaction does not handle response code (#41275)
---
 .../transaction/CloudGlobalTransactionMgr.java | 42 ++
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
 
b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 5508bca5773..10baf4a135c 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -1097,6 +1097,33 @@ public class CloudGlobalTransactionMgr implements 
GlobalTransactionMgrIface {
 }
 throw new UserException("abortTxn failed, errMsg:" + 
e.getMessage());
 }
+afterAbortTxnResp(abortTxnResponse, String.valueOf(transactionId), 
txnCommitAttachment);
+}
+
+private void afterAbortTxnResp(AbortTxnResponse abortTxnResponse, String 
txnIdOrLabel,
+TxnCommitAttachment txnCommitAttachment) throws UserException {
+if (abortTxnResponse.getStatus().getCode() != MetaServiceCode.OK) {
+LOG.warn("abortTxn failed, transaction:{}, response:{}", 
txnIdOrLabel, abortTxnResponse);
+// For routine load, it is necessary to release the write lock 
when abort transaction fails,
+// otherwise it will cause the lock added in beforeAborted to not 
be released.
+if (txnCommitAttachment != null && txnCommitAttachment instanceof 
RLTaskTxnCommitAttachment) {
+RLTaskTxnCommitAttachment rlTaskTxnCommitAttachment = 
(RLTaskTxnCommitAttachment) txnCommitAttachment;
+
Env.getCurrentEnv().getRoutineLoadManager().getJob(rlTaskTxnCommitAttachment.getJobId()).writeUnlock();
+}
+switch (abortTxnResponse.getStatus().getCode()) {
+case TXN_ID_NOT_FOUND:
+case TXN_LABEL_NOT_FOUND:
+case TXN_INVALID_STATUS:
+throw new TransactionNotFoundException("transaction [" + 
txnIdOrLabel + "] not found");
+case TXN_ALREADY_ABORTED:
+throw new TransactionNotFoundException("transaction [" + 
txnIdOrLabel + "] is already aborted");
+case TXN_ALREADY_VISIBLE:
+throw new UserException(
+"transaction [" + txnIdOrLabel + "] is already 
visible, " + ", could not abort");
+default:
+throw new 
UserException(abortTxnResponse.getStatus().getMsg());
+}
+}
 
 TransactionState txnState = 
TxnUtil.transactionStateFromPb(abortTxnResponse.getTxnInfo());
 TxnStateChangeCallback cb = 
callbackFactory.getCallback(txnState.getCallbackId());
@@ -1149,20 +1176,7 @@ public class CloudGlobalTransactionMgr implements 
GlobalTransactionMgrIface {
 LOG.warn("abortTxn failed, label:{}, exception:", label, e);
 throw new UserException("abortTxn failed, errMsg:" + 
e.getMessage());
 }
-
-TransactionState txnState = 
TxnUtil.transactionStateFromPb(abortTxnResponse.getTxnInfo());
-TxnStateChangeCallback cb = 
callbackFactory.getCallback(txnState.getCallbackId());
-if (cb == null) {
-LOG.info("no callback to run for this txn, txnId:{} 
callbackId:{}", txnState.getTransactionId(),
-txnState.getCallbackId());
-return;
-}
-
-LOG.info("run txn callback, txnId:{} callbackId:{}", 
txnState.getTransactionId(), txnState.getCallbackId());
-cb.afterAborted(txnState, true, txnState.getReason());
-if (MetricRepo.isInit) {
-MetricRepo.COUNTER_TXN_FAILED.increase(1L);
-}
+afterAbortTxnResp(abortTxnResponse, label, null);
 }
 
 @Override


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 [doris]

2024-09-25 Thread via GitHub



Jibing-Li merged PR #40982:
URL: https://github.com/apache/doris/pull/40982


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) abortTransaction does not handle response code [doris]

2024-09-25 Thread via GitHub



dataroaring merged PR #41275:
URL: https://github.com/apache/doris/pull/41275


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch master updated: [fix](array-funcs)fix array agg func with decimal type (#40839)

2024-09-25 Thread kxiao

This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
 new 16d05dc2f9d [fix](array-funcs)fix array agg func with decimal type 
(#40839)
16d05dc2f9d is described below

commit 16d05dc2f9ddacce88f40acf32b31c2586c779fa
Author: amory 
AuthorDate: Thu Sep 26 09:38:50 2024 +0800

[fix](array-funcs)fix array agg func with decimal type (#40839)
---
 .../functions/array/function_array_aggregation.cpp |  13 +
 .../functions/ComputePrecisionForArrayItemAgg.java |  10 +-
 .../functions/scalar/ArraysOverlap.java|   3 +-
 .../nereids_function_p0/scalar_function/Array.out  | 816 +
 .../test_array_large_decimal.csv   | 100 +++
 .../suites/nereids_function_p0/load.groovy |  22 +
 .../scalar_function/Array.groovy   |  33 +
 7 files changed, 994 insertions(+), 3 deletions(-)

diff --git a/be/src/vec/functions/array/function_array_aggregation.cpp 
b/be/src/vec/functions/array/function_array_aggregation.cpp
index d2edfe34fb6..18367816bc8 100644
--- a/be/src/vec/functions/array/function_array_aggregation.cpp
+++ b/be/src/vec/functions/array/function_array_aggregation.cpp
@@ -146,6 +146,18 @@ struct ArrayAggregateImpl {
 using Function = AggregateFunction>;
 const DataTypeArray* data_type_array =
 static_cast(remove_nullable(arguments[0]).get());
+if constexpr (operation != AggregateOperation::MIN &&
+  operation != AggregateOperation::MAX) {
+// only array_min and array_max support decimal256 type
+if 
(is_decimal(remove_nullable(data_type_array->get_nested_type( {
+const auto decimal_type = 
remove_nullable(data_type_array->get_nested_type());
+if (check_decimal(*decimal_type)) {
+throw doris::Exception(
+ErrorCode::INVALID_ARGUMENT, "Unexpected type {} 
for aggregation {}",
+data_type_array->get_nested_type()->get_name(), 
operation);
+}
+}
+}
 auto function = Function::create(data_type_array->get_nested_type());
 if (function) {
 return function->get_return_type();
@@ -175,6 +187,7 @@ struct ArrayAggregateImpl {
 execute_type(res, type, data, offsets) ||
 execute_type(res, type, data, offsets) ||
 execute_type(res, type, data, offsets) ||
+execute_type(res, type, data, offsets) ||
 execute_type(res, type, data, offsets) ||
 execute_type(res, type, data, offsets) ||
 execute_type(res, type, data, offsets) ||
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputePrecisionForArrayItemAgg.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputePrecisionForArrayItemAgg.java
index 50c9f1adfdb..05efb9b 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputePrecisionForArrayItemAgg.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputePrecisionForArrayItemAgg.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.types.ArrayType;
 import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.DecimalV3Type;
+import org.apache.doris.qe.ConnectContext;
 
 /** ComputePrecisionForSum */
 public interface ComputePrecisionForArrayItemAgg extends ComputePrecision {
@@ -29,8 +30,15 @@ public interface ComputePrecisionForArrayItemAgg extends 
ComputePrecision {
 if (getArgumentType(0) instanceof ArrayType) {
 DataType itemType = ((ArrayType) 
getArgument(0).getDataType()).getItemType();
 if (itemType instanceof DecimalV3Type) {
+boolean enableDecimal256 = false;
+ConnectContext connectContext = ConnectContext.get();
+if (connectContext != null) {
+enableDecimal256 = 
connectContext.getSessionVariable().isEnableDecimal256();
+}
 DecimalV3Type returnType = DecimalV3Type.createDecimalV3Type(
-DecimalV3Type.MAX_DECIMAL128_PRECISION, 
((DecimalV3Type) itemType).getScale());
+enableDecimal256 ? 
DecimalV3Type.MAX_DECIMAL256_PRECISION
+: DecimalV3Type.MAX_DECIMAL128_PRECISION,
+((DecimalV3Type) itemType).getScale());
 if (signature.returnType instanceof ArrayType) {
 signature = 
signature.withReturnType(ArrayType.of(returnType));
 } else {
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/f

Re: [PR] [improve](array-funcs)support arrays_overlap with invertedIndex (#41161) [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41286:
URL: https://github.com/apache/doris/pull/41286#issuecomment-2375569184

   PR approved by at least one committer and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [improve](array-funcs)support arrays_overlap with invertedIndex (#41161) [doris]

2024-09-25 Thread via GitHub



github-actions[bot] commented on PR #41286:
URL: https://github.com/apache/doris/pull/41286#issuecomment-2375569212

   PR approved by anyone and no changes requested.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](array-funcs)fix array agg func with decimal type [doris]

2024-09-25 Thread via GitHub



xiaokang merged PR #40839:
URL: https://github.com/apache/doris/pull/40839


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch branch-3.0 updated (79a610ec7f2 -> 75d6ae6b1a0)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


from 79a610ec7f2 [revert](storage) storage medium of partition should not 
inherit from table (#41192)
 new 951f31e4a0e [fix](schema change) fix delete predicate incorrect 
comparisons result with empty strings during schema change (#41064)
 new 75d6ae6b1a0 [fix](nereids) prevent null pointer exception if datetime 
value overflows (#39482)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/olap/delete_handler.cpp |  5 +-
 be/src/olap/delete_handler.h   |  3 +-
 be/src/olap/tablet_reader.cpp  |  5 +-
 .../rules/SimplifyComparisonPredicate.java |  5 +-
 .../functions/executable/TimeRoundSeries.java  | 25 ++---
 .../trees/expressions/literal/DateLiteral.java |  2 +-
 .../test_schema_change_delete.out} |  0
 .../datatype/test_datetime_overflow.groovy}| 43 +-
 .../test_schema_change_delete.groovy   | 65 ++
 9 files changed, 106 insertions(+), 47 deletions(-)
 copy 
regression-test/data/{correctness/test_constant_push_down_through_outer_join.out
 => schema_change_p0/test_schema_change_delete.out} (100%)
 copy regression-test/suites/{nereids_syntax_p0/ddl/add_column.groovy => 
nereids_p0/datatype/test_datetime_overflow.groovy} (58%)
 create mode 100644 
regression-test/suites/schema_change_p0/test_schema_change_delete.groovy


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch branch-3.0 updated (d2c0087f6be -> 6add3140098)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


from d2c0087f6be [fix](parser) should not use selectHint in any place 
(#41260)
 add 98758e4d644 [Fix](nereids) fix create view with nullable column 
(#41234)
 add 6add3140098 [fix](block_rule) SQL block rule not working after FE 
restart (#41228)

No new revisions were added by this update.

Summary of changes:
 .../main/java/org/apache/doris/common/Config.java  |  4 ++
 .../org/apache/doris/blockrule/SqlBlockRule.java   | 16 +---
 .../apache/doris/blockrule/SqlBlockRuleMgr.java|  5 +++
 .../trees/plans/commands/info/BaseViewInfo.java|  5 ++-
 .../doris/blockrule/SqlBlockRuleMgrTest.java   | 25 
 .../data/correctness/test_view_varchar_length.out  |  2 +-
 .../data/ddl_p0/test_create_view_nereids.out   | 10 +
 .../suites/ddl_p0/test_create_view_nereids.groovy  | 47 ++
 8 files changed, 105 insertions(+), 9 deletions(-)


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](analysis) Fix ColumnDef to sql result #41205 [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41323:
URL: https://github.com/apache/doris/pull/41323#issuecomment-2375715179

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](analysis) Fix ColumnDef to sql result #41205 [doris]

2024-09-25 Thread via GitHub



w41ter commented on PR #41325:
URL: https://github.com/apache/doris/pull/41325#issuecomment-2375744585

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [case](mtmv)fix alter job case [doris]

2024-09-25 Thread via GitHub



zddr opened a new pull request, #41326:
URL: https://github.com/apache/doris/pull/41326

   should allow task failed
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [case](mtmv)fix alter job case [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #41326:
URL: https://github.com/apache/doris/pull/41326#issuecomment-2375750148

   Thank you for your contribution to Apache Doris.
   Don't know what should be done next? See [How to process your 
PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
   
   Since 2024-03-18, the Document has been moved to 
[doris-website](https://github.com/apache/doris-website).
   See [Doris 
Document](https://cwiki.apache.org/confluence/display/DORIS/Doris+Document).


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) ensure afterCommit/afterAbort will always be called when commit/abort transaction fails [doris]

2024-09-25 Thread via GitHub



sollhui commented on PR #41267:
URL: https://github.com/apache/doris/pull/41267#issuecomment-2375750274

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) 01/04: [fix](mtmv) Mtmv support set both immediate and starttime (#39573)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 7ef4c1333050cc323c1df3e2cdafd4f2c093fb8c
Author: zhangdong <493738...@qq.com>
AuthorDate: Thu Aug 29 15:50:36 2024 +0800

[fix](mtmv) Mtmv support set both immediate and starttime (#39573)

extends: #36805

Previously, if the user set immediate execution, the current time would
replace the user's set start time
---
 .../java/org/apache/doris/mtmv/MTMVJobManager.java |  8 +--
 .../data/mtmv_p0/test_start_time_mtmv.out  | 17 +
 .../suites/mtmv_p0/test_start_time_mtmv.groovy | 76 ++
 3 files changed, 97 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java 
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
index 11089899b30..1ace738f1d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
@@ -105,14 +105,14 @@ public class MTMVJobManager implements MTMVHookService {
 
.setInterval(mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getInterval());
 timerDefinition
 
.setIntervalUnit(mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getTimeUnit());
-if (mtmv.getRefreshInfo().getBuildMode().equals(BuildMode.IMMEDIATE)) {
-jobExecutionConfiguration.setImmediate(true);
-} else if 
(mtmv.getRefreshInfo().getBuildMode().equals(BuildMode.DEFERRED) && !StringUtils
+if (!StringUtils
 
.isEmpty(mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getStartTime()))
 {
 timerDefinition.setStartTimeMs(TimeUtils.timeStringToLong(
 
mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getStartTime()));
 }
-
+if (mtmv.getRefreshInfo().getBuildMode().equals(BuildMode.IMMEDIATE)) {
+jobExecutionConfiguration.setImmediate(true);
+}
 jobExecutionConfiguration.setTimerDefinition(timerDefinition);
 }
 
diff --git a/regression-test/data/mtmv_p0/test_start_time_mtmv.out 
b/regression-test/data/mtmv_p0/test_start_time_mtmv.out
new file mode 100644
index 000..8e17dcacec9
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_start_time_mtmv.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you 
want to edit this
+-- !init --
+1  1
+2  2
+3  3
+
+-- !create --
+EVERY 2 HOUR STARTS -12-13 21:07:09
+
+-- !alter --
+EVERY 2 HOUR STARTS 9998-12-13 21:07:09
+
+-- !refresh --
+1  1
+2  2
+3  3
+
diff --git a/regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy 
b/regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy
new file mode 100644
index 000..89872adc253
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_start_time_mtmv","mtmv") {
+String suiteName = "test_start_time_mtmv"
+String tableName = "${suiteName}_table"
+String mvName = "${suiteName}_mv"
+
+sql """drop table if exists `${tableName}`"""
+sql """drop materialized view if exists ${mvName};"""
+
+sql """
+CREATE TABLE ${tableName}
+(
+k2 INT,
+k3 varchar(32)
+)
+DISTRIBUTED BY HASH(k2) BUCKETS 2
+PROPERTIES (
+"replication_num" = "1"
+);
+"""
+
+sql """
+insert into ${tableName} values(1,1),(2,2),(3,3);
+"""
+
+sql """
+CREATE MATERIALIZED VIEW ${mvName}
+BUILD immediate REFRESH AUTO ON SCHEDULE EVERY 2 HOUR STARTS 
"-12-13 21:07:09"
+DISTRIBUTED BY RANDOM BUCKETS 2
+PROPERTIES (
+'replication_num' = '1'
+)
+AS
+SELECT * from ${tableName};
+"""
+
+waitingMTMVTaskFinishedByMvName(mvName)
+order_qt_init "SELECT * FROM ${mvName}"
+
+or

(doris) 01/02: [fix](schema change) fix delete predicate incorrect comparisons result with empty strings during schema change (#41064)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 951f31e4a0e98d26ca1442eafcfabf25f4a3c7a0
Author: Luwei <814383...@qq.com>
AuthorDate: Thu Sep 26 10:35:27 2024 +0800

[fix](schema change) fix delete predicate incorrect comparisons result with 
empty strings during schema change (#41064)

### problem
```
insert into t1 (k1,  k2,  v1) value(1, '', 2);
delete form t1 where k1 = 1 and k2 = '';
alter table modify column v1 string
select * from t1 //  expect 0 rows in return, but get 1 row.
```
---
 be/src/olap/delete_handler.cpp |  5 +-
 be/src/olap/delete_handler.h   |  3 +-
 be/src/olap/tablet_reader.cpp  |  5 +-
 .../schema_change_p0/test_schema_change_delete.out |  3 +
 .../test_schema_change_delete.groovy   | 65 ++
 5 files changed, 72 insertions(+), 9 deletions(-)

diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp
index 66859a069cc..e7fc3eed263 100644
--- a/be/src/olap/delete_handler.cpp
+++ b/be/src/olap/delete_handler.cpp
@@ -391,8 +391,7 @@ template Status 
DeleteHandler::_parse_column_pred(
 DeleteConditions* delete_conditions);
 
 Status DeleteHandler::init(TabletSchemaSPtr tablet_schema,
-   const std::vector& 
delete_preds, int64_t version,
-   bool with_sub_pred_v2) {
+   const std::vector& 
delete_preds, int64_t version) {
 DCHECK(!_is_inited) << "reinitialize delete handler.";
 DCHECK(version >= 0) << "invalid parameters. version=" << version;
 _predicate_arena = std::make_unique();
@@ -407,7 +406,7 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema,
 const auto& delete_condition = delete_pred->delete_predicate();
 DeleteConditions temp;
 temp.filter_version = delete_pred->version().first;
-if (with_sub_pred_v2 && !delete_condition.sub_predicates_v2().empty()) 
{
+if (!delete_condition.sub_predicates_v2().empty()) {
 RETURN_IF_ERROR(_parse_column_pred(tablet_schema, 
delete_pred_related_schema,

delete_condition.sub_predicates_v2(), &temp));
 } else {
diff --git a/be/src/olap/delete_handler.h b/be/src/olap/delete_handler.h
index 0910795a81c..cc585c0abcf 100644
--- a/be/src/olap/delete_handler.h
+++ b/be/src/olap/delete_handler.h
@@ -104,8 +104,7 @@ public:
 // * Status::Error(): input parameters are 
not valid
 // * Status::Error(): alloc memory failed
 Status init(TabletSchemaSPtr tablet_schema,
-const std::vector& delete_preds, int64_t 
version,
-bool with_sub_pred_v2 = false);
+const std::vector& delete_preds, int64_t 
version);
 
 [[nodiscard]] bool empty() const { return _del_conds.empty(); }
 
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index dcdd8819e3d..9ab9e4b1b36 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -635,11 +635,8 @@ Status TabletReader::_init_delete_condition(const 
ReaderParams& read_params) {
 // However, queries will not use this condition but generate special where 
predicates to filter data.
 // (Though a lille bit confused, it is how the current logic working...)
 _filter_delete = _delete_sign_available || cumu_delete;
-auto* runtime_state = read_params.runtime_state;
-bool enable_sub_pred_v2 =
-runtime_state == nullptr ? true : 
runtime_state->enable_delete_sub_pred_v2();
 return _delete_handler.init(_tablet_schema, read_params.delete_predicates,
-read_params.version.second, 
enable_sub_pred_v2);
+read_params.version.second);
 }
 
 Status TabletReader::init_reader_params_and_create_block(
diff --git 
a/regression-test/data/schema_change_p0/test_schema_change_delete.out 
b/regression-test/data/schema_change_p0/test_schema_change_delete.out
new file mode 100644
index 000..f958424e65c
--- /dev/null
+++ b/regression-test/data/schema_change_p0/test_schema_change_delete.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you 
want to edit this
+-- !select --
+
diff --git 
a/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy 
b/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy
new file mode 100644
index 000..873cf2d4c0a
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Th

(doris) branch branch-3.0 updated (75d6ae6b1a0 -> 03e7e521ec9)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


from 75d6ae6b1a0 [fix](nereids) prevent null pointer exception if datetime 
value overflows (#39482)
 new 7ef4c133305 [fix](mtmv) Mtmv support set both immediate and starttime 
(#39573)
 new bddfcc93a7e [fix](Nereids) mow with sync mv could not be deleted 
(#39578)
 new 7070428c941 [fix](mtmv) mtmv getPlanBySql should not reuse ctx's 
StatementContext (#39690)
 new 03e7e521ec9 [Fix](Nereids) fix use cbo rule hint unused because of 
logic reverse (#39715)

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../java/org/apache/doris/mtmv/MTMVJobManager.java |   8 +-
 .../java/org/apache/doris/mtmv/MTMVPlanUtil.java   |  11 +-
 .../nereids/jobs/rewrite/CostBasedRewriteJob.java  |   2 +-
 .../trees/plans/commands/DeleteFromCommand.java|  14 +-
 ...ate_table_mtmv.out => test_start_time_mtmv.out} |  11 +-
 .../infer_set_operator_distinct.out| 230 +
 .../delete_p0/test_delete_with_sync_mv.groovy  |  76 +++
 ...iew_mtmv.groovy => test_start_time_mtmv.groovy} |  44 ++--
 .../four/load_four_step.groovy |   6 +-
 9 files changed, 276 insertions(+), 126 deletions(-)
 copy regression-test/data/mtmv_p0/{test_truncate_table_mtmv.out => 
test_start_time_mtmv.out} (50%)
 create mode 100644 
regression-test/suites/delete_p0/test_delete_with_sync_mv.groovy
 copy regression-test/suites/mtmv_p0/{test_create_view_mtmv.groovy => 
test_start_time_mtmv.groovy} (70%)


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](pipeline) Make all upstream tasks runnable if all tasks finishe… [doris]

2024-09-25 Thread via GitHub



Gabriel39 commented on PR #41292:
URL: https://github.com/apache/doris/pull/41292#issuecomment-2375750670

   run buildall


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](cloud) ensure afterCommit/afterAbort will always be called when commit/abort transaction fails [doris]

2024-09-25 Thread via GitHub



sollhui commented on code in PR #41267:
URL: https://github.com/apache/doris/pull/41267#discussion_r1774773594


##
fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java:
##
@@ -990,9 +994,22 @@ public boolean commitAndPublishTransaction(DatabaseIf db, 
long transactionId,
 }
 
 final CommitTxnRequest commitTxnRequest = builder.build();
-commitTxn(commitTxnRequest, transactionId, false, db.getId(),
-
subTransactionStates.stream().map(SubTransactionState::getTable)
-.collect(Collectors.toList()));
+TransactionState txnState = null;
+CommitTxnResponse commitTxnResponse = null;
+try {
+commitTxn(commitTxnRequest, transactionId, false, db.getId(),
+
subTransactionStates.stream().map(SubTransactionState::getTable)
+.collect(Collectors.toList()), txnState, 
commitTxnResponse);
+} finally {
+TxnStateChangeCallback cb = 
callbackFactory.getCallback(txnState.getCallbackId());
+if (cb != null) {
+LOG.info("commitTxn, run txn callback, transactionId:{} 
callbackId:{}, txnState:{}",
+txnState.getTransactionId(), txnState.getCallbackId(), 
txnState);
+cb.afterCommitted(txnState, true);

Review Comment:
   BeforeCommit will not be called in this method.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [cherry-pick](branch-21) fix tablet sink shuffle without project not match the output tuple (#40299)(#41293) [doris]

2024-09-25 Thread via GitHub



zhangstar333 opened a new pull request, #41327:
URL: https://github.com/apache/doris/pull/41327

   ## Proposed changes
   
   cherry-pick from master  (#40299)(#41293)
   
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [fix](analysis) Fix ColumnDef to sql result #41205 [doris]

2024-09-25 Thread via GitHub



w41ter opened a new pull request, #41325:
URL: https://github.com/apache/doris/pull/41325

   cherry pick from #41205


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) branch branch-3.0 updated: [revert](storage) storage medium of partition should not inherit from table (#41192)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
 new 79a610ec7f2 [revert](storage) storage medium of partition should not 
inherit from table (#41192)
79a610ec7f2 is described below

commit 79a610ec7f2a6fde8cda24a5f4b891c3e983762f
Author: zhangdong <493738...@qq.com>
AuthorDate: Tue Sep 24 14:35:21 2024 +0800

[revert](storage) storage medium of partition should not inherit from table 
(#41192)

revert #35644

table if not set storage medium, will set default by
`Config.default_storage_medium`,
at present, we cannot distinguish whether the storage medium of the
table is set by the user.
If it is not set by the user, it represents priority use rather than
mandatory use. If we directly inherit the storage medium of the table,
it may cause the partition to be forced to use 'Config.
default_storage_madium', resulting in the partition being unable to be
created
---
 .../src/main/java/org/apache/doris/datasource/InternalCatalog.java | 7 ---
 regression-test/suites/mtmv_p0/test_storage_medium_mtmv.groovy | 2 ++
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java 
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 033af006dd8..4edf61d8342 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1663,13 +1663,6 @@ public class InternalCatalog implements 
CatalogIf {
 if 
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) {
 properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, 
olapTable.getStoragePolicy());
 }
-if 
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM)) {
-TStorageMedium tableStorageMedium = 
olapTable.getStorageMedium();
-if (tableStorageMedium != null) {
-properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM,
-tableStorageMedium.name().toLowerCase());
-}
-}
 
 
singlePartitionDesc.analyze(partitionInfo.getPartitionColumns().size(), 
properties);
 partitionInfo.createAndCheckPartitionItem(singlePartitionDesc, 
isTempPartition);
diff --git a/regression-test/suites/mtmv_p0/test_storage_medium_mtmv.groovy 
b/regression-test/suites/mtmv_p0/test_storage_medium_mtmv.groovy
index 59116eb264e..0432245d808 100644
--- a/regression-test/suites/mtmv_p0/test_storage_medium_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_storage_medium_mtmv.groovy
@@ -18,6 +18,8 @@
 import org.junit.Assert;
 
 suite("test_storage_medium_mtmv","mtmv") {
+// current, can not support extend storage medium from table
+return;
 // cloud not support set storage medium
 if (isCloudMode()) {
 return;


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Re: [PR] [fix](statistics)Fix drop stats log editlog bug. Catch drop stats exception while truncate table. (#40738) [doris]

2024-09-25 Thread via GitHub



doris-robot commented on PR #40979:
URL: https://github.com/apache/doris/pull/40979#issuecomment-2375756069

   
   
   TPC-H: Total hot run time: 49733 ms
   
   ```
   machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
   scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
   Tpch sf100 test result on commit 32d4321d73d462358cee535f4bf7576b11f3c35e, 
data reload: false
   
   -- Round 1 --
   q1   17686   434042984298
   q2   2058158 144 144
   q3   10385   194719231923
   q4   10367   123213471232
   q5   8597396939173917
   q6   232 125 124 124
   q7   2063161015971597
   q8   9285274927222722
   q9   10357   996997869786
   q10  8661356036063560
   q11  420 251 257 251
   q12  466 300 297 297
   q13  18348   397740403977
   q14  349 335 347 335
   q15  520 489 457 457
   q16  529 474 457 457
   q17  1124989 939 939
   q18  7724727172507250
   q19  1706159815891589
   q20  530 336 290 290
   q21  4510417941754175
   q22  510 415 413 413
   Total cold run time: 116427 ms
   Total hot run time: 49733 ms
   
   - Round 2, with runtime_filter_mode=off -
   q1   4357435244024352
   q2   326 228 229 228
   q3   4167412441314124
   q4   2730273027342730
   q5   7205710570977097
   q6   234 117 117 117
   q7   3268287228632863
   q8   4376450044854485
   q9   13700   13569   13612   13569
   q10  4222429742804280
   q11  770 677 675 675
   q12  1032844 852 844
   q13  6874376637563756
   q14  458 421 416 416
   q15  495 455 448 448
   q16  639 593 595 593
   q17  3878393739063906
   q18  8689864386858643
   q19  1713166216631662
   q20  2383210921152109
   q21  8600844484958444
   q22  1011960 899 899
   Total cold run time: 81127 ms
   Total hot run time: 76240 ms
   ```
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

[PR] [cherry-pick](branch-30) execute expr should use local states instead of operators(#40189) [doris]

2024-09-25 Thread via GitHub



zhangstar333 opened a new pull request, #41324:
URL: https://github.com/apache/doris/pull/41324

   
   
   The expr of operator  cannot be executed concurrently, should use local 
state's expr.
   cherry-pick from master https://github.com/apache/doris/pull/40189
   
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

(doris) 02/02: [Fix](Column) refactor ColumnNullable to provide flags safety (#40769)

2024-09-25 Thread dataroaring

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 6d0b67a76e45ebcc5c31e738a76416ec49ae5701
Author: zclllhhjj 
AuthorDate: Sat Sep 14 10:22:04 2024 +0800

[Fix](Column) refactor ColumnNullable to provide flags safety (#40769)

## Proposed changes

Issue Number: close #xxx

In this pr we move null_map and relative flags to an individual base
class to constrain its visibility.
please reed the comment carefully

test in master:
```
xxx/column_nullable_test.cpp:187: Failure
Value of: null_dst->has_null()
  Actual: false
Expected: true

Error occurred in case0
[  FAILED  ] ColumnNullableTest.PredicateTest (0 ms)
[ RUN  ] ColumnNullableTest.HashTest
[   OK ] ColumnNullableTest.HashTest (0 ms)
[--] 3 tests from ColumnNullableTest (0 ms total)

[--] Global test environment tear-down
[==] 3 tests from 1 test suite ran. (1 ms total)
[  PASSED  ] 2 tests.
[  FAILED  ] 1 test, listed below:
[  FAILED  ] ColumnNullableTest.PredicateTest
```

and success after this pr

-

Co-authored-by: Jerry Hu 
---
 be/src/vec/columns/column_nullable.cpp |  85 +---
 be/src/vec/columns/column_nullable.h   | 153 ++---
 .../column_nullable_seriazlization_test.cpp|  75 +-
 be/test/vec/columns/column_nullable_test.cpp   | 112 +++
 be/test/vec/columns/column_nullable_test.h | 105 ++
 be/test/vec/columns/column_resize_test.cpp |   1 -
 be/test/vec/function/function_test_util.h  |   8 ++
 .../correctness/test_column_nullable_cache.out |   6 +
 .../correctness/test_column_nullable_cache.groovy  |  57 
 9 files changed, 426 insertions(+), 176 deletions(-)

diff --git a/be/src/vec/columns/column_nullable.cpp 
b/be/src/vec/columns/column_nullable.cpp
index 483ed5ca6cd..dbee5a2025a 100644
--- a/be/src/vec/columns/column_nullable.cpp
+++ b/be/src/vec/columns/column_nullable.cpp
@@ -31,18 +31,19 @@
 namespace doris::vectorized {
 
 ColumnNullable::ColumnNullable(MutableColumnPtr&& nested_column_, 
MutableColumnPtr&& null_map_)
-: nested_column(std::move(nested_column_)), 
null_map(std::move(null_map_)) {
+: NullMapProvider(std::move(null_map_)), 
nested_column(std::move(nested_column_)) {
 /// ColumnNullable cannot have constant nested column. But constant 
argument could be passed. Materialize it.
 nested_column = get_nested_column().convert_to_full_column_if_const();
 
 // after convert const column to full column, it may be a nullable column
 if (nested_column->is_nullable()) {
-assert_cast(*nested_column).apply_null_map((const 
ColumnUInt8&)*null_map);
-null_map = 
assert_cast(*nested_column).get_null_map_column_ptr();
+assert_cast(*nested_column)
+.apply_null_map(static_cast(get_null_map_column()));
+
reset_null_map(assert_cast(*nested_column).get_null_map_column_ptr());
 nested_column = 
assert_cast(*nested_column).get_nested_column_ptr();
 }
 
-if (is_column_const(*null_map)) {
+if (is_column_const(get_null_map_column())) [[unlikely]] {
 throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"ColumnNullable cannot have constant null map");
 __builtin_unreachable();
@@ -69,7 +70,7 @@ void ColumnNullable::update_xxHash_with_value(size_t start, 
size_t end, uint64_t
 nested_column->update_xxHash_with_value(start, end, hash, nullptr);
 } else {
 const auto* __restrict real_null_data =
-assert_cast(*null_map).get_data().data();
+assert_cast(get_null_map_column()).get_data().data();
 for (int i = start; i < end; ++i) {
 if (real_null_data[i] != 0) {
 hash = HashUtil::xxHash64NullWithSeed(hash);
@@ -85,7 +86,7 @@ void ColumnNullable::update_crc_with_value(size_t start, 
size_t end, uint32_t& h
 nested_column->update_crc_with_value(start, end, hash, nullptr);
 } else {
 const auto* __restrict real_null_data =
-assert_cast(*null_map).get_data().data();
+assert_cast(get_null_map_column()).get_data().data();
 for (int i = start; i < end; ++i) {
 if (real_null_data[i] != 0) {
 hash = HashUtil::zlib_crc_hash_null(hash);
@@ -110,7 +111,7 @@ void ColumnNullable::update_crcs_with_value(uint32_t* 
__restrict hashes, doris::
 auto s = rows;
 DCHECK(s == size());
 const auto* __restrict real_null_data =
-assert_cast(*null_map).get_data().data();
+assert_cast(get_null_map_column()).get_data().data();
 if (!has_null()) {
 nested_column->update_crcs_with_value(hashes,

1 2 3 4 5 6 7 >

1 - 100 of 681 matches

Mail list logo