[GitHub] [incubator-doris] weizuo93 commented on pull request #5841: [Bug][Stream load] Separate the execution of stream load tasks from the HTTP thread pool
weizuo93 commented on pull request #5841: URL: https://github.com/apache/incubator-doris/pull/5841#issuecomment-849392855 Please don't merge. there are some problems in UT, I will fix it later. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] acelyc111 commented on issue #5922: BE无法启动
acelyc111 commented on issue #5922: URL: https://github.com/apache/incubator-doris/issues/5922#issuecomment-849413595 What version you are using? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangzhg merged pull request #5894: use virtual hosted-style request to access object store
yangzhg merged pull request #5894: URL: https://github.com/apache/incubator-doris/pull/5894 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: use virtual hosted-style request to access object store (#5894)
This is an automated email from the ASF dual-hosted git repository. yangzhg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new ba38973 use virtual hosted-style request to access object store (#5894) ba38973 is described below commit ba38973209e70312e052f0220ef783180a1f4442 Author: Zhengguo Yang AuthorDate: Thu May 27 15:52:07 2021 +0800 use virtual hosted-style request to access object store (#5894) * use virtual hosted-style access request object store --- be/src/common/daemon.cpp | 18 +-- be/src/exec/s3_reader.cpp | 4 +- be/src/exec/s3_reader.h| 2 +- be/src/exec/s3_writer.cpp | 12 +- be/src/exec/s3_writer.h| 2 +- be/src/util/s3_storage_backend.cpp | 2 +- be/src/util/s3_storage_backend.h | 2 +- be/src/util/s3_util.cpp| 56 --- be/src/util/s3_util.h | 14 ++ be/test/exec/s3_reader_test.cpp| 15 +- .../java/org/apache/doris/backup/S3Storage.java| 167 - .../java/org/apache/doris/common/util/S3URI.java | 28 +++- .../org/apache/doris/persist/gson/GsonUtils.java | 4 +- 13 files changed, 192 insertions(+), 134 deletions(-) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 5b21ea0..d70ee07 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -17,11 +17,9 @@ #include "common/daemon.h" -#include - -#include #include #include +#include #include "common/config.h" #include "exprs/bitmap_function.h" @@ -68,8 +66,6 @@ namespace doris { bool k_doris_exit = false; -Aws::SDKOptions aws_options; - void Daemon::tcmalloc_gc_thread() { while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(10))) { size_t used_size = 0; @@ -268,14 +264,6 @@ void Daemon::init(int argc, char** argv, const std::vector& paths) { HllFunctions::init(); HashFunctions::init(); TopNFunctions::init(); -// disable EC2 metadata service -setenv("AWS_EC2_METADATA_DISABLED", "true", false); -Aws::Utils::Logging::LogLevel logLevel = static_cast(config::aws_log_level); -aws_options.loggingOptions.logLevel = logLevel; -aws_options.loggingOptions.logger_create_fn = [logLevel] { -return std::make_shared(logLevel); -}; -Aws::InitAPI(aws_options); LOG(INFO) << CpuInfo::debug_string(); LOG(INFO) << DiskInfo::debug_string(); @@ -303,7 +291,8 @@ void Daemon::start() { if (config::enable_metric_calculator) { CHECK(DorisMetrics::instance()->is_inited()) << "enable metric calculator failed, maybe you set enable_system_metrics to false " -<< " or there may be some hardware error which causes metric init failed, please check log first;" +<< " or there may be some hardware error which causes metric init failed, please " + "check log first;" << " you can set enable_metric_calculator = false to quickly recover "; st = Thread::create( @@ -325,7 +314,6 @@ void Daemon::stop() { if (_calculate_metrics_thread) { _calculate_metrics_thread->join(); } -Aws::ShutdownAPI(aws_options); } } // namespace doris diff --git a/be/src/exec/s3_reader.cpp b/be/src/exec/s3_reader.cpp index ddc71c6..ae5cc3b 100644 --- a/be/src/exec/s3_reader.cpp +++ b/be/src/exec/s3_reader.cpp @@ -41,8 +41,8 @@ S3Reader::S3Reader(const std::map& properties, const s _uri(path), _cur_offset(start_offset), _file_size(0), - _closed(false) { -_client = create_client(_properties); + _closed(false), + _client(ClientFactory::instance().create(_properties)) { DCHECK(_client) << "init aws s3 client error."; } diff --git a/be/src/exec/s3_reader.h b/be/src/exec/s3_reader.h index cd5c185..1676ba5 100644 --- a/be/src/exec/s3_reader.h +++ b/be/src/exec/s3_reader.h @@ -64,6 +64,6 @@ private: int64_t _cur_offset; int64_t _file_size; bool _closed; -std::unique_ptr _client; +std::shared_ptr _client; }; } // end namespace doris diff --git a/be/src/exec/s3_writer.cpp b/be/src/exec/s3_writer.cpp index 557c004..a8cb377 100644 --- a/be/src/exec/s3_writer.cpp +++ b/be/src/exec/s3_writer.cpp @@ -41,12 +41,10 @@ S3Writer::S3Writer(const std::map& properties, const s _path(path), _uri(path), _sync_needed(false), - _temp_file(Aws::MakeShared( - "S3WRITER", - // "/tmp/doris_tmp_", "s3tmp", + _client(ClientFactory::instance().create(_properties)), + _temp_file(std::make_shared( std::ios_base::binar
[GitHub] [incubator-doris] morningman opened a new pull request #5926: [Optimize] Optimize some tablet scheduling logic
morningman opened a new pull request #5926: URL: https://github.com/apache/incubator-doris/pull/5926 ## Proposed changes 1. The partitions set by the admin repair command are prioritized to ensure that the tablets of these partitions can be repaired as soon as possible. 2. Add an FE metric "query_begin" to monitor the number of queries submitted to the Doris. ## Types of changes What types of changes does your code introduce to Doris? _Put an `x` in the boxes that apply_ - [x] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation Update (if none of the other choices apply) - [ ] Code refactor (Modify the code structure, format the code, etc...) ## Checklist _Put an `x` in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code._ - [ ] I have created an issue on (Fix #ISSUE) and described the bug/feature there in detail - [ ] Compiling and unit tests pass locally with my changes - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] If these changes need document changes, I have updated the document - [ ] Any dependent changes have been merged ## Further comments If this is a relatively large or complex change, kick off the discussion at d...@doris.apache.org by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] geekxingyun opened a new issue #5927: build fail apache-doris-0.14.0 ,due to Forbidden!
geekxingyun opened a new issue #5927: URL: https://github.com/apache/incubator-doris/issues/5927 When I run the command as below: `sh build.sh` It display forbidden download the tar, could you fixed the access ? https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz **To Reproduce** Steps to reproduce the behavior: 1. sh build.sh 2. wait 3. you will found "Forbidden! " https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz 4. See error **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots**  **Desktop (please complete the following information):** - OS: Centos7 - Version: apache-doris-0.14.0,open jdk11,maven 3.8.1 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] DongTL opened a new issue #5928: 使用flink插入数据停止
DongTL opened a new issue #5928: URL: https://github.com/apache/incubator-doris/issues/5928 我使用flinkCDC读取mysql多张表数据写入doris, mysql中多张表数据共800+条,但doris中数据只有501条,分组查看结果如下: +-+--+ | date | count(1) | +-+--+ | 2021-01 | 135 | | 2021-02 | 28 | | 2021-03 | 192 | | 2021-04 | 138 | | 2021-05 | 8 | +-+--+ 通过对比,前四个月数据均正确,但5月份数据只有8条,实际有300+条 flink程序无报错,还在继续运行,但数据迟迟没有同步过来。 单独同步这五月这一张表是正常的。 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] stdpain commented on issue #5927: build fail apache-doris-0.14.0 ,due to Forbidden!
stdpain commented on issue #5927: URL: https://github.com/apache/incubator-doris/issues/5927#issuecomment-849526520 You can refer to the changes to `thirdparty/vars.sh` in this PR #5833 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] stdpain commented on pull request #5925: The storage format should not be converted from v1 to v2 by default when alter table's schema
stdpain commented on pull request #5925: URL: https://github.com/apache/incubator-doris/pull/5925#issuecomment-849527881 SegmentV1 will be deprecated in the future, so I don't think this change is necessary. Insisting on maintaining SegmentV1 will be a big burden in future vectorization support -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangmingjie2018 commented on issue #5928: 使用flink插入数据停止
yangmingjie2018 commented on issue #5928: URL: https://github.com/apache/incubator-doris/issues/5928#issuecomment-849529458 哥们 你跟我用的技术栈很相似,加个好友吧,我的微信号:yangmingjie2013 这种情况可能是长时间没有数据引起的,binlog的位移点消失; -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangmingjie2018 commented on issue #5928: 使用flink插入数据停止
yangmingjie2018 commented on issue #5928: URL: https://github.com/apache/incubator-doris/issues/5928#issuecomment-849529734 跟doris关系不大 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangmingjie2018 opened a new issue #5929: Doris有支持多表 物化视图 的计划吗?
yangmingjie2018 opened a new issue #5929: URL: https://github.com/apache/incubator-doris/issues/5929 例如: create materialized view store_amt as select a.store_id, sum(sale_amt) from sales_records a **join salerStore b on a.storeId = b.id** group by a.store_id; -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] mmjj opened a new issue #5930: 请问预计什么版本可以支持存储过程?或者是否由此计划?
mmjj opened a new issue #5930: URL: https://github.com/apache/incubator-doris/issues/5930 请问预计什么版本可以支持存储过程?或者是否由此计划? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #5880: [Optimize] Add db,table information in transaction exception log
morningman commented on a change in pull request #5880: URL: https://github.com/apache/incubator-doris/pull/5880#discussion_r640546507 ## File path: fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java ## @@ -526,11 +530,23 @@ public synchronized void addTableIndexes(OlapTable table) { @Override public String toString() { +Database db = Catalog.getCurrentCatalog().getDb(dbId); +String dbName = null; +String tableNameList = null; +if (db != null) { + dbName = db.getFullName(); + tableNameList = tableIdList.stream() + .map(db::getTable) + .filter(Objects::nonNull) + .map(Table::getName).collect(Collectors.joining(",")); +} Review comment: We have add `show database id` stmt to find database name from database id. So there is no need to print database name here. And a database may have thousands of tables, so print all table here may be disaster. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] xy720 opened a new issue #5931: [Bug] Timeout export job sometimes stuck in exporting state
xy720 opened a new issue #5931: URL: https://github.com/apache/incubator-doris/issues/5931 **Describe the bug** Sometimes export job will stuck in exporting state when it times out. **To Reproduce** Steps to reproduce the behavior: 1. start a export job and set a very short timeout. ``` export table testTbl to "hdfs://xxx:54310/app/xxx/palo/test_table/" properties ("timeout" = "1") with broker "doris" ("username" = "xxx", "password" = "xxx"); ``` 2. export job will stuck in exporting state. **Expected behavior** The timeout export job should be canceled. **Screenshots**  **Desktop (please complete the following information):** - OS: [e.g. iOS] - Browser [e.g. chrome, safari] - Version [e.g. 22] **Smartphone (please complete the following information):** - Device: [e.g. iPhone6] - OS: [e.g. iOS8.1] - Browser [e.g. stock browser, safari] - Version [e.g. 22] **Additional context** Add any other context about the problem here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] xy720 opened a new pull request #5932: [Bug] Fix export job sometimes stuck in exporting state after timeout
xy720 opened a new pull request #5932: URL: https://github.com/apache/incubator-doris/pull/5932 ## Proposed changes Fix #5931 The reason is that sometime the method `coordinate.exec()` is not call when the job is timeout, so that the query profile in this coordinate is not be initialized, which will cause an NPE error in the execution of ExportExportingTask. ## Types of changes What types of changes does your code introduce to Doris? _Put an `x` in the boxes that apply_ - [x] Bugfix (non-breaking change which fixes an issue) ## Checklist _Put an `x` in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code._ - [x] I have created an issue on (Fix #ISSUE) and described the bug/feature there in detail - [x] Compiling and unit tests pass locally with my changes ## Further comments If this is a relatively large or complex change, kick off the discussion at d...@doris.apache.org by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] caiconghui commented on pull request #5925: The storage format should not be converted from v1 to v2 by default when alter table's schema
caiconghui commented on pull request #5925: URL: https://github.com/apache/incubator-doris/pull/5925#issuecomment-849659166 > SegmentV1 will be deprecated in the future, so I don't think this change is necessary. > Insisting on maintaining SegmentV1 will be a big burden in future vectorization support Yes , I agree with you that SegmentV1 will be deprecated in the future, segment V2 will replace all segment v1 in the future. This PR is not Insisting on maintaining SegmentV1 but to be compatible with older doris version. For new doris user , This PR doesn't take any effect, all newer table will be created by segment v2 For older doris user, there are many tables with segment v1, and it need to gradually change V1 to v2, actually, for many older doris user, the most important thing is stable on production enviroment, but now, Very few alter operations succeed, most fail due to timeout, because convert v1 to v2 is a huge task which would cost a lot of time while there are many tablets in older table. and once timeout, all operations that convert v1 to v2 would rollback, which is not what we expect, and make doris not friendly to older user If we always alter table failed, it is meanless to convert v1 to v2 when do alter table, I think it is not a good idea to convert v1 to v2 when user just want to add column for table, which now seems an impossible goal to achive -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5864: [Docs] Fix typo in docs zh-CN CREATE TABLE.md
morningman merged pull request #5864: URL: https://github.com/apache/incubator-doris/pull/5864 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated (ba38973 -> cbe1b48)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git. from ba38973 use virtual hosted-style request to access object store (#5894) add cbe1b48 [Docs] Fix typo in docs zh-CN CREATE TABLE.md (#5864) No new revisions were added by this update. Summary of changes: .../sql-reference/sql-statements/Data Definition/CREATE TABLE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5865: [Doc] Fix demo in 'CREATE TABLE'
morningman merged pull request #5865: URL: https://github.com/apache/incubator-doris/pull/5865 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated (cbe1b48 -> 3cb0174)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git. from cbe1b48 [Docs] Fix typo in docs zh-CN CREATE TABLE.md (#5864) add 3cb0174 [Doc] Fix demo in 'CREATE TABLE' (#5865) No new revisions were added by this update. Summary of changes: docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md | 5 +++-- .../sql-reference/sql-statements/Data Definition/CREATE TABLE.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated (3cb0174 -> 0f4a39f)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git. from 3cb0174 [Doc] Fix demo in 'CREATE TABLE' (#5865) add 0f4a39f [LOG]Hiding stack info of memory exceed in the log (#5896) No new revisions were added by this update. Summary of changes: be/src/runtime/mem_tracker.cpp | 8 1 file changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5897: [Bug] Fix the problem that the result of query from the view is incorrect
morningman merged pull request #5897: URL: https://github.com/apache/incubator-doris/pull/5897 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #5860: [BUG] In view query, the value of NOT(con1 OR con2) expression is incorrect
morningman closed issue #5860: URL: https://github.com/apache/incubator-doris/issues/5860 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5896: [LOG]Hiding stack info of memory exceed in the log
morningman merged pull request #5896: URL: https://github.com/apache/incubator-doris/pull/5896 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated (0f4a39f -> d958bbe)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git. from 0f4a39f [LOG]Hiding stack info of memory exceed in the log (#5896) add d958bbe [Bug] Fix the problem that the result of query from the view is incorrect (#5860) (#5897) No new revisions were added by this update. Summary of changes: .../apache/doris/analysis/CompoundPredicate.java | 3 +- .../java/org/apache/doris/analysis/SelectStmt.java | 11 +++-- .../org/apache/doris/analysis/SelectStmtTest.java | 56 +++--- .../org/apache/doris/analysis/SqlModeTest.java | 2 +- .../org/apache/doris/qe/PartitionCacheTest.java| 14 +++--- 5 files changed, 45 insertions(+), 41 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5901: [Bug] Bucket Shuffle Join may cause:Failed to send brpc batch, Not connected to 0.0.0.0:0
morningman merged pull request #5901: URL: https://github.com/apache/incubator-doris/pull/5901 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [Bug] Bucket Shuffle Join may cause:Failed to send brpc batch, Not connected to 0.0.0.0:0 (#5901)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new ce3ae76 [Bug] Bucket Shuffle Join may cause:Failed to send brpc batch, Not connected to 0.0.0.0:0 (#5901) ce3ae76 is described below commit ce3ae764e5a2f61e4fcfd809ed8cfd1f59bf9de7 Author: HappenLee AuthorDate: Thu May 27 09:05:15 2021 -0500 [Bug] Bucket Shuffle Join may cause:Failed to send brpc batch, Not connected to 0.0.0.0:0 (#5901) --- .../src/main/java/org/apache/doris/planner/DataPartition.java | 4 fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 7 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java index ec47456..a8ee29a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java @@ -95,6 +95,10 @@ public class DataPartition { return type != TPartitionType.UNPARTITIONED; } +public boolean isBucketShuffleHashPartition() { +return type == TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED; +} + public TPartitionType getType() { return type; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index c891a03..1f8c38f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -828,7 +828,11 @@ public class Coordinator { params.instanceExecParams.size() + destParams.perExchNumSenders.get(exchId.asInt())); } -if (bucketShuffleJoinController.isBucketShuffleJoin(destFragment.getFragmentId().asInt())) { +if (sink.getOutputPartition().isBucketShuffleHashPartition()) { +// the destFragment must be bucket shuffle +Preconditions.checkState(bucketShuffleJoinController. + isBucketShuffleJoin(destFragment.getFragmentId().asInt())); + int bucketSeq = 0; int bucketNum = bucketShuffleJoinController.getFragmentBucketNum(destFragment.getFragmentId()); TNetworkAddress dummyServer = new TNetworkAddress("0.0.0.0", 0); @@ -1544,7 +1548,6 @@ public class Coordinator { return true; } -// One fragment could only have one HashJoinNode if (node instanceof HashJoinNode) { HashJoinNode joinNode = (HashJoinNode) node; if (joinNode.isBucketShuffle()) { - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #5900: [Bug] Bucket Shuffle Join cause query failed of 0.0.0.0:0
morningman closed issue #5900: URL: https://github.com/apache/incubator-doris/issues/5900 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated (ce3ae76 -> 4343354)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git. from ce3ae76 [Bug] Bucket Shuffle Join may cause:Failed to send brpc batch, Not connected to 0.0.0.0:0 (#5901) add 4343354 [BUG] Fix in memory table may cause a lot of CPU consumption when LRU Cache evict (#5908) No new revisions were added by this update. Summary of changes: be/src/olap/lru_cache.cpp | 58 --- be/src/olap/lru_cache.h | 7 -- 2 files changed, 40 insertions(+), 25 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #5907: [BUG] "In memory table" may cause a lot of CPU consumption when LRU Cache evict
morningman closed issue #5907: URL: https://github.com/apache/incubator-doris/issues/5907 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5908: [BUG] Fix "in memory table" may cause a lot of CPU consumption when LRU Cache evict
morningman merged pull request #5908: URL: https://github.com/apache/incubator-doris/pull/5908 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5910: [Bug-fix] Update correct data partition of fragment which contains Repeat Node
morningman merged pull request #5910: URL: https://github.com/apache/incubator-doris/pull/5910 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #5909: Error result of Grouping set when colocate is enable
morningman closed issue #5909: URL: https://github.com/apache/incubator-doris/issues/5909 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [Bug-fix] Update correct data partition of fragment which contains Repeat Node (#5910)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new aa17d40 [Bug-fix] Update correct data partition of fragment which contains Repeat Node (#5910) aa17d40 is described below commit aa17d408658124ee78f56fc7988ef93fd0a39c2f Author: EmmyMiao87 <522274...@qq.com> AuthorDate: Thu May 27 22:06:10 2021 +0800 [Bug-fix] Update correct data partition of fragment which contains Repeat Node (#5910) The Repeat Node will change the data partition of fragment when the origin data partition of fragment is HashPartition. The Repeat Node will generate some new rows. The distribution of these new rows is completely inconsistent with the original data distribution, their distribution is RANDOM. If the data distribution is not corrected, an error will occur when the agg node determines whether to perform colocate. Wrong data distribution will cause the agg node to think that agg can be colocated, leading to wrong results. For example, the following query can not be colocated although the distributed column of table is k1: ``` SELECT k1, k2, SUM( k3 ) FROM table GROUP BY GROUPING SETS ( (k1, k2), (k1), (k2), ( ) ) ``` --- .../apache/doris/planner/DistributedPlanner.java | 31 ++ .../org/apache/doris/planner/PlanFragment.java | 9 ++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java index 8f61a49..b0126bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java @@ -1043,6 +1043,37 @@ public class DistributedPlanner { throws UserException { repeatNode.setNumInstances(childFragment.getPlanRoot().getNumInstances()); childFragment.addPlanRoot(repeatNode); +/* +The Repeat Node will change the data partition of fragment + when the origin data partition of fragment is HashPartition. +For example, +Query: SELECT k1, k2, sum(v1) + FROM table + GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( )) +Table schema: table distributed by k1 +The Child Fragment: + Fragment 0 + Data partition: k1 + Repeat Node: repeat 3 lines [[0, 1], [0], [1], []] + OlapScanNode: table +Data before Repeat Node is partitioned by k1 such as: + | Node 1 | | Node 2 | + | 1, 1 | | 2, 1 | + | 1, 2 | | 2, 2 | +Data after Repeat Node is partitioned by RANDOM such as: + | Node 1 | | Node 2 | + | 1, 1 | | 2, 1 | + | 1, 2 | | 2, 2 | + | null,1 | | null,1 | + | null,2 | | null,2 | + ... +The Repeat Node will generate some new rows. +The distribution of these new rows is completely inconsistent with the original data distribution, + their distribution is RANDOM. +Therefore, the data distribution method of the fragment needs to be modified here. +Only the correct data distribution can make the correct result when judging **colocate**. + */ +childFragment.updateDataPartition(DataPartition.RANDOM); return childFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 234e7d3..f714132 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -96,7 +96,7 @@ public class PlanFragment extends TreeNode { // specification of the partition of the input of this fragment; // an UNPARTITIONED fragment is executed on only a single node // TODO: improve this comment, "input" is a bit misleading -private final DataPartition dataPartition; +private DataPartition dataPartition; // specification of the actually input partition of this fragment when transmitting to be. // By default, the value of the data partition in planner and the data partition transmitted to be are the same. @@ -267,6 +267,13 @@ public class PlanFragment extends TreeNode { return (dataPartition.getType() != TPartitionType.UNPARTITIONED); } +public void updateDataPartition(DataPartition dataPartition) { +if (this.dataPartition == DataPartition.UNPARTITIONED) { +return; +} +this.dataPartition = dataPartition; +} + public PlanFragmentId getId()
[incubator-doris] branch master updated: [BUG] BE core when FE get_stream_load_record (#5913)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new f4ebac0 [BUG] BE core when FE get_stream_load_record (#5913) f4ebac0 is described below commit f4ebac0210607f1ae41d882e726af59b96e4482d Author: stdpain <34912776+stdp...@users.noreply.github.com> AuthorDate: Thu May 27 22:06:26 2021 +0800 [BUG] BE core when FE get_stream_load_record (#5913) --- be/src/runtime/stream_load/stream_load_context.cpp | 14 +++--- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/be/src/runtime/stream_load/stream_load_context.cpp b/be/src/runtime/stream_load/stream_load_context.cpp index 21cf315..78bde90 100644 --- a/be/src/runtime/stream_load/stream_load_context.cpp +++ b/be/src/runtime/stream_load/stream_load_context.cpp @@ -199,31 +199,31 @@ void StreamLoadContext::parse_stream_load_record(const std::string& stream_load_ if (document.HasMember("NumberTotalRows")) { const rapidjson::Value& total_rows = document["NumberTotalRows"]; -stream_load_item.__set_total_rows(total_rows.GetInt()); -ss << ", NumberTotalRows: " << total_rows.GetInt(); +stream_load_item.__set_total_rows(total_rows.GetInt64()); +ss << ", NumberTotalRows: " << total_rows.GetInt64(); } if (document.HasMember("NumberLoadedRows")) { const rapidjson::Value& loaded_rows = document["NumberLoadedRows"]; -stream_load_item.__set_loaded_rows(loaded_rows.GetInt()); -ss << ", NumberLoadedRows: " << loaded_rows.GetInt(); +stream_load_item.__set_loaded_rows(loaded_rows.GetInt64()); +ss << ", NumberLoadedRows: " << loaded_rows.GetInt64(); } if (document.HasMember("NumberFilteredRows")) { const rapidjson::Value& filtered_rows = document["NumberFilteredRows"]; -stream_load_item.__set_filtered_rows(filtered_rows.GetInt()); +stream_load_item.__set_filtered_rows(filtered_rows.GetInt64()); ss << ", NumberFilteredRows: " << filtered_rows.GetInt64(); } if (document.HasMember("NumberUnselectedRows")) { const rapidjson::Value& unselected_rows = document["NumberUnselectedRows"]; -stream_load_item.__set_unselected_rows(unselected_rows.GetInt()); +stream_load_item.__set_unselected_rows(unselected_rows.GetInt64()); ss << ", NumberUnselectedRows: " << unselected_rows.GetInt64(); } if (document.HasMember("LoadBytes")) { const rapidjson::Value& load_bytes = document["LoadBytes"]; -stream_load_item.__set_load_bytes(load_bytes.GetInt()); +stream_load_item.__set_load_bytes(load_bytes.GetInt64()); ss << ", LoadBytes: " << load_bytes.GetInt64(); } - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5913: [BUG] BE core when FE get_stream_load_record
morningman merged pull request #5913: URL: https://github.com/apache/incubator-doris/pull/5913 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #5912: [BUG] BE may core when FE get_stream_load_record
morningman closed issue #5912: URL: https://github.com/apache/incubator-doris/issues/5912 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #5818: [BUG] The float value percentage of `storage_page_cache_limit` will cause the page_cache capacity calculation result to be wrong
morningman closed issue #5818: URL: https://github.com/apache/incubator-doris/issues/5818 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5916: [BUG] Fix parse error when the memory parameter is a float value percentage
morningman merged pull request #5916: URL: https://github.com/apache/incubator-doris/pull/5916 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [BUG] Fix calculation error when the memory parameter is a float value percentage (#5916)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new 80f0b5f [BUG] Fix calculation error when the memory parameter is a float value percentage (#5916) 80f0b5f is described below commit 80f0b5fd1ca63b7330b889ac8c501e7e0ab33564 Author: Xinyi Zou AuthorDate: Thu May 27 22:06:50 2021 +0800 [BUG] Fix calculation error when the memory parameter is a float value percentage (#5916) When parsing memory parameters in `ParseUtil::parse_mem_spec`, convert the percentage to `double` instead of `int`. The currently affected parameters include `mem_limit` and `storage_page_cache_limit` --- be/src/util/parse_util.cpp | 20 +--- be/test/util/parse_util_test.cpp | 6 ++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/be/src/util/parse_util.cpp b/be/src/util/parse_util.cpp index 39768d7..0a54dd9 100644 --- a/be/src/util/parse_util.cpp +++ b/be/src/util/parse_util.cpp @@ -69,8 +69,8 @@ int64_t ParseUtil::parse_mem_spec(const std::string& mem_spec_str, bool* is_perc StringParser::ParseResult result; int64_t bytes; -if (multiplier != -1) { -// Parse float - MB or GB +if (multiplier != -1 || *is_percent) { +// Parse float - MB or GB or percent double limit_val = StringParser::string_to_float(mem_spec_str.data(), number_str_len, &result); @@ -78,28 +78,26 @@ int64_t ParseUtil::parse_mem_spec(const std::string& mem_spec_str, bool* is_perc return -1; } -bytes = multiplier * limit_val; +if (multiplier != -1) { +bytes = multiplier * limit_val; +} else if (*is_percent) { +bytes = (static_cast(limit_val) / 100.0) * MemInfo::physical_mem(); +} } else { -// Parse int - bytes or percent +// Parse int - bytes int64_t limit_val = StringParser::string_to_int(mem_spec_str.data(), number_str_len, &result); if (result != StringParser::PARSE_SUCCESS) { return -1; } - -if (*is_percent) { -bytes = (static_cast(limit_val) / 100.0) * MemInfo::physical_mem(); -} else { -bytes = limit_val; -} +bytes = limit_val; } // Accept -1 as indicator for infinite memory that we report by a 0 return value. if (bytes == -1) { return 0; } - return bytes; } diff --git a/be/test/util/parse_util_test.cpp b/be/test/util/parse_util_test.cpp index f257fe6..27a97a0 100644 --- a/be/test/util/parse_util_test.cpp +++ b/be/test/util/parse_util_test.cpp @@ -55,6 +55,12 @@ TEST(TestParseMemSpec, Normal) { int64_t bytes = ParseUtil::parse_mem_spec("20%", &is_percent); ASSERT_GT(bytes, 0); ASSERT_TRUE(is_percent); + +MemInfo::_s_physical_mem = 1000; +is_percent = true; +bytes = ParseUtil::parse_mem_spec("0.1%", &is_percent); +ASSERT_EQ(bytes, 1); +ASSERT_TRUE(is_percent); } TEST(TestParseMemSpec, Bad) { - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5917: [BUG][Document] Fix the bug that failed to build the help module
morningman merged pull request #5917: URL: https://github.com/apache/incubator-doris/pull/5917 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [BUG][Document] Fix the bug that failed to build the help module (#5917)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new a29dd42 [BUG][Document] Fix the bug that failed to build the help module (#5917) a29dd42 is described below commit a29dd42b4758772069e6e03e84c905ab3a3ec81b Author: xy720 <22125576+xy...@users.noreply.github.com> AuthorDate: Thu May 27 22:07:15 2021 +0800 [BUG][Document] Fix the bug that failed to build the help module (#5917) There are multiple entries with same key in help documents, which will cause build help module failed. --- .../Data Manipulation/{SHOW DATABASE.md => SHOW DATABASE ID.md} | 2 +- .../Data Manipulation/{SHOW PARTITION.md => SHOW PARTITION ID.md} | 2 +- .../Data Manipulation/{SHOW TABLE.md => SHOW TABLE ID.md} | 2 +- .../Data Manipulation/{SHOW DATABASE.md => SHOW DATABASE ID.md} | 2 +- .../Data Manipulation/{SHOW PARTITION.md => SHOW PARTITION ID.md} | 2 +- .../Data Manipulation/{SHOW TABLE.md => SHOW TABLE ID.md} | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE ID.md similarity index 98% rename from docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE.md rename to docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE ID.md index cd4ed4d..87989ba 100644 --- a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE.md +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE ID.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# SHOW DATABASE +# SHOW DATABASE ID ## Description This statement is used to display database name according to database id (for administrators only) Grammar: diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION ID.md similarity index 98% rename from docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION.md rename to docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION ID.md index 6ccbac4..e443da4 100644 --- a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION.md +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION ID.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# SHOW PARTITION +# SHOW PARTITION ID ## Description This statement is used to display database name, table name, partition name according to partition id (for administrators only) Grammar: diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLE.md b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLE ID.md similarity index 98% rename from docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLE.md rename to docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLE ID.md index dc4306b..f526873 100644 --- a/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/SHOW TABLE ID.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# SHOW TABLE +# SHOW TABLE ID ## Description This statement is used to display database name, table name according to table id (for administrators only) Grammar: diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE ID.md similarity index 98% rename from docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE.md rename to docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE ID.md index aa8e43b..61ae6a1 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW DATABASE ID.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# SHOW DATABASE +# SHOW DATABASE ID ## description 该语句用于根据 database id 查找对应的 database name(仅管理员使用) 语法: diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION ID.md similarity index 98% rename from docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION.md rename to docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PARTITION ID.md index f41029b..0a618af 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/SHOW PAR
[GitHub] [incubator-doris] eyewire opened a new issue #5933: docker build spark-doris-connector fail
eyewire opened a new issue #5933: URL: https://github.com/apache/incubator-doris/issues/5933 **Describe the bug** docker build spark-doris-connector fail, error info is thrift command not found **To Reproduce** Steps to reproduce the behavior: 1. docker image tag apachedoris/doris-dev:build-env-1.2 2. download the doris source code and docker run -v 3. go to extensions/spark-doris-connector 4. run sh build.sh, about 3 mins will get this error message **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots**  **Desktop (please complete the following information):** - OS: macOS big sur - Version 11.2.3 **Additional context** docker image tag apachedoris/doris-dev:build-env-1.2 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [BUG] Fix the bug of `Desc Query` cause `Unknown error` and some doc revise (#5921)
This is an automated email from the ASF dual-hosted git repository. yangzhg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new c844e60 [BUG] Fix the bug of `Desc Query` cause `Unknown error` and some doc revise (#5921) c844e60 is described below commit c844e602a75372429ae8bf4246e8854df8d06669 Author: HappenLee AuthorDate: Thu May 27 22:21:31 2021 -0500 [BUG] Fix the bug of `Desc Query` cause `Unknown error` and some doc revise (#5921) --- docs/en/installing/compilation.md | 6 +++--- docs/zh-CN/administrator-guide/bucket-shuffle-join.md | 2 +- docs/zh-CN/installing/compilation.md | 6 +++--- fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java | 9 +++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/en/installing/compilation.md b/docs/en/installing/compilation.md index 927fb31..9fd8f2b 100644 --- a/docs/en/installing/compilation.md +++ b/docs/en/installing/compilation.md @@ -51,10 +51,10 @@ Note: For different versions of Oris, you need to download the corresponding mir |---|---|---| | apachedoris/doris-dev:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x | | apachedoris/doris-dev:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) or later | 0.10.x or later | -| apachedoris/doris-dev:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x or later | -| apachedoris/doris-dev:build-env-1.3 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) or later | 0.14.x or later | +| apachedoris/doris-dev:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) or later | 0.12.x - 0.14.0 | +| apachedoris/doris-dev:build-env-1.3 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) or later | later | -Warning: Doris 0.14.0 still used apachedoris/doris-dev:build-env-1.2 to compile. After thie version, the code will use apachedoris/doris-dev:build-env-1.3 to compile . **In the docker image of build-env-1.3, the default JDK version is upgraded to 11. So FE will use OPENJDK 11 to compile. If the docker image after build-env-1.3 is used for compilation of FE, the Java version of FE running env also needs to be upgraded to JDK11 or above. Otherwise unexpected running errors may be caused. ** +Warning: Doris 0.14.0 still used apachedoris/doris-dev:build-env-1.2 to compile. After thie version, the code will use apachedoris/doris-dev:build-env-1.3 to compile . **In the docker image of build-env-1.3, the default JDK version is upgraded to 11. So FE will use OPENJDK 11 to compile. If the docker image after build-env-1.3 is used for compilation of FE, the Java version of FE running env also needs to be upgraded to JDK11, Otherwise unexpected running errors may be caused. ** 2. Running Mirror diff --git a/docs/zh-CN/administrator-guide/bucket-shuffle-join.md b/docs/zh-CN/administrator-guide/bucket-shuffle-join.md index 38cd33a..6f629dd 100644 --- a/docs/zh-CN/administrator-guide/bucket-shuffle-join.md +++ b/docs/zh-CN/administrator-guide/bucket-shuffle-join.md @@ -70,7 +70,7 @@ Doris支持的常规分布式Join方式包括了shuffle join 和broadcast join set enable_bucket_shuffle_join = true; ``` -在FE进行分布式查询规划时,优先选择的顺序为 Colocate Join -> Bucket Shuffle Join -> Brocast Join -> Shuffle Join。但是如果用户显式hint了Join的类型,如: +在FE进行分布式查询规划时,优先选择的顺序为 Colocate Join -> Bucket Shuffle Join -> Broadcast Join -> Shuffle Join。但是如果用户显式hint了Join的类型,如: ``` select * from test join [shuffle] baseall on test.k1 = baseall.k1; diff --git a/docs/zh-CN/installing/compilation.md b/docs/zh-CN/installing/compilation.md index c54c3b2..16be7bc 100644 --- a/docs/zh-CN/installing/compilation.md +++ b/docs/zh-CN/installing/compilation.md @@ -51,10 +51,10 @@ under the License. |---|---|---| | apachedoris/doris-dev:build-env | before [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.8.x, 0.9.x | | apachedoris/doris-dev:build-env-1.1 | [ff0dd0d](https://github.com/apache/incubator-doris/commit/ff0dd0d2daa588f18b6db56f947e813a56d8ec81) | 0.10.x, 0.11.x | -| apachedoris/doris-dev:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-doris/commit/4ef5a8c8560351d7fff7ff8fd51c4c7a75e006a8) | 0.12.x, 0.13 | -| apachedoris/doris-dev:build-env-1.3 | [ad67dd3](https://github.com/apache/incubator-doris/commit/ad67dd34a04c1ca960cff38e5b335b30fc7d559f) | 0.14.x 或更新版本 | +| apachedoris/doris-dev:build-env-1.2 | [4ef5a8c](https://github.com/apache/incubator-dor
[GitHub] [incubator-doris] yangzhg closed issue #5920: [Bug] Desc query failed when `set is_report_success = true`
yangzhg closed issue #5920: URL: https://github.com/apache/incubator-doris/issues/5920 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangzhg merged pull request #5921: [BUG] Fix the bug of `Desc Query` cause `Unknown error` and some doc revise
yangzhg merged pull request #5921: URL: https://github.com/apache/incubator-doris/pull/5921 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman commented on issue #5930: 请问预计什么版本可以支持存储过程?或者是否由此计划?
morningman commented on issue #5930: URL: https://github.com/apache/incubator-doris/issues/5930#issuecomment-850150603 No plan for this now... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #5793: [Cache][Enhancement] Assure sql cache only one version
morningman merged pull request #5793: URL: https://github.com/apache/incubator-doris/pull/5793 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [Cache][Enhancement] Assure sql cache only one version (#5793)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git The following commit(s) were added to refs/heads/master by this push: new 63c99eb [Cache][Enhancement] Assure sql cache only one version (#5793) 63c99eb is described below commit 63c99eb4cbb9f1a42c0c41b5c6d65c36de8825dd Author: xinghuayu007 <1450306...@qq.com> AuthorDate: Fri May 28 13:45:47 2021 +0800 [Cache][Enhancement] Assure sql cache only one version (#5793) For PR #5792. This patch add a new param `cache type` to distinguish sql cache and partition cache. When update sql cache, we make assure one sql key only has one version cache. --- be/src/runtime/cache/result_node.cpp | 46 +- be/src/runtime/cache/result_node.h | 2 + be/test/runtime/cache/partition_cache_test.cpp | 55 -- .../org/apache/doris/qe/cache/RowBatchBuilder.java | 8 +++- gensrc/proto/internal_service.proto| 6 +++ 5 files changed, 99 insertions(+), 18 deletions(-) diff --git a/be/src/runtime/cache/result_node.cpp b/be/src/runtime/cache/result_node.cpp index f188c89..4a56349 100644 --- a/be/src/runtime/cache/result_node.cpp +++ b/be/src/runtime/cache/result_node.cpp @@ -77,7 +77,50 @@ PCacheStatus ResultNode::update_partition(const PUpdateCacheRequest* request, //Only one thread per SQL key can update the cache CacheWriteLock write_lock(_node_mtx); +if (request->cache_type() == CacheType::SQL_CACHE) { +return update_sql_cache(request, is_update_firstkey); +} else { +return update_partition_cache(request, is_update_firstkey); +} +} + +PCacheStatus ResultNode::update_sql_cache(const PUpdateCacheRequest *request, bool &is_update_firstkey) { +PartitionRowBatch* partition = NULL; +if (request->values_size() > 1) { +return PCacheStatus::PARAM_ERROR; +} +is_update_firstkey = true; +const PCacheValue& value = request->values(0); +PartitionKey partition_key = value.param().partition_key(); +// no cache exist, create new cache node +if (_partition_map.size() == 0) { +partition = new PartitionRowBatch(partition_key); +partition->set_row_batch(value); +_partition_map[partition_key] = partition; +_partition_list.push_back(partition); +} else { +// compatible with previous version +for (auto it = _partition_list.begin(); it != _partition_list.end(); it++) { +_data_size -= (*it)->get_data_size(); +} +// clear old cache, and create new cache node +for (auto it = _partition_list.begin(); it != _partition_list.end();) { +(*it)->clear(); +SAFE_DELETE(*it); +it = _partition_list.erase(it); +} +_partition_map.clear(); +partition = new PartitionRowBatch(partition_key); +partition->set_row_batch(value); +_partition_map[partition_key] = partition; +_partition_list.push_back(partition); +} +_data_size += partition->get_data_size(); +VLOG(1) << "finish update sql cache batches:" << _partition_list.size(); +return PCacheStatus::CACHE_OK; +} +PCacheStatus ResultNode::update_partition_cache(const PUpdateCacheRequest *request, bool &is_update_firstkey) { PartitionKey first_key = kint64max; if (_partition_list.size() == 0) { is_update_firstkey = true; @@ -115,7 +158,7 @@ PCacheStatus ResultNode::update_partition(const PUpdateCacheRequest* request, _data_size += partition->get_data_size(); } _partition_list.sort(compare_partition); -LOG(INFO) << "finish update batches:" << _partition_list.size(); +VLOG(1) << "finish update partition cache batches:" << _partition_list.size(); while (config::query_cache_max_partition_count > 0 && _partition_list.size() > config::query_cache_max_partition_count) { if (prune_first() == 0) { @@ -247,6 +290,7 @@ size_t ResultNode::prune_first() { PartitionRowBatch* part_node = *_partition_list.begin(); size_t prune_size = part_node->get_data_size(); _partition_list.erase(_partition_list.begin()); +_partition_map.erase(part_node->get_partition_key()); part_node->clear(); SAFE_DELETE(part_node); _data_size -= prune_size; diff --git a/be/src/runtime/cache/result_node.h b/be/src/runtime/cache/result_node.h index 2d5ca0d..d2c5d1d 100644 --- a/be/src/runtime/cache/result_node.h +++ b/be/src/runtime/cache/result_node.h @@ -128,6 +128,8 @@ public: PCacheStatus update_partition(const PUpdateCacheRequest* request, bool& is_update_firstkey); PCacheStatus fetch_partition(const PFetchCacheRequest* request, PartitionRowBatchList& rowBatchList, bool& is_hit_firstkey); +PCacheStatus update_sql_cache(const PUpdateCacheRequest* requ
[GitHub] [incubator-doris] morningman commented on a change in pull request #5831: [BE] Optimize version retrieval efficiency.
morningman commented on a change in pull request #5831: URL: https://github.com/apache/incubator-doris/pull/5831#discussion_r641310036 ## File path: be/src/olap/version_graph.cpp ## @@ -537,95 +560,58 @@ OLAPStatus VersionGraph::capture_consistent_versions(const Version& spec_version return OLAP_ERR_INPUT_PARAMETER_ERROR; } -// bfs_queue's element is vertex_index. -std::queue bfs_queue; -// predecessor[i] means the predecessor of vertex_index 'i'. -std::vector predecessor(_version_graph.size()); -// visited[int64_t]==true means it had entered bfs_queue. -std::vector visited(_version_graph.size()); -// [start_vertex_value, end_vertex_value) -int64_t start_vertex_value = spec_version.first; -int64_t end_vertex_value = spec_version.second + 1; -// -1 is invalid vertex index. -int64_t start_vertex_index = -1; -// -1 is valid vertex index. -int64_t end_vertex_index = -1; - -for (size_t i = 0; i < _version_graph.size(); ++i) { -if (_version_graph[i].value == start_vertex_value) { -start_vertex_index = i; -} -if (_version_graph[i].value == end_vertex_value) { -end_vertex_index = i; +int64_t cur_idx = -1; +for (size_t i = 0; i < _version_graph.size(); i++) { +if (_version_graph[i].value == spec_version.first) { +cur_idx = i; +break; } } -if (start_vertex_index < 0 || end_vertex_index < 0) { -LOG(WARNING) << "fail to find path in version_graph. " +if (cur_idx < 0) { +LOG(WARNING) << "failed to find path in version_graph. " << "spec_version: " << spec_version.first << "-" << spec_version.second; return OLAP_ERR_VERSION_NOT_EXIST; } -for (size_t i = 0; i < _version_graph.size(); ++i) { -visited[i] = false; -} - -bfs_queue.push(start_vertex_index); -visited[start_vertex_index] = true; -// The predecessor of root is itself. -predecessor[start_vertex_index] = start_vertex_index; - -while (bfs_queue.empty() == false && visited[end_vertex_index] == false) { -int64_t top_vertex_index = bfs_queue.front(); -bfs_queue.pop(); -for (const auto& it : _version_graph[top_vertex_index].edges) { -if (visited[it] == false) { -// If we don't support reverse version in the path, and start vertex -// value is larger than the end vertex value, we skip this edge. -if (_version_graph[top_vertex_index].value > _version_graph[it].value) { -continue; -} -visited[it] = true; -predecessor[it] = top_vertex_index; -bfs_queue.push(it); +int64_t end_value = spec_version.second + 1; +while (_version_graph[cur_idx].value < end_value) { +int64_t next_idx = -1; +for (const auto& it : _version_graph[cur_idx].edges) { +// Only consider incremental versions +if (_version_graph[it].value < _version_graph[cur_idx].value) { Review comment: If we already sort edges in descending order, can we "break" here? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org