[GitHub] [incubator-doris] morningman closed issue #8512: [Bug] Don't rewrite nested union statement
morningman closed issue #8512: [Bug] Don't rewrite nested union statement URL: https://github.com/apache/incubator-doris/issues/8512 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #8513: [Bug] fix don't rewrite nested union statement bug
morningman merged PR #8513: URL: https://github.com/apache/incubator-doris/pull/8513 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [fix](planner) fix don't rewrite nested union statement bug (#8513)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a7cca930b9 [fix](planner) fix don't rewrite nested union statement bug
(#8513)
a7cca930b9 is described below
commit a7cca930b964d96b778381967b257c1aa7222519
Author: shee <13843187+qz...@users.noreply.github.com>
AuthorDate: Fri Jun 10 19:43:45 2022 +0800
[fix](planner) fix don't rewrite nested union statement bug (#8513)
Issue Number: close #8512
---
.../java/org/apache/doris/analysis/StmtRewriter.java | 14 ++
.../java/org/apache/doris/planner/PlannerTest.java | 20 +++-
2 files changed, 29 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
index 42a0188a84..4f0ea9c059 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
@@ -321,10 +321,16 @@ public class StmtRewriter {
private static void rewriteUnionStatement(SetOperationStmt stmt, Analyzer
analyzer)
throws AnalysisException {
for (SetOperationStmt.SetOperand operand : stmt.getOperands()) {
-Preconditions.checkState(operand.getQueryStmt() instanceof
SelectStmt);
-QueryStmt rewrittenQueryStmt = StmtRewriter.rewriteSelectStatement(
-(SelectStmt) operand.getQueryStmt(),
operand.getAnalyzer());
-operand.setQueryStmt(rewrittenQueryStmt);
+QueryStmt queryStmt = operand.getQueryStmt();
+if (queryStmt instanceof SelectStmt) {
+QueryStmt rewrittenQueryStmt =
rewriteSelectStatement((SelectStmt) queryStmt, operand.getAnalyzer());
+operand.setQueryStmt(rewrittenQueryStmt);
+} else if (queryStmt instanceof SetOperationStmt) {
+rewriteUnionStatement((SetOperationStmt) queryStmt,
operand.getAnalyzer());
+} else {
+throw new IllegalStateException("Rewrite union statement
failed. "
++ "Because QueryStmt is neither SelectStmt nor
SetOperationStmt");
+}
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
index 1a4832b420..73fe9bf24b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
@@ -70,7 +70,13 @@ public class PlannerTest extends TestWithFeService {
+ " distributed by hash(k1)"
+ " properties('replication_num' = '1');";
-createTables(tbl1, tbl2, tbl3, tbl4);
+String tbl5 = "create table db1.tbl5("
++ "k1 int,"
++ "k2 int) "
++ "DISTRIBUTED BY HASH(k2) "
++ "BUCKETS 3 PROPERTIES ('replication_num' = '1');";
+
+createTables(tbl1, tbl2, tbl3, tbl4, tbl5);
}
@Test
@@ -494,4 +500,16 @@ public class PlannerTest extends TestWithFeService {
String plan = planner.getExplainString(fragments, new
ExplainOptions(false, false));
Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1\n"));
}
+
+@Test
+public void testRewriteNestedUnionStmt() throws Exception {
+String qSQL = "SELECT k1 FROM db1.tbl5 WHERE k1 IN "
++ "( SELECT k1 FROM ( SELECT k1 FROM db1.tbl5 ORDER BY k2
DESC, k1 DESC LIMIT 300 INTERSECT "
++ "(SELECT k1 FROM db1.tbl5 ORDER BY k2 DESC, k1 DESC LIMIT 9
EXCEPT SELECT k1 "
++ "FROM db1.tbl5 ORDER BY k2 DESC, k1 DESC LIMIT 2) ) t )";
+
+StmtExecutor stmtExecutor = new StmtExecutor(connectContext, qSQL);
+stmtExecutor.execute();
+}
+
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #9887: [Enhancement] add some metrics for cpu and memory
morningman merged PR #9887: URL: https://github.com/apache/incubator-doris/pull/9887 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated (a7cca930b9 -> 990a2940ca)
This is an automated email from the ASF dual-hosted git repository. morningman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git from a7cca930b9 [fix](planner) fix don't rewrite nested union statement bug (#8513) add 990a2940ca [metric] add some metrics for cpu and memory (#9887) No new revisions were added by this update. Summary of changes: be/src/util/system_metrics.cpp | 226 +++ be/src/util/system_metrics.h | 13 +- be/test/util/system_metrics_test.cpp | 93 +++--- be/test/util/test_data/vmstat_normal | 12 ++ 4 files changed, 308 insertions(+), 36 deletions(-) create mode 100644 be/test/util/test_data/vmstat_normal - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
eldenmoon commented on PR #10065: URL: https://github.com/apache/incubator-doris/pull/10065#issuecomment-1152285619 Sorry for my issue in this PR I'm currently using Array, found that when I build BE with ASAN, using the column type like `ARRAY`, after I insert with vectorized engine enabled, there will be a coredump which is heap-use-after-free, see issue #9934 for more details. I commented the clue I found in this PR https://github.com/apache/incubator-doris/pull/9466 (the last comment) , but I'm not quite sure how to solve it -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
adonis0147 commented on PR #10065: URL: https://github.com/apache/incubator-doris/pull/10065#issuecomment-1152291339 Hi @eldenmoon , thanks for your feedback. I will investigate the issue you mentioned and try to solve it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangzhg commented on pull request #9964: add hyperscan and its dependency ragel to thirdparty
yangzhg commented on PR #9964: URL: https://github.com/apache/incubator-doris/pull/9964#issuecomment-1152319451 hyperscan is not suport arm platform -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangzhg opened a new pull request, #10068: [enhancement] support simd instructions on arm cpus through sse2neon
yangzhg opened a new pull request, #10068: URL: https://github.com/apache/incubator-doris/pull/10068 # Proposed changes support simd instructions on arm cpus through sse2neon ## Problem Summary: sse2neon can covert sse instructions to arm neon instructions this pr not need to update docker image ## Checklist(Required) 1. Does it affect the original behavior: (No) 2. Has unit tests been added: (No Need) 3. Has document been added or modified: (No Need) 4. Does it need to update dependencies: (Yes) 5. Are there any changes that cannot be rolled back: (No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yinzhijian opened a new pull request, #10069: [test] add cast array regression test
yinzhijian opened a new pull request, #10069: URL: https://github.com/apache/incubator-doris/pull/10069 # Proposed changes Issue Number: close #xxx ## Problem Summary: add P0 test for #9995 1. cast scalar type(except varchar) to array is not allowed 2. cast array to scalar type is not allowed ## Checklist(Required) 1. Does it affect the original behavior: (No) 3. Has unit tests been added: (Yes) 4. Has document been added or modified: (No Need) 5. Does it need to update dependencies: (No) 6. Are there any changes that cannot be rolled back: (No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] Gabriel39 commented on a diff in pull request #10060: [Bug] Fix bug push value predicate of unique table when have sequence column
Gabriel39 commented on code in PR #10060:
URL: https://github.com/apache/incubator-doris/pull/10060#discussion_r894527198
##
be/src/olap/rowset/beta_rowset_reader.cpp:
##
@@ -237,4 +235,12 @@ Status BetaRowsetReader::next_block(vectorized::Block*
block) {
return Status::OK();
}
+bool BetaRowsetReader::_should_push_down_value_predicates() const {
Review Comment:
should we check if there is a sequence column in tablet schema?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10062: Do not send drop task when replay drop table
github-actions[bot] commented on PR #10062: URL: https://github.com/apache/incubator-doris/pull/10062#issuecomment-1152372323 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10062: Do not send drop task when replay drop table
github-actions[bot] commented on PR #10062: URL: https://github.com/apache/incubator-doris/pull/10062#issuecomment-1152372351 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on pull request #10069: [test] add cast array regression test
cambyzju commented on PR #10069: URL: https://github.com/apache/incubator-doris/pull/10069#issuecomment-1152385119 LGTM -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju opened a new pull request, #10070: [enhancement] Change array offset type from UInt32 to Int64
cambyzju opened a new pull request, #10070: URL: https://github.com/apache/incubator-doris/pull/10070 # Proposed changes Issue Number: close #7570 ## Problem Summary: Now column `Array` contains column `offsets` and `data`, and type of column `offsets` is UInt32 now. In compute layer, if we call array_union to merge arrays, the size may overflow. ## Checklist(Required) 1. Does it affect the original behavior: (Yes/No/I Don't know) 2. Has unit tests been added: (Yes/No/No Need) 3. Has document been added or modified: (Yes/No/No Need) 4. Does it need to update dependencies: (Yes/No) 5. Are there any changes that cannot be rolled back: (Yes/No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on issue #9934: [Bug] heap-use-after-free when using type array
adonis0147 commented on issue #9934: URL: https://github.com/apache/incubator-doris/issues/9934#issuecomment-1152409954 I will investigate this issue. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] xinyiZzz opened a new issue, #10071: [Bug] memtracker _child_trackers_lock deadlock
xinyiZzz opened a new issue, #10071: URL: https://github.com/apache/incubator-doris/issues/10071 ### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and found no similar issues. ### Version trunk ### What's Wrong?  ### What You Expected? run ### How to Reproduce? _No response_ ### Anything Else? _No response_ ### Are you willing to submit PR? - [X] Yes I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] xinyiZzz opened a new pull request, #10072: [fix] (mem tracker) Fix some memory leaks, inaccurate statistics, core dump, deadlock bugs
xinyiZzz opened a new pull request, #10072: URL: https://github.com/apache/incubator-doris/pull/10072 # Proposed changes Issue Number: close #10006 #10071 ## Problem Summary: 1. When the load task is canceled, the `IndexChannel` and `NodeChannel` mem trackers cannot be released in time, causing memory leaks. 2. Fix the inaccuracy of `LoadChannel` mem tracker limit, and rewrite the variable name of `mem limit` in `LoadChannel`. 3. When repairing logout task mem tracker, phmap erase fails and causes core dump. 4. When repairing add_child_tracker, the mem limit exceeds, it will cause deadlock when log_usage is called. 5. Avoid frequent Log printing when thread mem tracker limit exceeds, which will affect readability and performance. 6. Optimize some details of mem tracker display. ## Checklist(Required) 1. Does it affect the original behavior: (No】) 2. Has unit tests been added: (No Need) 3. Has document been added or modified: (No Need) 4. Does it need to update dependencies: (No) 5. Are there any changes that cannot be rolled back: (No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju closed pull request #10070: [enhancement] Change array offset type from UInt32 to Int64
cambyzju closed pull request #10070: [enhancement] Change array offset type from UInt32 to Int64 URL: https://github.com/apache/incubator-doris/pull/10070 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
adonis0147 commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894648585
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
Review Comment:
Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
adonis0147 commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894652397
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
+auto& column_offsets =
+
static_cast(*column_offsets_ptr);
+auto& offsets_data = column_offsets.get_data();
+for (int i = start; i < offsets_data.size(); ++i) {
Review Comment:
It is not the case here, because the size of `offsets_data` is limited to
the number of entries read from file and the number is limited to the batch
size.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
adonis0147 commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894654169
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
+auto& column_offsets =
+
static_cast(*column_offsets_ptr);
+auto& offsets_data = column_offsets.get_data();
+for (int i = start; i < offsets_data.size(); ++i) {
+offsets_data[i] += offsets_data[i - 1]; // -1 is ok
+}
+
+auto column_items_ptr = column_array->get_data().assume_mutable();
+for (int64_t i = start; i < offsets_data.size(); ++i) {
+auto offset = offsets_data[i - 1]; // -1 is ok.
+auto size = offsets_data[i] - offset;
+if (size == 0) {
+continue;
+}
+size_t num_items = size;
+bool items_has_null = false;
+_item_iterator->next_batch(&num_items, column_items_ptr,
&items_has_null);
Review Comment:
Refined.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
adonis0147 commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894656962
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
+auto& column_offsets =
+
static_cast(*column_offsets_ptr);
+auto& offsets_data = column_offsets.get_data();
+for (int i = start; i < offsets_data.size(); ++i) {
+offsets_data[i] += offsets_data[i - 1]; // -1 is ok
+}
Review Comment:
The value of `*n` was updated at
[L413](https://github.com/apache/incubator-doris/blob/c4bdfb2d1194f16b923837785719a0edb2eaf78c/be/src/olap/rowset/segment_v2/column_reader.cpp#L413)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] 03/03: [Bug] [Vectorized] code dump on aggregate node over union node (#10040)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 519082831984fbf6706146211f2d9c0ce8f2475c
Author: morningman
AuthorDate: Fri Jun 10 23:48:07 2022 +0800
[Bug] [Vectorized] code dump on aggregate node over union node (#10040)
* miss check passthrough on vectorized
* format and add test
* update
---
.../src/main/java/org/apache/doris/planner/SetOperationNode.java | 7 +++
1 file changed, 7 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
index 93082e3e30..238f772726 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
@@ -270,9 +270,16 @@ public abstract class SetOperationNode extends PlanNode {
return false;
}
if (VectorizedUtil.isVectorized()) {
+// On vectorized engine, we have more chance to do passthrough.
if (childSlotRef.getDesc().getSlotOffset() !=
setOpSlotRef.getDesc().getSlotOffset()) {
return false;
}
+if (childSlotRef.isNullable() != setOpSlotRef.isNullable()) {
+return false;
+}
+if (childSlotRef.getDesc().getType() !=
setOpSlotRef.getDesc().getType()) {
+return false;
+}
} else {
if
(!childSlotRef.getDesc().LayoutEquals(setOpSlotRef.getDesc())) {
return false;
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] 02/03: [fix](load) fix streamload failure due to false unhealthy replica in concurrent stream load (#10007)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 4bc3a364f8f8c4396a642fd2471e3204b8fb3d80
Author: yixiutt <102007456+yixi...@users.noreply.github.com>
AuthorDate: Fri Jun 10 09:15:14 2022 +0800
[fix](load) fix streamload failure due to false unhealthy replica in
concurrent stream load (#10007)
in concurrent stream load, fe will run publish version task concurrently,
which cause publish task disorder in be.
For example:
fe publish task with version 1 2 3 4
be may handle task with sequence 1 2 4 3
In case above, when report tablet info, be found that version 4
published but version 3 not visible, it'll report version miss to fe,
and fe will set replica lastFailedVersion, and finally makes transaction
commits fail while no quorum health replicas。
Add a time condition if a version miss for 60 seconds then report version
miss.
---
be/src/olap/tablet.cpp | 31 ---
be/src/olap/tablet.h | 6 +-
be/src/olap/tablet_manager.cpp | 2 +-
3 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 8214c44494..6f84b1764f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -76,7 +76,9 @@ Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir*
data_dir,
_cumulative_compaction_type(cumulative_compaction_type),
_last_record_scan_count(0),
_last_record_scan_count_timestamp(time(nullptr)),
- _is_clone_occurred(false) {
+ _is_clone_occurred(false),
+ _last_missed_version(-1),
+ _last_missed_time_s(0) {
// construct _timestamped_versioned_tracker from rs and stale rs meta
_timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas(),
_tablet_meta->all_stale_rs_metas());
@@ -1285,7 +1287,9 @@ bool Tablet::_contains_rowset(const RowsetId rowset_id) {
return false;
}
-void Tablet::build_tablet_report_info(TTabletInfo* tablet_info) {
+// need check if consecutive version missing in full report
+// alter tablet will ignore this check
+void Tablet::build_tablet_report_info(TTabletInfo* tablet_info, bool
enable_consecutive_missing_check) {
ReadLock rdlock(_meta_lock);
tablet_info->tablet_id = _tablet_meta->tablet_id();
tablet_info->schema_hash = _tablet_meta->schema_hash();
@@ -1298,7 +1302,28 @@ void Tablet::build_tablet_report_info(TTabletInfo*
tablet_info) {
Version max_version;
bool has_version_cross;
_max_continuous_version_from_beginning_unlocked(&cversion, &max_version,
&has_version_cross);
-tablet_info->__set_version_miss(cversion.second < max_version.second);
+// cause publish version task runs concurrently, version may be flying
+// so we add a consecutive miss check to solve this problem:
+// if publish version 5 arrives but version 4 flying, we may judge replica
miss version
+// and set version miss in tablet_info, which makes fe treat this replica
as unhealth
+// and lead to other problems
+if (enable_consecutive_missing_check) {
+if (cversion.second < max_version.second) {
+if (_last_missed_version == cversion.second + 1) {
+if (_last_missed_time_s - MonotonicSeconds() >= 60) {
+// version missed for over 60 seconds
+tablet_info->__set_version_miss(true);
+_last_missed_version = -1;
+_last_missed_time_s = 0;
+}
+} else {
+_last_missed_version = cversion.second + 1;
+_last_missed_time_s = MonotonicSeconds();
+}
+}
+} else {
+tablet_info->__set_version_miss(cversion.second < max_version.second);
+}
// find rowset with max version
auto iter = _rs_version_map.find(max_version);
if (iter == _rs_version_map.end()) {
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 5d0ad5ad05..114ca26e72 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -234,7 +234,8 @@ public:
// Rowset whose version range is not covered by this tablet is also useful.
bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta);
-void build_tablet_report_info(TTabletInfo* tablet_info);
+void build_tablet_report_info(TTabletInfo* tablet_info,
+ bool enable_consecutive_missing_check =
false);
void generate_tablet_meta_copy(TabletMetaSharedPtr new_tablet_meta) const;
// caller should hold the _meta_lock before calling this method
@@ -351,6 +352,9 @@ private:
// whether clone task occurred during the tablet is in thread pool queue
to wait for compaction
std::atomic _is_clone_occurre
[incubator-doris] branch dev-1.0.1 updated (83d44f8df8 -> 5190828319)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
from 83d44f8df8 fix: fix a thread safe problem in LoadAction.java (#9955)
new b8323e9f70 [feature] datetime column type support auto-initialized
with default … (#9972)
new 4bc3a364f8 [fix](load) fix streamload failure due to false unhealthy
replica in concurrent stream load (#10007)
new 5190828319 [Bug] [Vectorized] code dump on aggregate node over union
node (#10040)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/olap/tablet.cpp | 31 ++--
be/src/olap/tablet.h | 6 ++-
be/src/olap/tablet_manager.cpp | 2 +-
fe/fe-core/src/main/cup/sql_parser.cup | 8 ++-
.../java/org/apache/doris/analysis/ColumnDef.java | 37 --
.../org/apache/doris/analysis/DataDescription.java | 4 +-
.../apache/doris/analysis/DefaultValueExprDef.java | 58 ++
.../apache/doris/analysis/FunctionCallExpr.java| 17 +++
.../java/org/apache/doris/analysis/InsertStmt.java | 12 +++--
.../main/java/org/apache/doris/catalog/Column.java | 19 +--
.../src/main/java/org/apache/doris/load/Load.java | 12 -
.../org/apache/doris/planner/LoadScanNode.java | 6 ++-
.../org/apache/doris/planner/SetOperationNode.java | 7 +++
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
...ap_serialize.out => test_current_timestamp.out} | 5 +-
.../test_current_timestamp_streamload.csv | 4 ++
.../correctness/test_current_timestamp.groovy | 55
17 files changed, 261 insertions(+), 23 deletions(-)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
copy regression-test/data/correctness/{test_bitmap_serialize.out =>
test_current_timestamp.out} (69%)
create mode 100644
regression-test/data/correctness/test_current_timestamp_streamload.csv
create mode 100644
regression-test/suites/correctness/test_current_timestamp.groovy
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] 01/03: [feature] datetime column type support auto-initialized with default … (#9972)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit b8323e9f70bd233f8c161e45aa028e7dba74a314
Author: BePPPower <43782773+bepppo...@users.noreply.github.com>
AuthorDate: Thu Jun 9 00:28:03 2022 +0800
[feature] datetime column type support auto-initialized with default …
(#9972)
---
fe/fe-core/src/main/cup/sql_parser.cup | 8 ++-
.../java/org/apache/doris/analysis/ColumnDef.java | 37 --
.../org/apache/doris/analysis/DataDescription.java | 4 +-
.../apache/doris/analysis/DefaultValueExprDef.java | 58 ++
.../apache/doris/analysis/FunctionCallExpr.java| 17 +++
.../java/org/apache/doris/analysis/InsertStmt.java | 12 +++--
.../main/java/org/apache/doris/catalog/Column.java | 19 +--
.../src/main/java/org/apache/doris/load/Load.java | 12 -
.../org/apache/doris/planner/LoadScanNode.java | 6 ++-
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../data/correctness/test_current_timestamp.out| 7 +++
.../test_current_timestamp_streamload.csv | 4 ++
.../correctness/test_current_timestamp.groovy | 55
13 files changed, 223 insertions(+), 17 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index ce45f1cf56..ce3600ecf4 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -237,7 +237,7 @@ parser code {:
// Total keywords of doris
terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL,
KW_ALTER, KW_AND, KW_ANTI, KW_APPEND, KW_AS, KW_ASC, KW_AUTHORS, KW_ARRAY,
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG,
KW_BITMAP, KW_BITMAP_UNION, KW_BLOB, KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY,
KW_BUILTIN,
-KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK,
KW_CLUSTER, KW_CLUSTERS, KW_CLEAN,
+KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK,
KW_CLUSTER, KW_CLUSTERS, KW_CLEAN, KW_CURRENT_TIMESTAMP,
KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT,
KW_COMMITTED, KW_COMPACT,
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT,
KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT,
KW_CURRENT_USER,
KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DAY,
KW_DECIMAL, KW_DECOMMISSION, KW_DEFAULT, KW_DESC, KW_DESCRIBE,
@@ -2414,6 +2414,10 @@ opt_default_value ::=
{:
RESULT = ColumnDef.DefaultValue.NULL_DEFAULT_VALUE;
:}
+| KW_DEFAULT KW_CURRENT_TIMESTAMP
+{:
+RESULT = ColumnDef.DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE;
+:}
;
opt_is_key ::=
@@ -5730,6 +5734,8 @@ keyword ::=
{: RESULT = id; :}
| KW_VARCHAR:id
{: RESULT = id; :}
+| KW_CURRENT_TIMESTAMP:id
+{: RESULT = id; :}
;
// Identifier that contain keyword
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index cf61e09218..db794c730b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -58,12 +58,31 @@ public class ColumnDef {
public static class DefaultValue {
public boolean isSet;
public String value;
+// used for column which defaultValue is an expression.
+public DefaultValueExprDef defaultValueExprDef;
public DefaultValue(boolean isSet, String value) {
this.isSet = isSet;
this.value = value;
+this.defaultValueExprDef = null;
}
+/**
+ * used for column which defaultValue is an expression.
+ * @param isSet is Set DefaultValue
+ * @param value default value
+ * @param exprName default value expression
+ */
+public DefaultValue(boolean isSet, String value, String exprName) {
+this.isSet = isSet;
+this.value = value;
+this.defaultValueExprDef = new DefaultValueExprDef(exprName);
+}
+
+// default "CURRENT_TIMESTAMP", only for DATETIME type
+public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
+public static String NOW = "now";
+public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new
DefaultValue(true, CURRENT_TIMESTAMP, NOW);
// no default value
public static DefaultValue NOT_SET = new DefaultValue(false, null);
// default null
@@ -245,11 +264,13 @@ public class ColumnDef {
}
if (defaultValue.isSet && defaultValue.value != null) {
-validateDefaultValue(type, defaultValue.value);
+validateDefaultValue(type, defaultValue.value,
defaultValue.defaultValue
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #9803: [fix](brpc) Embed serialized request into the attachment and transmit it through http brpc
github-actions[bot] commented on PR #9803: URL: https://github.com/apache/incubator-doris/pull/9803#issuecomment-1152514412 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #9803: [fix](brpc) Embed serialized request into the attachment and transmit it through http brpc
github-actions[bot] commented on PR #9803: URL: https://github.com/apache/incubator-doris/pull/9803#issuecomment-1152514470 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman opened a new pull request, #10073: [fix] fix bug that show proc "/cluster_balance/history_tablets" return malformat error
morningman opened a new pull request, #10073: URL: https://github.com/apache/incubator-doris/pull/10073 # Proposed changes Issue Number: close #xxx ## Problem Summary: 1. Fix bug that show proc "/cluster_balance/history_tablets" return malformat error 2. Modify the explain str of some exec node to simplify the explain result. ## Checklist(Required) 1. Does it affect the original behavior: (No) 3. Has unit tests been added: (No Need) 4. Has document been added or modified: (No Need) 5. Does it need to update dependencies: (No) 6. Are there any changes that cannot be rolled back: (No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman opened a new issue, #10074: [Bug] show proc "/cluster_balance/history_tablets" return Malformed packet
morningman opened a new issue, #10074: URL: https://github.com/apache/incubator-doris/issues/10074 ### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and found no similar issues. ### Version dev-1.0.1 ### What's Wrong? ``` mysql> show proc "/cluster_balance/history_tablets"; ERROR 2027 (HY000): Malformed packet ``` ### What You Expected? no error ### How to Reproduce? _No response_ ### Anything Else? _No response_ ### Are you willing to submit PR? - [X] Yes I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch dependabot/maven/fe/org.mortbay.jetty-jetty-6.1.23 created (now ad9350867a)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/fe/org.mortbay.jetty-jetty-6.1.23 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git at ad9350867a Bump jetty from 6.1.14 to 6.1.23 in /fe This branch includes the following new commits: new ad9350867a Bump jetty from 6.1.14 to 6.1.23 in /fe The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] 01/01: Bump jetty from 6.1.14 to 6.1.23 in /fe
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch dependabot/maven/fe/org.mortbay.jetty-jetty-6.1.23 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git commit ad9350867a79defdec2b1bf53bd9065cf4e787d6 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Jun 10 22:46:08 2022 + Bump jetty from 6.1.14 to 6.1.23 in /fe Bumps jetty from 6.1.14 to 6.1.23. --- updated-dependencies: - dependency-name: org.mortbay.jetty:jetty dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- fe/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/pom.xml b/fe/pom.xml index 43bab8adab..72f4a21df9 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -206,7 +206,7 @@ under the License. 3.18.2-GA 3.0.1 18.3.12 -6.1.14 +6.1.23 1.4.3 1.49 2.10.1 - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch dependabot/maven/fe/org.mortbay.jetty-jetty-6.1.23 updated (ad9350867a -> 3b9997a856)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/fe/org.mortbay.jetty-jetty-6.1.23 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git discard ad9350867a Bump jetty from 6.1.14 to 6.1.23 in /fe add 3b9997a856 Bump jetty from 6.1.14 to 6.1.23 in /fe This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (ad9350867a) \ N -- N -- N refs/heads/dependabot/maven/fe/org.mortbay.jetty-jetty-6.1.23 (3b9997a856) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] dependabot[bot] opened a new pull request, #10075: Bump jetty from 6.1.14 to 6.1.23 in /fe
dependabot[bot] opened a new pull request, #10075: URL: https://github.com/apache/incubator-doris/pull/10075 Bumps jetty from 6.1.14 to 6.1.23. [](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) --- Dependabot commands and options You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) - `@dependabot use these labels` will set the current labels as the default for future PRs for this repo and language - `@dependabot use these reviewers` will set the current reviewers as the default for future PRs for this repo and language - `@dependabot use these assignees` will set the current assignees as the default for future PRs for this repo and language - `@dependabot use this milestone` will set the current milestone as the default for future PRs for this repo and language You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/incubator-doris/network/alerts). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangzhg commented on pull request #9803: [fix](brpc) Embed serialized request into the attachment and transmit it through http brpc
yangzhg commented on PR #9803: URL: https://github.com/apache/incubator-doris/pull/9803#issuecomment-1152809258 `PEchoRequest` has not been renamed, should not merge it before resolve it -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [Bug] [Vectorized] code dump on aggregate node over union node (#10040)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 495c34fa29 [Bug] [Vectorized] code dump on aggregate node over union
node (#10040)
495c34fa29 is described below
commit 495c34fa293e688cd02211efe12f5fdf5215f08f
Author: Pxl
AuthorDate: Fri Jun 10 15:02:14 2022 +0800
[Bug] [Vectorized] code dump on aggregate node over union node (#10040)
* miss check passthrough on vectorized
* format and add test
* update
---
.../src/main/java/org/apache/doris/planner/SetOperationNode.java | 7 +++
regression-test/data/query/union/test_union.out | 9 +
regression-test/suites/query/union/test_union.groovy | 1 +
3 files changed, 17 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
index 12b3a93fe9..257ff517cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
@@ -284,9 +284,16 @@ public abstract class SetOperationNode extends PlanNode {
return false;
}
if (VectorizedUtil.isVectorized()) {
+// On vectorized engine, we have more chance to do passthrough.
if (childSlotRef.getDesc().getSlotOffset() !=
setOpSlotRef.getDesc().getSlotOffset()) {
return false;
}
+if (childSlotRef.isNullable() != setOpSlotRef.isNullable()) {
+return false;
+}
+if (childSlotRef.getDesc().getType() !=
setOpSlotRef.getDesc().getType()) {
+return false;
+}
} else {
if
(!childSlotRef.getDesc().layoutEquals(setOpSlotRef.getDesc())) {
return false;
diff --git a/regression-test/data/query/union/test_union.out
b/regression-test/data/query/union/test_union.out
index 56538122b0..9dd5905feb 100644
--- a/regression-test/data/query/union/test_union.out
+++ b/regression-test/data/query/union/test_union.out
@@ -20,3 +20,12 @@
8 255
9 1991
+-- !select --
+1985 1
+1986 2
+1989 2
+1991 1
+1992 1
+2551
+32767 1
+
diff --git a/regression-test/suites/query/union/test_union.groovy
b/regression-test/suites/query/union/test_union.groovy
index 52ea4c3127..300f3a484b 100644
--- a/regression-test/suites/query/union/test_union.groovy
+++ b/regression-test/suites/query/union/test_union.groovy
@@ -17,4 +17,5 @@
suite("test_union", "query") {
order_qt_select "select k1, k2 from test_query_db.baseall union select k2,
k3 from test_query_db.test"
+order_qt_select "select k2, count(k1) from ((select k2, avg(k1) k1 from
test_query_db.baseall group by k2) union all (select k2, count(k1) k1 from
test_query_db.test group by k2) )b group by k2 having k2 > 0 order by k2;"
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yiguolei closed issue #10038: [Bug] [Vectorized] code dump on aggregate node over union node
yiguolei closed issue #10038: [Bug] [Vectorized] code dump on aggregate node over union node URL: https://github.com/apache/incubator-doris/issues/10038 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yiguolei merged pull request #10040: [Bug] [Vectorized] code dump on aggregate node over union node
yiguolei merged PR #10040: URL: https://github.com/apache/incubator-doris/pull/10040 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yiguolei merged pull request #9995: [fix](fe) select stmt will make BE coredump when its castExpr is like cast(int as array<>)
yiguolei merged PR #9995: URL: https://github.com/apache/incubator-doris/pull/9995 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10053: [doc] add description of [DEFAULT CURRENT_TIMESTAMP]
github-actions[bot] commented on PR #10053: URL: https://github.com/apache/incubator-doris/pull/10053#issuecomment-1152038776 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [fix](fe) select stmt will make BE coredump when its castExpr is like cast(int as array<>) (#9995)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4135e59f77 [fix](fe) select stmt will make BE coredump when its
castExpr is like cast(int as array<>) (#9995)
4135e59f77 is described below
commit 4135e59f77c1aa5a99c84ebc4175dc3c77b58bbd
Author: yinzhijian <373141...@qq.com>
AuthorDate: Fri Jun 10 15:03:09 2022 +0800
[fix](fe) select stmt will make BE coredump when its castExpr is like
cast(int as array<>) (#9995)
* [fix](fe) select stmt will make BE coredump when its castExpr is like
cast(int as array<>)
* fix implicit cast scalar type bug
* Revert "fix implicit cast scalar type bug"
This reverts commit 1f05b6bab72430214dca88f386b50ef9a081e60a.
* only check array cast, retrigger
---
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java | 9 +
1 file changed, 9 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 10cd2d065c..f5dd01c112 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -265,6 +265,15 @@ public class CastExpr extends Expr {
noOp = true;
return;
}
+// select stmt will make BE coredump when its castExpr is like
cast(int as array<>),
+// it is necessary to check if it is castable before creating fn.
+// char type will fail in canCastTo, so for compatibility, only the
cast of array type is checked here.
+if (type.isArrayType() || childType.isArrayType()) {
+if (!Type.canCastTo(childType, type)) {
+throw new AnalysisException("Invalid type cast of " +
getChild(0).toSql()
++ " from " + childType + " to " + type);
+}
+}
this.opcode = TExprOpcode.CAST;
FunctionName fnName = new FunctionName(getFnName(type));
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yiguolei closed pull request #9303: [feature](planner) Support MetaScan for global dict
yiguolei closed pull request #9303: [feature](planner) Support MetaScan for global dict URL: https://github.com/apache/incubator-doris/pull/9303 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yiguolei commented on pull request #9303: [feature](planner) Support MetaScan for global dict
yiguolei commented on PR #9303: URL: https://github.com/apache/incubator-doris/pull/9303#issuecomment-1152050238 duplicate with https://github.com/apache/incubator-doris/pull/9842 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10058: [Improvement] add regression test for outfile
github-actions[bot] commented on PR #10058: URL: https://github.com/apache/incubator-doris/pull/10058#issuecomment-1152063796 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10058: [Improvement] add regression test for outfile
github-actions[bot] commented on PR #10058: URL: https://github.com/apache/incubator-doris/pull/10058#issuecomment-1152063822 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] qimumu9406 opened a new issue, #10059: [Enhancement] Document image loading problem
qimumu9406 opened a new issue, #10059: URL: https://github.com/apache/incubator-doris/issues/10059 ### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and found no similar issues. ### Description  The image in the document could not be loaded ### Solution _No response_ ### Are you willing to submit PR? - [X] Yes I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] HappenLee opened a new pull request, #10060: [Bug] Fix bug push value predicate of unique table when have sequence column
HappenLee opened a new pull request, #10060: URL: https://github.com/apache/incubator-doris/pull/10060 # Proposed changes Issue Number: close #xxx ## Problem Summary: Describe the overview of changes. ## Checklist(Required) 1. Does it affect the original behavior: (Yes/No/I Don't know) 2. Has unit tests been added: (Yes/No/No Need) 3. Has document been added or modified: (Yes/No/No Need) 4. Does it need to update dependencies: (Yes/No) 5. Are there any changes that cannot be rolled back: (Yes/No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yangzhg commented on a diff in pull request #10060: [Bug] Fix bug push value predicate of unique table when have sequence column
yangzhg commented on code in PR #10060:
URL: https://github.com/apache/incubator-doris/pull/10060#discussion_r894278746
##
be/src/olap/rowset/beta_rowset_reader.h:
##
@@ -53,6 +53,8 @@ class BetaRowsetReader : public RowsetReader {
RowsetTypePB type() const override { return RowsetTypePB::BETA_ROWSET; }
private:
+bool _should_push_down_value_predicates() const;
Review Comment:
private function should not with prefix `_`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10018: [dependency][enhancement] support build libhdfs in arm cpus
github-actions[bot] commented on PR #10018: URL: https://github.com/apache/incubator-doris/pull/10018#issuecomment-1152109361 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10018: [dependency][enhancement] support build libhdfs in arm cpus
github-actions[bot] commented on PR #10018: URL: https://github.com/apache/incubator-doris/pull/10018#issuecomment-1152109321 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10060: [Bug] Fix bug push value predicate of unique table when have sequence column
github-actions[bot] commented on PR #10060: URL: https://github.com/apache/incubator-doris/pull/10060#issuecomment-1152117124 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10060: [Bug] Fix bug push value predicate of unique table when have sequence column
github-actions[bot] commented on PR #10060: URL: https://github.com/apache/incubator-doris/pull/10060#issuecomment-1152117075 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] yiguolei opened a new pull request, #10061: [refactor] remove useless plugin test code
yiguolei opened a new pull request, #10061: URL: https://github.com/apache/incubator-doris/pull/10061 # Proposed changes Issue Number: close #xxx ## Problem Summary: Describe the overview of changes. ## Checklist(Required) 1. Does it affect the original behavior: (Yes/No/I Don't know) 2. Has unit tests been added: (Yes/No/No Need) 3. Has document been added or modified: (Yes/No/No Need) 4. Does it need to update dependencies: (Yes/No) 5. Are there any changes that cannot be rolled back: (Yes/No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris-website] branch master updated: remove
This is an automated email from the ASF dual-hosted git repository.
jiafengzheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new 39fe56efe8f remove
39fe56efe8f is described below
commit 39fe56efe8f8961190e6f1568bce5d63469ceef2
Author: jiafeng.zhang
AuthorDate: Fri Jun 10 17:08:16 2022 +0800
remove
remove
---
.../doris-storage-reader-compaction.md | 228
.../DorisInternals/doris-storage-struct-design.md | 303 -
.../DorisInternals/doris-storage-writer-delete.md | 150 --
3 files changed, 681 deletions(-)
diff --git a/blogs/en/DorisInternals/doris-storage-reader-compaction.md
b/blogs/en/DorisInternals/doris-storage-reader-compaction.md
deleted file mode 100644
index 7672ad38eb8..000
--- a/blogs/en/DorisInternals/doris-storage-reader-compaction.md
+++ /dev/null
@@ -1,228 +0,0 @@
-{
- "title": "Apache Doris storage layer design three reading process,
Compaction process analysis",
- "description": "This article introduces in detail the internal
implementation process of the Doris system during the data writing process, as
well as the implementation process of Doris's conditional deletion of data and
batch deletion by key.",
- "date": "2022-05-20",
- "metaTitle": "Apache Doris storage layer design three reading process,
Compaction process analysis",
- "isArticle": true,
- "language": "en",
- "author": "ApacheDoris",
- "layout": "Article",
- "sidebar": false,
- "categories": "DorisInternals",
-}
-
-
-
-# Apache Doris storage layer design three reading process, Compaction process
analysis
-
-## 1 Overall introduction
-
-Doris is an interactive SQL data warehouse based on MPP architecture, mainly
used to solve near real-time reports and multi-dimensional analysis. The
efficient import and query of Doris is inseparable from the sophisticated
design of its storage structure.
-
-This article mainly analyzes the implementation principle of the storage layer
of the Doris BE module by reading the code of the Doris BE module, and expounds
and decrypts the core technology behind the efficient writing and query
capabilities of Doris. It includes Doris column storage design, index design,
data read and write process, Compaction process and other functions.
-
-This article introduces in detail the internal implementation process of the
Doris system during the data writing process, as well as the implementation
process of Doris's conditional deletion of data and batch deletion by key.
-
-## 2 Read process
-
-### 2.1 Overall reading process
-
-The read process is the reverse process of writing, but the read process is
relatively complicated, mainly because of a large number of read optimizations.
The entire reading process is divided into two stages, one is the init process,
and the other is the process of obtaining the next_block data block. The
specific process is shown in the following figure:
-
-
-
-The hierarchical relationship is as follows:
-
-OlapScanner encapsulates the overall read operation of a tablet data;
-
-Reader processes the read parameters, and provides differentiated processing
for reading according to three different models;
-
-CollectIterator contains multiple RowsetReaders in the tablet. These
RowsetReaders have version order. CollectIterator merges these RowsetReaders
into a unified Iterator function and provides a merged comparator;
-
-RowsetReader is responsible for reading a Rowset;
-
-RowwiseIterator provides an Iterator function for unified access to all
Segments in a Rowset. The merge strategy here can use Merge or Union according
to the data sorting;
-
-SegmentIterator corresponds to the data read of a segment. The segment read
will calculate the corresponding line number information read according to the
query conditions and the index, seek to the corresponding page, and read the
data. Among them, after filtering conditions, a bitmap will be generated for
the accessible row information to record, BitmapRangeIterator is a separate
iterator that can access this bitmap according to the range;
-
-ColumnIterator provides an iterator for uniform access to a column's related
data and indexes. ColumnReader, each IndexReader, etc. correspond to the
reading of specific data and index information.
-
-### 2.2 The main process of reading the Init stage
-
-The execution flow of the initialization phase is as follows:
-
-
-
- 2.2.1 OlapScanner query parameter construction
-
-Find the RowsetReader that needs to be read according to the version specified
by the query (depending on the rowset_graph version path map of version
management to obtain the shortest path of the query version range);
-
-1. Set
[GitHub] [incubator-doris] morrySnow commented on a diff in pull request #9942: [Enhancement] (Nereids) scalar expression rewrite framework
morrySnow commented on code in PR #9942:
URL: https://github.com/apache/incubator-doris/pull/9942#discussion_r894323373
##
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/RewriteHelper.java:
##
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rewrite;
+
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Literal;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Expression rewrite helper class.
+ */
+public class RewriteHelper {
+
+public static boolean isConstant(Expression expr) {
+return expr.isConstant();
+}
+
+public static Expression convertLiteralToSlot(Expression expr) {
+ExpressionRewriter rewriter = new
ExpressionRewriter(ConvertLiteralToSlotRule.INSTANCE);
+return rewriter.rewrite(expr);
+}
+
+/**
+ * For test.
+ * convert literal to slot
Review Comment:
if just for test, it's better move it to test folder
##
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTest.java:
##
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rewrite;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.nereids.parser.SqlParser;
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeExpressionRule;
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule;
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * all expr rewrite rule test case.
+ */
+public class ExpressionRewriteTest {
+private static final Logger LOG = LogManager.getLogger(Database.class);
+
+private static final SqlParser PARSER = new SqlParser();
+private ExpressionRewriter rewriter;
+
+@Test
+public void testNotExpressionRewrite() {
+rewriter = new ExpressionRewriter(SimplifyNotExprRule.INSTANCE);
+
+assertRewrite("NOT x > y", "x <= y");
+assertRewrite("NOT x < y", "x >= y");
+assertRewrite("NOT x >= y", "x < y");
+assertRewrite("NOT x <= y", "x > y");
+assertRewrite("NOT x = y", "NOT x = y");
+assertRewrite("NOT NOT x > y", "x > y");
+assertRewrite("NOT NOT NOT x > y", "x <= y");
+}
+
+@Test
+public void testNormalizeExpressionRewrite() {
+rewriter = new ExpressionRewriter(NormalizeExpressionRule.INSTANCE);
+
+assertRewrite("2 > x", "x < 2");
+assertRewrite("2 >= x", "x <= 2");
+assertRewrite("2 < x", "x > 2");
+assertRewrite("2 <= x", "x >= 2");
+assertRewrite("2 = x", "x = 2");
+/*
Review Comment:
remove it if unused
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.a
[GitHub] [incubator-doris] liutang123 opened a new pull request, #10062: Do not send drop task when replay drop table
liutang123 opened a new pull request, #10062: URL: https://github.com/apache/incubator-doris/pull/10062 # Proposed changes Issue Number: close #xxx ## Problem Summary: When doing checkpoint, FE will sends DropTask to BE. This PR prohibit this conduct. ## Checklist(Required) 1. Does it affect the original behavior: (Yes/No/I Don't know) 2. Has unit tests been added: (Yes/No/No Need) 3. Has document been added or modified: (Yes/No/No Need) 4. Does it need to update dependencies: (Yes/No) 5. Are there any changes that cannot be rolled back: (Yes/No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10019: [Enhancement](Nereids) generate pattern by operator
github-actions[bot] commented on PR #10019: URL: https://github.com/apache/incubator-doris/pull/10019#issuecomment-1152164876 PR approved by anyone and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] SWJTU-ZhangLei opened a new issue, #10063: [Bug] asan be core doris::NotInListPredicate::evaluate
SWJTU-ZhangLei opened a new issue, #10063: URL: https://github.com/apache/incubator-doris/issues/10063 ### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and found no similar issues. ### Version root@regtest-15-bj:/mnt/hdd01/DORIS_MASTER_ASAN/be/log# /mnt/hdd01/DORIS_MASTER_ASAN/be/lib/palo_be --version trunk DEBUG (build git://regtest-15-bj/mnt/hdd01/repo_center/doris_master/be/../@73a30935399108a6fc2abf8b5017bd1ff2872dc6) Built on Fri, 10 Jun 2022 07:22:07 CST by root@regtest-15-bj ### What's Wrong? (gdb) bt #0 0x7f343452ffbb in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x7f3434515864 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x557b91b3fbf2 in google::DumpStackTraceAndExit () at src/utilities.cc:160 #3 0x557b9840511d in google::LogMessage::Fail () at src/logging.cc:1650 #4 0x557b98407659 in google::LogMessage::SendToLog (this=0x7f340c638720) at src/logging.cc:1607 #5 0x557b98404c86 in google::LogMessage::Flush (this=0x7f340c638720) at src/logging.cc:1476 #6 0x557b98407cc9 in google::LogMessageFatal::~LogMessageFatal (this=, __in_chrg=) at src/logging.cc:2226 #7 0x557b929b1872 in doris::NotInListPredicate::evaluate (this=0x606000699980, column=..., sel=0x7f340c638ac0, size=0x7f340c638be0) at /mnt/hdd01/repo_center/doris_master/be/src/olap/in_list_predicate.cpp:191 #8 0x557b94d90ad1 in doris::segment_v2::SegmentIterator::_evaluate_short_circuit_predicate (this=0x616000394880, vec_sel_rowid_idx=0x7f340c638ac0, selected_size_ptr=0x7f340c638be0) at /mnt/hdd01/repo_center/doris_master/be/src/olap/rowset/segment_v2/segment_iterator.cpp:917 #9 0x557b94d9288a in doris::segment_v2::SegmentIterator::next_batch (this=0x616000394880, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1015 #10 0x557b92c10af7 in doris::BetaRowsetReader::next_block (this=0x6130007ac480, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/olap/rowset/beta_rowset_reader.cpp:189 #11 0x557b940e8767 in doris::vectorized::VCollectIterator::Level0Iterator::next (this=0x6070040151c0, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/vcollect_iterator.cpp:233 #12 0x557b940eb80a in doris::vectorized::VCollectIterator::Level1Iterator::_normal_next (this=0x60bdbc20, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/vcollect_iterator.cpp:424 #13 0x557b940ebab6 in doris::vectorized::VCollectIterator::Level1Iterator::_normal_next (this=0x60bdbc20, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/vcollect_iterator.cpp:433 #14 0x557b940e9349 in doris::vectorized::VCollectIterator::Level1Iterator::next (this=0x60bdbc20, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/vcollect_iterator.cpp:294 #15 0x557b940e70ff in doris::vectorized::VCollectIterator::next (this=0x61b0002a9e98, block=0x6070042b6dc0) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/vcollect_iterator.cpp:173 #16 0x557b940f931c in doris::vectorized::BlockReader::_direct_next_block (this=0x61b0002a9a80, block=0x6070042b6dc0, mem_pool=0x0, agg_pool=0x0, eof=0x7f340c640840) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/block_reader.cpp:162 #17 0x557b92687ccb in doris::vectorized::BlockReader::next_block_with_aggregation (this=0x61b0002a9a80, block=0x6070042b6dc0, mem_pool=0x0, agg_pool=0x0, eof=0x7f340c640840) at /mnt/hdd01/repo_center/doris_master/be/src/vec/olap/block_reader.h:49 #18 0x557b951618f5 in doris::vectorized::VOlapScanner::get_block (this=0x617000347980, state=0x62397080, block=0x6070042b6dc0, eof=0x7f340c640840) at /mnt/hdd01/repo_center/doris_master/be/src/vec/exec/volap_scanner.cpp:56 #19 0x557b9512ec93 in doris::vectorized::VOlapScanNode::scanner_thread (this=0x621000e11500, scanner=0x617000347980) at /mnt/hdd01/repo_center/doris_master/be/src/vec/exec/volap_scan_node.cpp:219 #20 0x557b9514cbad in std::__invoke_impl (__f= @0x6030019ea150: (void (doris::vectorized::VOlapScanNode::*)(doris::vectorized::VOlapScanNode * const, doris::vectorized::VOlapScanner *)) 0x557b9512d392 , __t=@0x6030019ea168: 0x621000e11500) at /var/local/ldb_toolchain/include/c++/11/bits/invoke.h:74 #21 0x557b9514c765 in std::__invoke ( __fn= @0x6030019ea150: (void (doris::vectorized::VOlapScanNode::*)(doris::vectorized::VOlapScanNode * const, doris::vectorized::VOlapScanner *)) 0x557b9512d392 ) at /var/local/ldb_toolchain/include/c++/11/bits/invoke.h:96 #22 0x557b9514c097 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul,
[GitHub] [incubator-doris] wangshuo128 commented on a diff in pull request #9942: [Enhancement] (Nereids) scalar expression rewrite framework
wangshuo128 commented on code in PR #9942:
URL: https://github.com/apache/incubator-doris/pull/9942#discussion_r894345674
##
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriter.java:
##
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rewrite;
+
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeExpressionRule;
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.LeafExpression;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Expression rewrite entry, which contains all rewrite rules.
+ */
+public class ExpressionRewriter {
Review Comment:
I'm afraid this is duplicated with `ExpressionVisitor`.
We can traverse the expression tree in the `applyRule` or by the visitor,
it's not a good design.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
eldenmoon commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r894355137
##
be/src/vec/olap/olap_data_convertor.cpp:
##
@@ -700,4 +621,55 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap()
{
+const ColumnArray* column_array = nullptr;
+const DataTypeArray* data_type_ptr_array = nullptr;
+if (_nullmap) {
+const auto* nullable_column =
+assert_cast(_typed_column.column.get());
+column_array =
+assert_cast(nullable_column->get_nested_column_ptr().get());
+data_type_ptr_array = assert_cast(
+(assert_cast(_typed_column.type.get())->get_nested_type())
+.get());
+} else {
+column_array = assert_cast(_typed_column.column.get());
+data_type_ptr_array = assert_cast(_typed_column.type.get());
+}
+assert(column_array);
+assert(data_type_ptr_array);
+
+const auto& offsets = column_array->get_offsets();
+CollectionValue* collection_value = _values.data();
+for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+int64_t cur_pos = _row_pos + i;
+int64_t prev_pos = cur_pos - 1;
+if (_nullmap && _nullmap[cur_pos]) {
+continue;
+}
+auto offset = offsets[prev_pos];
+auto size = offsets[cur_pos] - offsets[prev_pos];
+new (collection_value) CollectionValue(size);
+
+if (size == 0) {
+continue;
+}
+
+if (column_array->get_data().is_nullable()) {
+const auto& data_nullable_column =
+assert_cast(column_array->get_data());
+const auto* data_null_map =
data_nullable_column.get_null_map_data().data();
+collection_value->set_has_null(true);
+collection_value->set_null_signs(
+const_cast(reinterpret_cast(data_null_map + offset)));
+}
+ColumnWithTypeAndName item_typed_column =
{column_array->get_data_ptr(),
+
data_type_ptr_array->get_nested_type(), ""};
+_item_convertor->set_source_column(item_typed_column, offset, size);
Review Comment:
if item_type is or varchar or string, `set_source_column` will `resize()`
the `_item_convertor._slice` and the slices from the previous
`convert_to_olap()` will be freed, and then lead to heap use after free in the
subsequent use of the collection_value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
eldenmoon commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r894357886
##
be/src/vec/olap/olap_data_convertor.cpp:
##
@@ -700,4 +621,55 @@ Status
OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap()
{
+const ColumnArray* column_array = nullptr;
+const DataTypeArray* data_type_ptr_array = nullptr;
+if (_nullmap) {
+const auto* nullable_column =
+assert_cast(_typed_column.column.get());
+column_array =
+assert_cast(nullable_column->get_nested_column_ptr().get());
+data_type_ptr_array = assert_cast(
+(assert_cast(_typed_column.type.get())->get_nested_type())
+.get());
+} else {
+column_array = assert_cast(_typed_column.column.get());
+data_type_ptr_array = assert_cast(_typed_column.type.get());
+}
+assert(column_array);
+assert(data_type_ptr_array);
+
+const auto& offsets = column_array->get_offsets();
+CollectionValue* collection_value = _values.data();
+for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+int64_t cur_pos = _row_pos + i;
+int64_t prev_pos = cur_pos - 1;
+if (_nullmap && _nullmap[cur_pos]) {
+continue;
+}
+auto offset = offsets[prev_pos];
+auto size = offsets[cur_pos] - offsets[prev_pos];
+new (collection_value) CollectionValue(size);
+
+if (size == 0) {
+continue;
+}
+
+if (column_array->get_data().is_nullable()) {
+const auto& data_nullable_column =
+assert_cast(column_array->get_data());
+const auto* data_null_map =
data_nullable_column.get_null_map_data().data();
+collection_value->set_has_null(true);
+collection_value->set_null_signs(
+const_cast(reinterpret_cast(data_null_map + offset)));
+}
+ColumnWithTypeAndName item_typed_column =
{column_array->get_data_ptr(),
+
data_type_ptr_array->get_nested_type(), ""};
+_item_convertor->set_source_column(item_typed_column, offset, size);
Review Comment:
the same as array, hll or bitmap
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 opened a new issue, #10064: [Bug][array-type] Failed to select array type data after enable storage vectorization.
adonis0147 opened a new issue, #10064:
URL: https://github.com/apache/incubator-doris/issues/10064
### Search before asking
- [X] I had searched in the
[issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and
found no similar issues.
### Version
master
### What's Wrong?
In commit #9848 , the configuration `enable_storage_vectorization` in `BE`
was enabled by default.
However, the `ArrayFileColumnIterator` didn't implement the interface
`Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool*
has_null)`. Thereby, the array type data can't be retrieved from underlying
data file.
### What You Expected?
Select array type data successfully.
### How to Reproduce?
```shell
mysql> create table array_type_table_1(k1 INT, k2 Array) duplicate key
(k1)
-> distributed by hash(k1) buckets 1 properties('replication_num' = '1');
mysql> set enable_array_type = true;
mysql> set enable_vectorized_engine = true;
mysql> insert into array_type_table values (0, []), (1, [NULL]), (2, [1, 2,
3]), (3, [1, NULL, 3]);
mysql> select * from array_type_table;
Empty set (0.03 sec)
```
### Anything Else?
_No response_
### Are you willing to submit PR?
- [X] Yes I am willing to submit a PR!
### Code of Conduct
- [X] I agree to follow this project's [Code of
Conduct](https://www.apache.org/foundation/policies/conduct)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] adonis0147 opened a new pull request, #10065: [feature-wip](array-type) Support loading data in vectorized format
adonis0147 opened a new pull request, #10065: URL: https://github.com/apache/incubator-doris/pull/10065 # Proposed changes Issue Number: close #10064 ## Problem Summary: Please refer to #10064 ## Checklist(Required) 1. Does it affect the original behavior: No 2. Has unit tests been added: Yes 3. Has document been added or modified: No Need 4. Does it need to update dependencies: No 5. Are there any changes that cannot be rolled back: No ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
cambyzju commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894368499
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
Review Comment:
RETURN_IF_ERROR
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
cambyzju commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894370999
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
+auto& column_offsets =
+
static_cast(*column_offsets_ptr);
+auto& offsets_data = column_offsets.get_data();
+for (int i = start; i < offsets_data.size(); ++i) {
Review Comment:
```suggestion
for (size_t i = start; i < offsets_data.size(); ++i) {
```
we will change offsets of array to int64 later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
cambyzju commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894374280
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
+auto& column_offsets =
+
static_cast(*column_offsets_ptr);
+auto& offsets_data = column_offsets.get_data();
+for (int i = start; i < offsets_data.size(); ++i) {
+offsets_data[i] += offsets_data[i - 1]; // -1 is ok
+}
+
+auto column_items_ptr = column_array->get_data().assume_mutable();
+for (int64_t i = start; i < offsets_data.size(); ++i) {
+auto offset = offsets_data[i - 1]; // -1 is ok.
+auto size = offsets_data[i] - offset;
+if (size == 0) {
+continue;
+}
+size_t num_items = size;
+bool items_has_null = false;
+_item_iterator->next_batch(&num_items, column_items_ptr,
&items_has_null);
Review Comment:
First get total item size, then call next_batch repeatedly until reach total
item size, we could reduce the call of next_batch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
cambyzju commented on code in PR #10065:
URL: https://github.com/apache/incubator-doris/pull/10065#discussion_r894376462
##
be/src/olap/rowset/segment_v2/column_reader.cpp:
##
@@ -461,6 +461,46 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
return Status::OK();
}
+Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+const auto* column_array =
vectorized::check_and_get_column(
+dst->is_nullable() ?
static_cast(*dst).get_nested_column()
+ : *dst);
+
+bool offsets_has_null = false;
+auto column_offsets_ptr =
column_array->get_offsets_column().assume_mutable();
+auto start = column_offsets_ptr->size();
+_length_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null);
+auto& column_offsets =
+
static_cast(*column_offsets_ptr);
+auto& offsets_data = column_offsets.get_data();
+for (int i = start; i < offsets_data.size(); ++i) {
+offsets_data[i] += offsets_data[i - 1]; // -1 is ok
+}
Review Comment:
we need set `n` after read data success.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
eldenmoon commented on PR #10065: URL: https://github.com/apache/incubator-doris/pull/10065#issuecomment-1152217635 Sorry for my issue in this PR. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on pull request #10065: [feature-wip](array-type) Support loading data in vectorized format
eldenmoon commented on PR #10065: URL: https://github.com/apache/incubator-doris/pull/10065#issuecomment-115425 Sorry for my issue in this PR, I'm currently using Array, found that when BE build with ASAN, the column type is like ARRAY, there will be a coredump which is heap-use-after-free after I insert a data with `enable_vectorized_engine` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] platoneko opened a new issue, #10066: [Enhancement] Shall we replace SpinLock with std::mutex in MetricRegistry and MetricEntity?
platoneko opened a new issue, #10066: URL: https://github.com/apache/incubator-doris/issues/10066 ### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-doris/issues?q=is%3Aissue) and found no similar issues. ### Description Currently, MetricRegistry and MetricEntity use SpinLock to solve concurrency problems. However when `MetricRegistry` call trigger_all_hooks, these triggered hook functions must hold other mutex, and these mutex may be held during some long time work. https://user-images.githubusercontent.com/36853835/173046811-0d40637a-29ce-4bfc-a46d-d5109757a47f.png";> https://user-images.githubusercontent.com/36853835/173047086-8c19b6ec-b39f-4476-8f06-36ea217e3e18.png";> i.e. LoadChannelMgr::_lock may be held during MemTable flush. ### Solution _No response_ ### Are you willing to submit PR? - [X] Yes I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] qzsee commented on a diff in pull request #9942: [Enhancement] (Nereids) scalar expression rewrite framework
qzsee commented on code in PR #9942:
URL: https://github.com/apache/incubator-doris/pull/9942#discussion_r894390127
##
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriter.java:
##
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rewrite;
+
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeExpressionRule;
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.LeafExpression;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * Expression rewrite entry, which contains all rewrite rules.
+ */
+public class ExpressionRewriter {
Review Comment:
you are right . I fixed it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] platoneko commented on issue #10066: [Enhancement] Shall we replace SpinLock with std::mutex in MetricRegistry and MetricEntity?
platoneko commented on issue #10066: URL: https://github.com/apache/incubator-doris/issues/10066#issuecomment-1152228358 Or can we avoid acquiring lock in metrics hook functions? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #10018: [dependency][enhancement] support build libhdfs in arm cpus
github-actions[bot] commented on PR #10018: URL: https://github.com/apache/incubator-doris/pull/10018#issuecomment-1152251581 PR approved by at least one committer and no changes requested. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] wangshuo128 opened a new pull request, #10067: Add bind slot reference rules
wangshuo128 opened a new pull request, #10067: URL: https://github.com/apache/incubator-doris/pull/10067 # Proposed changes Issue Number: close #xxx ## Problem Summary: Describe the overview of changes. ## Checklist(Required) 1. Does it affect the original behavior: (Yes/No/I Don't know) 2. Has unit tests been added: (Yes/No/No Need) 3. Has document been added or modified: (Yes/No/No Need) 4. Does it need to update dependencies: (Yes/No) 5. Are there any changes that cannot be rolled back: (Yes/No) ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] luozenglin closed issue #9737: [Bug] InPredicate child expressions type conversion related exceptions
luozenglin closed issue #9737: [Bug] InPredicate child expressions type conversion related exceptions URL: https://github.com/apache/incubator-doris/issues/9737 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman closed issue #10011: [Enhancement] Support build doris with libhdfs3 on ARM CPUs
morningman closed issue #10011: [Enhancement] Support build doris with libhdfs3 on ARM CPUs URL: https://github.com/apache/incubator-doris/issues/10011 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #10018: [dependency][enhancement] support build libhdfs in arm cpus
morningman merged PR #10018: URL: https://github.com/apache/incubator-doris/pull/10018 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
[incubator-doris] branch master updated: [dependency][enhancement] support build libhdfs in arm cpus (#10018)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e0cf2677a0 [dependency][enhancement] support build libhdfs in arm cpus
(#10018)
e0cf2677a0 is described below
commit e0cf2677a02db31f386ccf72a760b22e825af315
Author: Zhengguo Yang
AuthorDate: Fri Jun 10 19:40:41 2022 +0800
[dependency][enhancement] support build libhdfs in arm cpus (#10018)
Supports native hdfs functionality on arm cpu
This pr mainly upgrades libdfs3 and supports running on arm，and make
libhdfs3 with kerberos as default
---
be/CMakeLists.txt| 84 ++-
be/src/exec/CMakeLists.txt | 18
be/src/io/CMakeLists.txt | 22 ++---
be/src/io/broker_reader.cpp | 2 +-
be/src/io/broker_reader.h| 2 +-
be/src/io/broker_writer.cpp | 2 +-
be/src/io/broker_writer.h| 2 +-
be/src/io/buffered_reader.cpp| 2 +-
be/src/io/buffered_reader.h | 2 +-
be/src/io/file_factory.cpp | 18 ++--
be/src/io/file_factory.h | 4 +-
be/src/io/hdfs_file_reader.cpp | 2 +-
be/src/io/hdfs_file_reader.h | 2 +-
be/src/io/hdfs_reader_writer.cpp | 16 +---
be/src/io/hdfs_reader_writer.h | 4 +-
be/src/io/hdfs_writer.cpp| 2 +-
be/src/io/hdfs_writer.h | 2 +-
be/src/io/local_file_reader.cpp | 2 +-
be/src/io/local_file_reader.h| 2 +-
be/src/io/local_file_writer.cpp | 2 +-
be/src/io/local_file_writer.h| 2 +-
be/src/io/s3_reader.cpp | 2 +-
be/src/io/s3_reader.h| 2 +-
be/src/io/s3_writer.cpp | 2 +-
be/src/io/s3_writer.h| 2 +-
build.sh | 5 --
dist/LICENSE-dist.txt| 1 +
run-be-ut.sh | 1 -
thirdparty/CHANGELOG.md | 5 ++
thirdparty/build-thirdparty.sh | 33 ++--
thirdparty/download-thirdparty.sh| 13 +--
thirdparty/patches/libhdfs3-master.patch | 138 ---
thirdparty/vars.sh | 26 ++
33 files changed, 92 insertions(+), 332 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 480a3c6e46..999801ea26 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -85,7 +85,6 @@ set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
set(ENV{DORIS_HOME} "${BASE_DIR}/../")
set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
-set(HDFS3_KRB5_INSTALL_DIR
"$ENV{DORIS_THIRDPARTY}/installed/libhdfs_with_kerberos/")
set(GENSRC_DIR "${BASE_DIR}/../gensrc/build/")
set(SRC_DIR "${BASE_DIR}/src/")
set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
@@ -98,7 +97,6 @@ else()
endif()
message(STATUS "make test: ${MAKE_TEST}")
option(WITH_MYSQL "Support access MySQL" ON)
-option(WITH_KERBEROS "Enable or disable Kereberos support" ${WITH_KERBEROS})
# Check gcc
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -330,43 +328,32 @@ set_target_properties(minizip PROPERTIES
IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
add_library(idn STATIC IMPORTED)
set_target_properties(idn PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libidn.a)
-if (WITH_KERBEROS)
-# kerberos lib for libhdfs3
-add_library(gsasl STATIC IMPORTED)
-set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION
${HDFS3_KRB5_INSTALL_DIR}/lib/libgsasl.a)
+add_library(xml2 STATIC IMPORTED)
+set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/libxml2.a)
-add_library(krb5support STATIC IMPORTED)
-set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION
${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5support.a)
-
-add_library(krb5 STATIC IMPORTED)
-set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION
${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5.a)
+add_library(lzma STATIC IMPORTED)
+set_target_properties(lzma PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib64/liblzma.a)
-add_library(com_err STATIC IMPORTED)
-set_target_properties(com_err PROPERTIES IMPORTED_LOCATION
${HDFS3_KRB5_INSTALL_DIR}/lib/libcom_err.a)
+add_library(gsasl STATIC IMPORTED)
+set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib/libgsasl.a)
-add_library(k5crypto STATIC IMPORTED)
-set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION
${HDFS3_KRB5_INSTALL_DIR}/lib/libk5crypto.a)
+add_library(krb5support STATIC IMPORTED)
+set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib/libkrb5support.a)
-add_library(gssapi_krb5 STATIC IMPORTED)
-set_target_propert
