This is an automated email from the ASF dual-hosted git repository.
lichaoyong pushed a commit to branch branch-0.11
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/branch-0.11 by this push:
new fcc2d40 Fix bug when use ZoneMap/BloomFiter on column with
REPLACE/REPLACE_IF_NOT_NULL (#3288)
fcc2d40 is described below
commit fcc2d402474539b249e9bf97b60d9e1ed42a44e3
Author: lichaoyong <[email protected]>
AuthorDate: Fri Apr 10 10:22:21 2020 +0800
Fix bug when use ZoneMap/BloomFiter on column with
REPLACE/REPLACE_IF_NOT_NULL (#3288)
Now, column with REPLACE/REPLACE_IF_NOT_NULL can be filtered by
ZoneMap/BloomFilter
when the rowset is base(version starts with zero). Always we think is an
optimization.
But when some case, it will occurs bug.
create table test(
k1 int,
v1 int replace,
v2 int sum
);
If I have two records on different two versions
1 2 2 on version [0-10]
1 3 1 on version 11
If I perform a query
select * from test where k1 = 1 and v1 = 3;
The result will be 1 3 1, this is not right because of the first record is
filtered.
The right answer is 1 3 3, the v2 should be summed.
Remove this optimization is necessity to make the result is right.
---
be/src/olap/rowset/beta_rowset_writer.cpp | 2 +-
be/src/olap/rowset/segment_reader.cpp | 8 ++------
.../main/java/org/apache/doris/common/util/PropertyAnalyzer.java | 3 +--
.../test/java/org/apache/doris/common/PropertyAnalyzerTest.java | 4 ++--
4 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index ccc30be..4281a94 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -184,4 +184,4 @@ OLAPStatus BetaRowsetWriter::_flush_segment_writer() {
return OLAP_SUCCESS;
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_reader.cpp
b/be/src/olap/rowset/segment_reader.cpp
index 2ccc4a2..b6dddbf 100644
--- a/be/src/olap/rowset/segment_reader.cpp
+++ b/be/src/olap/rowset/segment_reader.cpp
@@ -491,9 +491,7 @@ OLAPStatus SegmentReader::_pick_row_groups(uint32_t
first_block, uint32_t last_b
for (auto& i : _conditions->columns()) {
FieldAggregationMethod aggregation =
_get_aggregation_by_index(i.first);
- bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE
- || (aggregation == OLAP_FIELD_AGGREGATION_REPLACE
- && _segment_group->version().first == 0));
+ bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE);
if (!is_continue) {
continue;
}
@@ -532,9 +530,7 @@ OLAPStatus SegmentReader::_pick_row_groups(uint32_t
first_block, uint32_t last_b
for (uint32_t i : _load_bf_columns) {
FieldAggregationMethod aggregation = _get_aggregation_by_index(i);
- bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE
- || (aggregation == OLAP_FIELD_AGGREGATION_REPLACE
- && _segment_group->version().first == 0));
+ bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE);
if (!is_continue) {
continue;
}
diff --git
a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 440d03e..9c164d0 100644
--- a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -275,8 +275,7 @@ public class PropertyAnalyzer {
throw new AnalysisException(type + " is not
supported in bloom filter index. "
+ "invalid column: " + bfColumn);
} else if (column.isKey()
- || column.getAggregationType() ==
AggregateType.NONE
- || column.getAggregationType() ==
AggregateType.REPLACE) {
+ || column.getAggregationType() ==
AggregateType.NONE) {
if (!bfColumnSet.add(bfColumn)) {
throw new AnalysisException("Reduplicated
bloom filter column: " + bfColumn);
}
diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 45430e4..3aaaf20 100644
--- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -49,10 +49,10 @@ public class PropertyAnalyzerTest {
columns.get(1).setIsKey(true);
Map<String, String> properties = Maps.newHashMap();
- properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1,v1");
+ properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1");
Set<String> bfColumns =
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
- Assert.assertEquals(Sets.newHashSet("k1", "v1"), bfColumns);
+ Assert.assertEquals(Sets.newHashSet("k1"), bfColumns);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
