This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f5fe622a1b [Bug](materialized view) fix create materialized view fail
f5fe622a1b is described below
commit f5fe622a1b9f8e345b5ddb5817b0bdffdf59b713
Author: Pxl <pxl...@qq.com>
AuthorDate: Fri Aug 12 09:49:16 2022 +0800
[Bug](materialized view) fix create materialized view fail
1. remove referenced_column(seems unused now).
2. fix mv slot ref id wrong.
3. add type check for hll_hash.
4. enable non-nullable column change to nullable column.
---
be/src/olap/base_tablet.h | 4 ++--
be/src/olap/schema_change.cpp | 26 +++-------------------
be/src/olap/schema_change.h | 2 +-
be/src/olap/tablet_schema.cpp | 15 -------------
be/src/olap/tablet_schema.h | 7 ------
be/src/vec/exprs/vslot_ref.cpp | 5 ++---
.../vec/functions/function_always_not_nullable.h | 13 +++++++----
.../java/org/apache/doris/alter/RollupJobV2.java | 24 ++++++++++++++------
.../doris/analysis/CreateMaterializedViewStmt.java | 7 +++---
gensrc/proto/olap_common.proto | 2 +-
gensrc/proto/olap_file.proto | 4 ++--
11 files changed, 41 insertions(+), 68 deletions(-)
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 43e0e822ad..52a2ca2be9 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -58,7 +58,7 @@ public:
int64_t replica_id() const;
int32_t schema_hash() const;
int16_t shard_id() const;
- bool equal(int64_t tablet_id, int32_t schema_hash);
+ bool equal(int64_t tablet_id, int32_t schema_hash) const;
const std::string& storage_policy() const { return
_tablet_meta->storage_policy(); }
@@ -142,7 +142,7 @@ inline int16_t BaseTablet::shard_id() const {
return _tablet_meta->shard_id();
}
-inline bool BaseTablet::equal(int64_t id, int32_t hash) {
+inline bool BaseTablet::equal(int64_t id, int32_t hash) const {
return (tablet_id() == id) && (schema_hash() == hash);
}
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 291a8bc8fe..28272d7572 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -576,7 +576,8 @@ bool count_field(RowCursor* read_helper, RowCursor*
write_helper, const TabletCo
}
Status RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t
data_version,
- RowBlock* mutable_block, uint64_t*
filtered_rows) const {
+ RowBlock* mutable_block,
+ const uint64_t* filtered_rows) const {
if (mutable_block == nullptr) {
LOG(FATAL) << "mutable block is uninitialized.";
return Status::OLAPInternalError(OLAP_ERR_NOT_INITED);
@@ -852,13 +853,8 @@ Status
RowBlockChanger::_check_cast_valid(vectorized::ColumnPtr ref_column,
vectorized::ColumnPtr new_column)
const {
// TODO: rethink this check
// This check is to prevent schema-change from causing data loss,
- // But it is possible to generate null data in material-view or rollup.
- if (ref_column->is_nullable() != new_column->is_nullable()) {
- return Status::DataQualityError("column.is_nullable() is changed!");
- }
-
- if (ref_column->is_nullable()) {
+ if (ref_column->is_nullable() && new_column->is_nullable()) {
auto* ref_null_map =
vectorized::check_and_get_column<vectorized::ColumnNullable>(ref_column)
->get_null_map_column()
@@ -2132,22 +2128,6 @@ Status SchemaChangeHandler::_parse_request(
ColumnMapping* column_mapping =
rb_changer->get_mutable_column_mapping(i);
column_mapping->new_column = &new_column;
- if (new_column.has_reference_column()) {
- int32_t column_index =
base_tablet_schema->field_index(new_column.referenced_column());
-
- if (column_index < 0) {
- LOG(WARNING) << "referenced column was missing. "
- << "[column=" << column_name << "
referenced_column=" << column_index
- << "]";
- return Status::OLAPInternalError(OLAP_ERR_CE_CMD_PARAMS_ERROR);
- }
-
- column_mapping->ref_column = column_index;
- VLOG_NOTICE << "A column refered to existed column will be added
after schema changing."
- << "column=" << column_name << ", ref_column=" <<
column_index;
- continue;
- }
-
if (materialized_function_map.find(column_name) !=
materialized_function_map.end()) {
auto mvParam = materialized_function_map.find(column_name)->second;
column_mapping->materialized_function = mvParam.mv_expr;
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index d3623216fe..36dfed53fa 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -50,7 +50,7 @@ public:
const SchemaMapping& get_schema_mapping() const { return _schema_mapping; }
Status change_row_block(const RowBlock* ref_block, int32_t data_version,
- RowBlock* mutable_block, uint64_t* filtered_rows)
const;
+ RowBlock* mutable_block, const uint64_t*
filtered_rows) const;
Status change_block(vectorized::Block* ref_block, vectorized::Block*
new_block) const;
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index b2040f5849..d3668c838a 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -385,10 +385,6 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
} else {
_has_bitmap_index = false;
}
- _has_referenced_column = column.has_referenced_column_id();
- if (_has_referenced_column) {
- _referenced_column_id = column.referenced_column_id();
- }
if (column.has_aggregation()) {
_aggregation = get_aggregation_type_by_string(column.aggregation());
}
@@ -422,9 +418,6 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
column->set_is_bf_column(_is_bf_column);
}
column->set_aggregation(get_string_by_aggregation_type(_aggregation));
- if (_has_referenced_column) {
- column->set_referenced_column_id(_referenced_column_id);
- }
if (_has_bitmap_index) {
column->set_has_bitmap_index(_has_bitmap_index);
}
@@ -443,9 +436,6 @@ uint32_t TabletColumn::mem_size() const {
if (_has_default_value) {
size += _default_value.size();
}
- if (_has_referenced_column) {
- size += _referenced_column.size();
- }
for (auto& sub_column : _sub_columns) {
size += sub_column.mem_size();
}
@@ -713,11 +703,6 @@ bool operator==(const TabletColumn& a, const TabletColumn&
b) {
if (a._length != b._length) return false;
if (a._index_length != b._index_length) return false;
if (a._is_bf_column != b._is_bf_column) return false;
- if (a._has_referenced_column != b._has_referenced_column) return false;
- if (a._has_referenced_column) {
- if (a._referenced_column_id != b._referenced_column_id) return false;
- if (a._referenced_column != b._referenced_column) return false;
- }
if (a._has_bitmap_index != b._has_bitmap_index) return false;
return true;
}
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index aa0206cbbe..cea9a46bb8 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -62,9 +62,6 @@ public:
}
bool has_default_value() const { return _has_default_value; }
std::string default_value() const { return _default_value; }
- bool has_reference_column() const { return _has_referenced_column; }
- int32_t referenced_column_id() const { return _referenced_column_id; }
- std::string referenced_column() const { return _referenced_column; }
size_t length() const { return _length; }
size_t index_length() const { return _index_length; }
void set_index_length(size_t index_length) { _index_length = index_length;
}
@@ -109,10 +106,6 @@ private:
bool _is_bf_column = false;
- bool _has_referenced_column = false;
- int32_t _referenced_column_id;
- std::string _referenced_column;
-
bool _has_bitmap_index = false;
bool _visible = true;
diff --git a/be/src/vec/exprs/vslot_ref.cpp b/be/src/vec/exprs/vslot_ref.cpp
index 2732a341e7..0e44be9eb5 100644
--- a/be/src/vec/exprs/vslot_ref.cpp
+++ b/be/src/vec/exprs/vslot_ref.cpp
@@ -22,8 +22,7 @@
#include "runtime/descriptors.h"
namespace doris::vectorized {
-using doris::Status;
-using doris::SlotDescriptor;
+
VSlotRef::VSlotRef(const doris::TExprNode& node)
: VExpr(node), _slot_id(node.slot_ref.slot_id), _column_id(-1),
_column_name(nullptr) {
if (node.__isset.is_nullable) {
@@ -47,7 +46,7 @@ Status VSlotRef::prepare(doris::RuntimeState* state, const
doris::RowDescriptor&
return Status::OK();
}
const SlotDescriptor* slot_desc =
state->desc_tbl().get_slot_descriptor(_slot_id);
- if (slot_desc == NULL) {
+ if (slot_desc == nullptr) {
return Status::InternalError("couldn't resolve slot descriptor {}",
_slot_id);
}
_column_id = desc.get_column_id(_slot_id);
diff --git a/be/src/vec/functions/function_always_not_nullable.h
b/be/src/vec/functions/function_always_not_nullable.h
index 1c476c56c0..e15e1a91e9 100644
--- a/be/src/vec/functions/function_always_not_nullable.h
+++ b/be/src/vec/functions/function_always_not_nullable.h
@@ -20,7 +20,6 @@
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_string.h"
#include "vec/functions/function.h"
-#include "vec/functions/function_helpers.h"
namespace doris::vectorized {
@@ -49,6 +48,12 @@ public:
MutableColumnPtr column_result =
get_return_type_impl({})->create_column();
column_result->resize(input_rows_count);
+ auto type_error = [&]() {
+ return Status::RuntimeError("Illegal column {} of argument of
function {}",
+
block.get_by_position(arguments[0]).column->get_name(),
+ get_name());
+ };
+
if (const ColumnNullable* col_nullable =
check_and_get_column<ColumnNullable>(column.get())) {
const ColumnString* col =
@@ -62,6 +67,8 @@ public:
block.replace_by_position(result, std::move(column_result));
return Status::OK();
+ } else {
+ return type_error();
}
} else if (const ColumnString* col =
check_and_get_column<ColumnString>(column.get())) {
Function::vector(col->get_chars(), col->get_offsets(),
column_result);
@@ -69,9 +76,7 @@ public:
block.replace_by_position(result, std::move(column_result));
return Status::OK();
} else {
- return Status::RuntimeError("Illegal column {} of argument of
function {}",
-
block.get_by_position(arguments[0]).column->get_name(),
- get_name());
+ return type_error();
}
block.replace_by_position(result, std::move(column_result));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index a1e7d0b149..77ae343eb8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.MVColumnItem;
import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.TupleDescriptor;
@@ -78,6 +79,7 @@ import org.apache.logging.log4j.Logger;
import java.io.DataOutput;
import java.io.IOException;
import java.io.StringReader;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -369,20 +371,28 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
long baseTabletId = tabletIdMap.get(rollupTabletId);
Map<String, Expr> defineExprs = Maps.newHashMap();
- for (Column column : rollupSchema) {
- if (column.getDefineExpr() != null) {
- defineExprs.put(column.getName(),
column.getDefineExpr());
- }
- }
- List<Column> fullSchema = tbl.getBaseSchema(true);
DescriptorTable descTable = new DescriptorTable();
TupleDescriptor destTupleDesc =
descTable.createTupleDescriptor();
- for (Column column : fullSchema) {
+ Map<String, SlotDescriptor> descMap = Maps.newHashMap();
+ for (Column column : tbl.getFullSchema()) {
SlotDescriptor destSlotDesc =
descTable.addSlotDescriptor(destTupleDesc);
destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());
+
+ descMap.put(column.getName(), destSlotDesc);
+ }
+
+ for (Column column : tbl.getFullSchema()) {
+ if (column.getDefineExpr() != null) {
+ defineExprs.put(column.getName(),
column.getDefineExpr());
+
+ List<SlotRef> slots = new ArrayList<>();
+ column.getDefineExpr().collect(SlotRef.class,
slots);
+ Preconditions.checkArgument(slots.size() == 1);
+
slots.get(0).setDesc(descMap.get(slots.get(0).getColumnName()));
+ }
}
List<Replica> rollupReplicas = rollupTablet.getReplicas();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 5a44022748..28926ac73e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -34,6 +34,8 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
@@ -54,6 +56,8 @@ import java.util.Set;
* [PROPERTIES ("key" = "value")]
*/
public class CreateMaterializedViewStmt extends DdlStmt {
+ private static final Logger LOG =
LogManager.getLogger(CreateMaterializedViewStmt.class);
+
public static final String MATERIALIZED_VIEW_NAME_PREFIX = "mv_";
public static final Map<String, MVColumnPattern> FN_NAME_TO_PATTERN;
@@ -400,9 +404,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
mvColumnName = baseColumnName;
} else {
mvColumnName = mvColumnBuilder(functionName,
baseColumnName);
- if (!functionChild0.getType().isStringType()) {
- functionChild0.uncheckedCastChild(Type.VARCHAR, 0);
- }
defineExpr = functionChild0;
}
mvAggregateType =
AggregateType.valueOf(functionName.toUpperCase());
diff --git a/gensrc/proto/olap_common.proto b/gensrc/proto/olap_common.proto
index 7326b884e7..a452e0ff6a 100644
--- a/gensrc/proto/olap_common.proto
+++ b/gensrc/proto/olap_common.proto
@@ -28,7 +28,7 @@ message ColumnMessage {
required uint32 length = 4; // ColumnPB.length
required bool is_key = 5; // ColumnPB.is_key
optional string default_value = 6; // ColumnPB.default_value
- optional string referenced_column = 7; // ColumnPB.
+ optional string referenced_column = 7; // deprecated
optional uint32 index_length = 8; // ColumnPB.index_length
optional uint32 precision = 9 [default = 27]; // ColumnPB.precision
optional uint32 frac = 10 [default = 9]; // ColumnPB.frac
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 1bea2eb071..14bb5ecb5e 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -193,8 +193,8 @@ message ColumnPB {
optional int32 length = 10; // ColumnMessage.length
optional int32 index_length = 11; // ColumnMessage.index_length
optional bool is_bf_column = 12; // ColumnMessage.is_bf_column
- optional int32 referenced_column_id = 13; //
- optional string referenced_column = 14; // ColumnMessage.referenced_column?
+ optional int32 referenced_column_id = 13; // deprecated
+ optional string referenced_column = 14; // deprecated
optional bool has_bitmap_index = 15 [default=false]; //
ColumnMessage.has_bitmap_index
optional bool visible = 16 [default=true];
repeated ColumnPB children_columns = 17;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
