This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new c57cbf9ffa [fix](schema_change) remove shadow prefix of schema for
tablesink (#18822)
c57cbf9ffa is described below
commit c57cbf9ffaeb6d2026d1bc360646add63f89a8d7
Author: Yongqiang YANG <[email protected]>
AuthorDate: Sun Apr 30 22:46:36 2023 +0800
[fix](schema_change) remove shadow prefix of schema for tablesink (#18822)
LSC updates tablet's schema in writing. Be optimized adding columns via
linked schema change and
it distinguishes adding by comparing column name. e.g. if new column's name
is not found in old schema,
then it is a newly-add column.
When a table is under schema-changing, it adds __doris_shadow_ prefix in
name of columns in shadow index.
Then writes during schema-changing would bring schema with __doris_shadow_
to be.
If schema change request arrives at be after writes, then be do it as a
add-column schema change due to
__doris_shadow_ is not in base tablet.
---
be/src/olap/schema_change.cpp | 12 +++++++++---
be/src/olap/tablet.h | 2 ++
.../src/main/java/org/apache/doris/catalog/Column.java | 4 ++++
.../main/java/org/apache/doris/planner/OlapTableSink.java | 2 +-
.../decimalv3/test_agg_keys_schema_change_decimalv3.groovy | 1 +
5 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 12c9ce5853..2ceff6f7c6 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -2284,7 +2284,7 @@ Status SchemaChangeHandler::_parse_request(const
SchemaChangeParams& sc_params,
for (int i = 0, new_schema_size =
new_tablet->tablet_schema()->num_columns();
i < new_schema_size; ++i) {
const TabletColumn& new_column =
new_tablet->tablet_schema()->column(i);
- const string& column_name = new_column.name();
+ const std::string& column_name = new_column.name();
ColumnMapping* column_mapping =
rb_changer->get_mutable_column_mapping(i);
column_mapping->new_column = &new_column;
@@ -2310,6 +2310,11 @@ Status SchemaChangeHandler::_parse_request(const
SchemaChangeParams& sc_params,
continue;
}
+ if (column_name.find("__doris_shadow_") == 0) {
+ // Should delete in the future, just a protection for bug.
+ LOG(INFO) << "a shadow column is encountered " << column_name;
+ return Status::InternalError("failed due to operate on shadow
column");
+ }
// Newly added column go here
column_mapping->ref_column = -1;
@@ -2319,8 +2324,9 @@ Status SchemaChangeHandler::_parse_request(const
SchemaChangeParams& sc_params,
RETURN_IF_ERROR(
_init_column_mapping(column_mapping, new_column,
new_column.default_value()));
- VLOG_TRACE << "A column with default value will be added after schema
changing. "
- << "column=" << column_name << ", default_value=" <<
new_column.default_value();
+ LOG(INFO) << "A column with default value will be added after schema
changing. "
+ << "column=" << column_name << ", default_value=" <<
new_column.default_value()
+ << " to table " << new_tablet->get_table_id();
}
// Check if re-aggregation is needed.
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index adb4fecf66..d34597323a 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -381,6 +381,8 @@ public:
void update_max_version_schema(const TabletSchemaSPtr& tablet_schema);
+ int64_t get_table_id() { return _tablet_meta->table_id(); }
+
private:
Status _init_once_action();
void _print_missed_versions(const std::vector<Version>& missed_versions)
const;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 51486bd74b..03b8c99b24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -227,6 +227,10 @@ public class Column implements Writable,
GsonPostProcessable {
return this.name;
}
+ public String getNonShadowName() {
+ return removeNamePrefix(name);
+ }
+
public String getDisplayName() {
if (defineExpr == null) {
return name;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
index 5a2eddb17a..422d7e2ddd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -209,7 +209,7 @@ public class OlapTableSink extends DataSink {
MaterializedIndexMeta indexMeta = pair.getValue();
List<String> columns = Lists.newArrayList();
List<TColumn> columnsDesc = Lists.newArrayList();
-
columns.addAll(indexMeta.getSchema().stream().map(Column::getName).collect(Collectors.toList()));
+
columns.addAll(indexMeta.getSchema().stream().map(Column::getNonShadowName).collect(Collectors.toList()));
for (Column column : indexMeta.getSchema()) {
TColumn tColumn = column.toThrift();
column.setIndexFlag(tColumn, table);
diff --git
a/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy
b/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy
index 7b3a4bfb6f..3cf7874ab1 100644
---
a/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy
+++
b/regression-test/suites/schema_change_p0/decimalv3/test_agg_keys_schema_change_decimalv3.groovy
@@ -21,6 +21,7 @@ suite("test_agg_keys_schema_change_decimalv3") {
def tbName = "test_agg_keys_schema_change_decimalv3"
def getJobState = { tableName ->
def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE
IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+ logger.info(jobStateResult.toString());
return jobStateResult[0][9]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
