This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d2d4423c88 [feature](schema change) Light schema change support rollup
(#11494)
d2d4423c88 is described below
commit d2d4423c8825b058649b5e91dd0d55aad8fbc2ae
Author: Lei Zhang <1091517...@qq.com>
AuthorDate: Mon Aug 15 21:39:27 2022 +0800
[feature](schema change) Light schema change support rollup (#11494)
1. Move max colUniqueId from OlapTable to IndexMeta.
2. Add updateSlotUniqueId.
---
.../doris/alter/MaterializedViewHandler.java | 41 +++-
.../java/org/apache/doris/alter/RollupJobV2.java | 15 +-
.../apache/doris/alter/SchemaChangeHandler.java | 209 ++++++++++++---------
.../org/apache/doris/alter/SchemaChangeJobV2.java | 23 ++-
.../main/java/org/apache/doris/catalog/Column.java | 1 -
.../main/java/org/apache/doris/catalog/Env.java | 6 +-
.../doris/catalog/MaterializedIndexMeta.java | 33 ++++
.../java/org/apache/doris/catalog/OlapTable.java | 43 ++---
.../org/apache/doris/catalog/TableProperty.java | 14 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 8 +-
.../doris/datasource/InternalDataSource.java | 18 +-
.../org/apache/doris/planner/OlapScanNode.java | 21 +++
.../doris/alter/SchemaChangeHandlerTest.java | 6 +-
13 files changed, 270 insertions(+), 168 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index 504db3aa71..ce0e30f3bf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -504,10 +504,18 @@ public class MaterializedViewHandler extends AlterHandler
{
newMVColumns.add(new Column(olapTable.getSequenceCol()));
}
- // set MV column unique id to Column.COLUMN_UNIQUE_ID_INIT_VALUE
support old unique id rule.
- newMVColumns.stream().forEach(column -> {
- column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE);
- });
+ if (olapTable.getEnableLightSchemaChange()) {
+ int nextColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE + 1;
+ for (Column column : newMVColumns) {
+ column.setUniqueId(nextColUniqueId);
+ nextColUniqueId++;
+ }
+ } else {
+ newMVColumns.stream().forEach(column -> {
+ column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE);
+ });
+ }
+ LOG.debug("lightSchemaChange:{}, newMVColumns:{}",
olapTable.getEnableLightSchemaChange(), newMVColumns);
return newMVColumns;
}
@@ -549,25 +557,27 @@ public class MaterializedViewHandler extends AlterHandler
{
for (Column column : olapTable.getSchemaByIndexId(baseIndexId, true)) {
baseColumnNameToColumn.put(column.getName(), column);
}
+ LOG.debug("baseSchema:{}", olapTable.getSchemaByIndexId(baseIndexId,
true));
if (keysType.isAggregationFamily()) {
int keysNumOfRollup = 0;
for (String columnName : rollupColumnNames) {
- Column oneColumn = baseColumnNameToColumn.get(columnName);
- if (oneColumn == null) {
+ Column baseColumn = baseColumnNameToColumn.get(columnName);
+ if (baseColumn == null) {
throw new DdlException("Column[" + columnName + "] does
not exist");
}
- if (oneColumn.isKey() && meetValue) {
+ if (baseColumn.isKey() && meetValue) {
throw new DdlException("Invalid column order. value should
be after key");
}
- if (oneColumn.isKey()) {
+ if (baseColumn.isKey()) {
keysNumOfRollup += 1;
hasKey = true;
} else {
meetValue = true;
- if (oneColumn.getAggregationType().isReplaceFamily()) {
+ if (baseColumn.getAggregationType().isReplaceFamily()) {
meetReplaceValue = true;
}
}
+ Column oneColumn = new Column(baseColumn);
rollupSchema.add(oneColumn);
}
@@ -696,6 +706,19 @@ public class MaterializedViewHandler extends AlterHandler {
}
}
}
+ if (olapTable.getEnableLightSchemaChange()) {
+ int nextColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE + 1;
+ for (Column column : rollupSchema) {
+ column.setUniqueId(nextColUniqueId);
+ nextColUniqueId++;
+ }
+ } else {
+ rollupSchema.stream().forEach(column -> {
+ column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE);
+ });
+ }
+ LOG.debug("lightSchemaChange:{}, rollupSchema:{}, baseSchema:{}",
+ olapTable.getEnableLightSchemaChange(), rollupSchema,
olapTable.getSchemaByIndexId(baseIndexId, true));
return rollupSchema;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 77ae343eb8..e2cad5aceb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -319,7 +319,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
}
tbl.setIndexMeta(rollupIndexId, rollupIndexName, rollupSchema, 0 /*
init schema version */,
- rollupSchemaHash, rollupShortKeyColumnCount,
TStorageType.COLUMN, rollupKeysType, origStmt);
+ rollupSchemaHash, rollupShortKeyColumnCount,
TStorageType.COLUMN,
+ rollupKeysType, origStmt);
tbl.rebuildFullSchema();
}
@@ -532,6 +533,18 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
}
partition.visualiseShadowIndex(rollupIndexId, false);
}
+ //update max column unique id
+ int maxColUniqueId =
tbl.getIndexMetaByIndexId(rollupIndexId).getMaxColUniqueId();
+ for (Column column :
tbl.getIndexMetaByIndexId(rollupIndexId).getSchema(true)) {
+ if (column.getUniqueId() > maxColUniqueId) {
+ maxColUniqueId = column.getUniqueId();
+ }
+ }
+
+
tbl.getIndexMetaByIndexId(rollupIndexId).setMaxColUniqueId(maxColUniqueId);
+
+ LOG.debug("rollupIndexId:{}, maxColUniqueId:{}, indexIdToSchema:{}",
rollupIndexId, maxColUniqueId,
+ tbl.getIndexIdToSchema(true));
tbl.rebuildFullSchema();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index e2c3d0f8b3..685cda5d02 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -134,12 +134,13 @@ public class SchemaChangeHandler extends AlterHandler {
* @param alterClause
* @param olapTable
* @param indexSchemaMap
- * @param colUniqueIdSupplier for multi add columns clause, we need stash
middle state of maxColUniqueId
+ * @param colUniqueIdSupplierMap for multi add columns clause, we need
stash middle state of maxColUniqueId
* @return true: can light schema change, false: cannot light schema change
* @throws DdlException
*/
private boolean processAddColumn(AddColumnClause alterClause, OlapTable
olapTable,
- Map<Long, LinkedList<Column>> indexSchemaMap, IntSupplier
colUniqueIdSupplier) throws DdlException {
+ Map<Long, LinkedList<Column>> indexSchemaMap,
+ Map<Long, IntSupplier> colUniqueIdSupplierMap) throws DdlException
{
Column column = alterClause.getColumn();
ColumnPosition columnPos = alterClause.getColPos();
String targetIndexName = alterClause.getRollupName();
@@ -156,13 +157,8 @@ public class SchemaChangeHandler extends AlterHandler {
Set<String> newColNameSet = Sets.newHashSet(column.getName());
- //only new table generate ColUniqueId, exist table do not.
- if (olapTable.getMaxColUniqueId() >
Column.COLUMN_UNIQUE_ID_INIT_VALUE) {
- column.setUniqueId(colUniqueIdSupplier.getAsInt());
- }
-
return addColumnInternal(olapTable, column, columnPos, targetIndexId,
baseIndexId, indexSchemaMap,
- newColNameSet, false);
+ newColNameSet, false, colUniqueIdSupplierMap);
}
private void processAddColumn(AddColumnClause alterClause, Table
externalTable, List<Column> newSchema)
@@ -192,13 +188,13 @@ public class SchemaChangeHandler extends AlterHandler {
* @param olapTable
* @param indexSchemaMap
* @param ignoreSameColumn
- * @param colUniqueIdSupplier for multi add columns clause, we need stash
middle state of maxColUniqueId
+ * @param colUniqueIdSupplierMap for multi add columns clause, we need
stash middle state of maxColUniqueId
* @return true: can light schema change, false: cannot light schema change
* @throws DdlException
*/
public boolean processAddColumns(AddColumnsClause alterClause, OlapTable
olapTable,
- Map<Long, LinkedList<Column>> indexSchemaMap, boolean
ignoreSameColumn, IntSupplier colUniqueIdSupplier)
- throws DdlException {
+ Map<Long, LinkedList<Column>> indexSchemaMap, boolean
ignoreSameColumn,
+ Map<Long, IntSupplier> colUniqueIdSupplierMap) throws DdlException
{
List<Column> columns = alterClause.getColumns();
String targetIndexName = alterClause.getRollupName();
checkIndexExists(olapTable, targetIndexName);
@@ -217,22 +213,15 @@ public class SchemaChangeHandler extends AlterHandler {
targetIndexId = olapTable.getIndexIdByName(targetIndexName);
}
- //for new table calculate column unique id
- if (olapTable.getMaxColUniqueId() >
Column.COLUMN_UNIQUE_ID_INIT_VALUE) {
- for (Column column : columns) {
- column.setUniqueId(colUniqueIdSupplier.getAsInt());
- }
- }
-
- boolean ligthSchemaChange = true;
+ boolean lightSchemaChange = true;
for (Column column : columns) {
boolean result = addColumnInternal(olapTable, column, null,
targetIndexId, baseIndexId, indexSchemaMap,
- newColNameSet, ignoreSameColumn);
+ newColNameSet, ignoreSameColumn, colUniqueIdSupplierMap);
if (!result) {
- ligthSchemaChange = false;
+ lightSchemaChange = false;
}
}
- return ligthSchemaChange;
+ return lightSchemaChange;
}
private void processDropColumn(DropColumnClause alterClause, Table
externalTable, List<Column> newSchema)
@@ -273,19 +262,20 @@ public class SchemaChangeHandler extends AlterHandler {
private boolean processDropColumn(DropColumnClause alterClause, OlapTable
olapTable,
Map<Long, LinkedList<Column>> indexSchemaMap, List<Index> indexes)
throws DdlException {
- boolean ligthSchemaChange = false;
- if (olapTable.getMaxColUniqueId() >
Column.COLUMN_UNIQUE_ID_INIT_VALUE) {
- //assume can light schema change.
- ligthSchemaChange = true;
- }
-
String dropColName = alterClause.getColName();
String targetIndexName = alterClause.getRollupName();
checkIndexExists(olapTable, targetIndexName);
String baseIndexName = olapTable.getName();
checkAssignedTargetIndexName(baseIndexName, targetIndexName);
+ long baseIndexId = olapTable.getBaseIndexId();
+ long targetIndexId = -1L;
+ if (targetIndexName != null) {
+ targetIndexId = olapTable.getIndexIdByName(targetIndexName);
+ }
+
+ boolean lightSchemaChange = olapTable.getEnableLightSchemaChange();
/*
* UNIQUE:
* Can not drop any key column.
@@ -293,12 +283,11 @@ public class SchemaChangeHandler extends AlterHandler {
* Can not drp any key column is has value with REPLACE method
*/
if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
- long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
boolean isKey = false;
for (Column column : baseSchema) {
if (column.isKey() &&
column.getName().equalsIgnoreCase(dropColName)) {
- ligthSchemaChange = false;
+ lightSchemaChange = false;
isKey = true;
break;
}
@@ -311,14 +300,13 @@ public class SchemaChangeHandler extends AlterHandler {
} else if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
if (null == targetIndexName) {
// drop column in base table
- long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
boolean isKey = false;
boolean hasReplaceColumn = false;
for (Column column : baseSchema) {
if (column.isKey() &&
column.getName().equalsIgnoreCase(dropColName)) {
isKey = true;
- ligthSchemaChange = false;
+ lightSchemaChange = false;
} else if (AggregateType.REPLACE ==
column.getAggregationType()
|| AggregateType.REPLACE_IF_NOT_NULL ==
column.getAggregationType()) {
hasReplaceColumn = true;
@@ -330,7 +318,6 @@ public class SchemaChangeHandler extends AlterHandler {
}
} else {
// drop column in rollup and base index
- long targetIndexId =
olapTable.getIndexIdByName(targetIndexName);
// find column
List<Column> targetIndexSchema =
indexSchemaMap.get(targetIndexId);
boolean isKey = false;
@@ -338,7 +325,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Column column : targetIndexSchema) {
if (column.isKey() &&
column.getName().equalsIgnoreCase(dropColName)) {
isKey = true;
- ligthSchemaChange = false;
+ lightSchemaChange = false;
} else if (AggregateType.REPLACE ==
column.getAggregationType()
|| AggregateType.REPLACE_IF_NOT_NULL ==
column.getAggregationType()) {
hasReplaceColumn = true;
@@ -350,11 +337,10 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
} else if (KeysType.DUP_KEYS == olapTable.getKeysType()) {
- long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
for (Column column : baseSchema) {
if (column.isKey() &&
column.getName().equalsIgnoreCase(dropColName)) {
- ligthSchemaChange = false;
+ lightSchemaChange = false;
break;
}
}
@@ -371,7 +357,6 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
- long baseIndexId = olapTable.getBaseIndexId();
if (targetIndexName == null) {
// if not specify rollup index, column should be dropped from both
base and rollup indexes.
List<Long> indexIds = new ArrayList<Long>();
@@ -401,16 +386,13 @@ public class SchemaChangeHandler extends AlterHandler {
while (iter.hasNext()) {
Column column = iter.next();
if (column.getName().equalsIgnoreCase(dropColName)) {
- ligthSchemaChange = false;
iter.remove();
break;
}
}
} // end for index names
} else {
- ligthSchemaChange = false;
// if specify rollup index, only drop column from specified rollup
index
- long targetIndexId = olapTable.getIndexIdByName(targetIndexName);
// find column
List<Column> targetIndexSchema = indexSchemaMap.get(targetIndexId);
boolean found = false;
@@ -420,6 +402,9 @@ public class SchemaChangeHandler extends AlterHandler {
if (column.getName().equalsIgnoreCase(dropColName)) {
iter.remove();
found = true;
+ if (column.isKey()) {
+ lightSchemaChange = false;
+ }
break;
}
}
@@ -427,7 +412,7 @@ public class SchemaChangeHandler extends AlterHandler {
throw new DdlException("Column does not exists: " +
dropColName);
}
}
- return ligthSchemaChange;
+ return lightSchemaChange;
}
// User can modify column type and column position
@@ -837,24 +822,19 @@ public class SchemaChangeHandler extends AlterHandler {
* @param indexSchemaMap Modified schema will be saved in 'indexSchemaMap'
* @param newColNameSet
* @param ignoreSameColumn
+ * @param colUniqueIdSupplierMap
* @return true: can light schema change, false: cannot
* @throws DdlException
*/
private boolean addColumnInternal(OlapTable olapTable, Column newColumn,
ColumnPosition columnPos,
long targetIndexId, long baseIndexId, Map<Long,
LinkedList<Column>> indexSchemaMap,
- Set<String> newColNameSet, boolean ignoreSameColumn) throws
DdlException {
+ Set<String> newColNameSet, boolean ignoreSameColumn,
+ Map<Long, IntSupplier> colUniqueIdSupplierMap) throws DdlException
{
//only new table generate ColUniqueId, exist table do not.
- boolean ligthSchemaChange = olapTable.getMaxColUniqueId() >
Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+ boolean lightSchemaChange = olapTable.getEnableLightSchemaChange();
String newColName = newColumn.getName();
- //make sure olapTable has locked
- if (newColumn.getUniqueId() > Integer.MAX_VALUE) {
- throw new DdlException("schema change add column times overflow: "
+ newColName);
- }
- LOG.debug("table: {}, newColumn: {}, uniqueId: {}",
olapTable.getName(), newColumn.getName(),
- newColumn.getUniqueId());
-
// check the validation of aggregation method on column.
// also fill the default aggregation method if not specified.
if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
@@ -909,7 +889,8 @@ public class SchemaChangeHandler extends AlterHandler {
//type key column do not allow light schema change.
if (newColumn.isKey()) {
- ligthSchemaChange = false;
+ LOG.debug("newColumn: {}, isKey()==true", newColumn);
+ lightSchemaChange = false;
}
// check if the new column already exist in base schema.
@@ -959,62 +940,94 @@ public class SchemaChangeHandler extends AlterHandler {
for (Map.Entry<Long, LinkedList<Column>> entry :
indexSchemaMap.entrySet()) {
modIndexSchema = entry.getValue();
boolean isBaseIdex = entry.getKey() == baseIndexId;
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, isBaseIdex);
+ IntSupplier colUniqueIdSupplier =
colUniqueIdSupplierMap.get(entry.getKey());
+ int newColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? colUniqueIdSupplier.getAsInt() :
Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ isBaseIdex, newColumnUniqueId);
}
} else {
// 1. add to base table
modIndexSchema = indexSchemaMap.get(baseIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, true);
+ IntSupplier baseIndexColUniqueIdSupplier =
colUniqueIdSupplierMap.get(baseIndexId);
+ int baseIndexNewColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? baseIndexColUniqueIdSupplier.getAsInt()
+ : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ true, baseIndexNewColumnUniqueId);
if (targetIndexId == -1L) {
- return ligthSchemaChange;
+ return lightSchemaChange;
}
// 2. add to rollup
- ligthSchemaChange = false;
modIndexSchema = indexSchemaMap.get(targetIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, false);
+ IntSupplier targetIndexColUniqueIdSupplier =
colUniqueIdSupplierMap.get(targetIndexId);
+ int rollUpNewColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? targetIndexColUniqueIdSupplier.getAsInt()
+ : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ false, rollUpNewColumnUniqueId);
}
} else if (KeysType.DUP_KEYS == olapTable.getKeysType()) {
+ //get baseIndexColUniqueIdSupplier
+ IntSupplier baseIndexColUniqueIdSupplier =
colUniqueIdSupplierMap.get(baseIndexId);
+ int baseIndexNewColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? baseIndexColUniqueIdSupplier.getAsInt()
+ : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+
if (targetIndexId == -1L) {
// add to base index
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, true);
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ true, baseIndexNewColumnUniqueId);
// no specified target index. return
- return ligthSchemaChange;
+ return lightSchemaChange;
} else {
// add to rollup index
- ligthSchemaChange = false;
List<Column> modIndexSchema =
indexSchemaMap.get(targetIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, false);
+ IntSupplier targetIndexColUniqueIdSupplier =
colUniqueIdSupplierMap.get(targetIndexId);
+ int rollUpNewColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? targetIndexColUniqueIdSupplier.getAsInt() :
Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ false, rollUpNewColumnUniqueId);
if (newColumn.isKey()) {
/*
* if add column in rollup is key,
* then put the column in base table as the last key column
*/
+
modIndexSchema = indexSchemaMap.get(baseIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, null,
newColNameSet, true);
+ checkAndAddColumn(modIndexSchema, newColumn, null,
newColNameSet,
+ true, baseIndexNewColumnUniqueId);
} else {
modIndexSchema = indexSchemaMap.get(baseIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, true);
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ true, baseIndexNewColumnUniqueId);
}
}
} else {
// check if has default value. this should be done in Analyze phase
// 1. add to base index first
+ IntSupplier baseIndexColUniqueIdSupplier =
colUniqueIdSupplierMap.get(baseIndexId);
+ int baseIndexNewColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? baseIndexColUniqueIdSupplier.getAsInt() :
Column.COLUMN_UNIQUE_ID_INIT_VALUE;
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, true);
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet,
+ true, baseIndexNewColumnUniqueId);
if (targetIndexId == -1L) {
// no specified target index. return
- return ligthSchemaChange;
+ return lightSchemaChange;
}
- ligthSchemaChange = false;
// 2. add to rollup index
+ IntSupplier targetIndexColUniqueIdSupplier =
colUniqueIdSupplierMap.get(targetIndexId);
+ int rollUpNewColumnUniqueId =
olapTable.getEnableLightSchemaChange()
+ ? targetIndexColUniqueIdSupplier.getAsInt() :
Column.COLUMN_UNIQUE_ID_INIT_VALUE;
modIndexSchema = indexSchemaMap.get(targetIndexId);
- checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, false);
+ checkAndAddColumn(modIndexSchema, newColumn, columnPos,
newColNameSet, false, rollUpNewColumnUniqueId);
}
- return ligthSchemaChange;
+ return lightSchemaChange;
}
/*
@@ -1027,7 +1040,7 @@ public class SchemaChangeHandler extends AlterHandler {
* So that k1 will be added to base index 'twice', and we just ignore this
repeat adding.
*/
private void checkAndAddColumn(List<Column> modIndexSchema, Column
newColumn, ColumnPosition columnPos,
- Set<String> newColNameSet, boolean isBaseIndex) throws
DdlException {
+ Set<String> newColNameSet, boolean isBaseIndex, int
newColumnUniqueId) throws DdlException {
int posIndex = -1;
int lastVisibleIdx = -1;
String newColName = newColumn.getName();
@@ -1077,6 +1090,7 @@ public class SchemaChangeHandler extends AlterHandler {
hasPos = true;
}
+ newColumn.setUniqueId(newColumnUniqueId);
if (hasPos) {
modIndexSchema.add(posIndex + 1, newColumn);
} else if (newColumn.isKey()) {
@@ -1089,6 +1103,7 @@ public class SchemaChangeHandler extends AlterHandler {
// value
modIndexSchema.add(newColumn);
}
+ LOG.debug("newColumn setUniqueId({}), modIndexSchema:{}",
newColumnUniqueId, modIndexSchema);
}
private void checkIndexExists(OlapTable olapTable, String targetIndexName)
throws DdlException {
@@ -1595,22 +1610,34 @@ public class SchemaChangeHandler extends AlterHandler {
try {
//alterClauses can or cannot light schema change
boolean lightSchemaChange = true;
- //for multi add colmuns clauses
- IntSupplier colUniqueIdSupplier = new IntSupplier() {
- public int pendingMaxColUniqueId =
olapTable.getMaxColUniqueId();
-
- @Override
- public int getAsInt() {
- pendingMaxColUniqueId++;
- return pendingMaxColUniqueId;
- }
- };
// index id -> index schema
Map<Long, LinkedList<Column>> indexSchemaMap = new HashMap<>();
+
+ //for multi add colmuns clauses
+ //index id -> index col_unique_id supplier
+ Map<Long, IntSupplier> colUniqueIdSupplierMap = new HashMap<>();
for (Map.Entry<Long, List<Column>> entry :
olapTable.getIndexIdToSchema(true).entrySet()) {
indexSchemaMap.put(entry.getKey(), new
LinkedList<>(entry.getValue()));
+
+ IntSupplier colUniqueIdSupplier = null;
+ if (olapTable.getEnableLightSchemaChange()) {
+ colUniqueIdSupplier = new IntSupplier() {
+ public int pendingMaxColUniqueId = olapTable
+
.getIndexMetaByIndexId(entry.getKey()).getMaxColUniqueId();
+ public long indexId = entry.getKey();
+ @Override
+ public int getAsInt() {
+ pendingMaxColUniqueId++;
+ LOG.debug("index id:{}, pendingMaxColUniqueId:{}",
indexId, pendingMaxColUniqueId);
+ return pendingMaxColUniqueId;
+ }
+ };
+ }
+ colUniqueIdSupplierMap.put(entry.getKey(),
colUniqueIdSupplier);
}
+ LOG.debug("in process indexSchemaMap:{}", indexSchemaMap);
+
List<Index> newIndexes = olapTable.getCopiedIndexes();
Map<String, String> propertyMap = new HashMap<>();
for (AlterClause alterClause : alterClauses) {
@@ -1682,14 +1709,14 @@ public class SchemaChangeHandler extends AlterHandler {
if (alterClause instanceof AddColumnClause) {
// add column
boolean clauseCanLigthSchemaChange =
processAddColumn((AddColumnClause) alterClause, olapTable,
- indexSchemaMap, colUniqueIdSupplier);
+ indexSchemaMap, colUniqueIdSupplierMap);
if (clauseCanLigthSchemaChange == false) {
lightSchemaChange = false;
}
} else if (alterClause instanceof AddColumnsClause) {
// add columns
boolean clauseCanLigthSchemaChange =
processAddColumns((AddColumnsClause) alterClause, olapTable,
- indexSchemaMap, false, colUniqueIdSupplier);
+ indexSchemaMap, false, colUniqueIdSupplierMap);
if (clauseCanLigthSchemaChange == false) {
lightSchemaChange = false;
}
@@ -1727,8 +1754,8 @@ public class SchemaChangeHandler extends AlterHandler {
}
} // end for alter clauses
- LOG.debug("processAddColumns, table: {}({}), lightSchemaChange:
{}", olapTable.getName(), olapTable.getId(),
- lightSchemaChange);
+ LOG.debug("table: {}({}), lightSchemaChange: {},
indexSchemaMap:{}", olapTable.getName(), olapTable.getId(),
+ lightSchemaChange, indexSchemaMap);
if (lightSchemaChange) {
long jobId = Env.getCurrentEnv().getNextId();
@@ -2233,18 +2260,18 @@ public class SchemaChangeHandler extends AlterHandler {
int currentSchemaVersion = currentIndexMeta.getSchemaVersion();
int newSchemaVersion = currentSchemaVersion + 1;
currentIndexMeta.setSchemaVersion(newSchemaVersion);
- }
- olapTable.setIndexes(indexes);
- olapTable.rebuildFullSchema();
- //update max column unique id
- int maxColUniqueId = olapTable.getMaxColUniqueId();
- for (Column column : indexSchemaMap.get(olapTable.getBaseIndexId())) {
- if (column.getUniqueId() > maxColUniqueId) {
- maxColUniqueId = column.getUniqueId();
+ //update max column unique id
+ int maxColUniqueId = currentIndexMeta.getMaxColUniqueId();
+ for (Column column : indexSchema) {
+ if (column.getUniqueId() > maxColUniqueId) {
+ maxColUniqueId = column.getUniqueId();
+ }
}
+ currentIndexMeta.setMaxColUniqueId(maxColUniqueId);
}
- olapTable.setMaxColUniqueId(maxColUniqueId);
+ olapTable.setIndexes(indexes);
+ olapTable.rebuildFullSchema();
if (!isReplay) {
TableAddOrDropColumnsInfo info = new
TableAddOrDropColumnsInfo(db.getId(), olapTable.getId(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index c0190fcd40..3793a45ea1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -352,7 +352,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion,
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash,
indexShortKeyMap.get(shadowIdxId), TStorageType.COLUMN,
-
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)));
+ tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)));
}
tbl.rebuildFullSchema();
@@ -486,7 +486,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
// and the job will be in RUNNING state forever.
Database db = Env.getCurrentInternalCatalog()
.getDbOrException(dbId, s -> new
AlterCancelException("Database " + s + " does not exist"));
-
OlapTable tbl;
try {
tbl = (OlapTable) db.getTableOrMetaException(tableId,
TableType.OLAP);
@@ -620,6 +619,16 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
long originIdxId = entry.getValue();
String shadowIdxName = tbl.getIndexNameById(shadowIdxId);
String originIdxName = tbl.getIndexNameById(originIdxId);
+ int maxColUniqueId =
tbl.getIndexMetaByIndexId(originIdxId).getMaxColUniqueId();
+ for (Column column : indexSchemaMap.get(shadowIdxId)) {
+ if (column.getUniqueId() > maxColUniqueId) {
+ maxColUniqueId = column.getUniqueId();
+ }
+ }
+
tbl.getIndexMetaByIndexId(shadowIdxId).setMaxColUniqueId(maxColUniqueId);
+ LOG.debug("originIdxId:{}, shadowIdxId:{}, maxColUniqueId:{},
indexSchema:{}",
+ originIdxId, shadowIdxId, maxColUniqueId,
indexSchemaMap.get(shadowIdxId));
+
tbl.deleteIndexInfo(originIdxName);
// the shadow index name is '__doris_shadow_xxx', rename it to
origin name 'xxx'
// this will also remove the prefix of columns
@@ -648,16 +657,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.setStorageFormat(storageFormat);
}
- // update max column unique id
- int maxColUniqueId = tbl.getMaxColUniqueId();
- for (Column column : tbl.getFullSchema()) {
- if (column.getUniqueId() > maxColUniqueId) {
- maxColUniqueId = column.getUniqueId();
- }
- }
- tbl.setMaxColUniqueId(maxColUniqueId);
- LOG.debug("fullSchema:{}, maxColUniqueId:{}", tbl.getFullSchema(),
maxColUniqueId);
-
tbl.setState(OlapTableState.NORMAL);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index b3cd503cc8..632a7b36ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -551,7 +551,6 @@ public class Column implements Writable {
if (StringUtils.isNotBlank(comment)) {
sb.append(" COMMENT '").append(getComment(true)).append("'");
}
-
return sb.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 14b70e894f..4d61db2942 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -2897,9 +2897,9 @@ public class Env {
}
// show lightSchemaChange only when it is set true
- if (olapTable.getUseLightSchemaChange()) {
-
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE).append("\"
= \"");
- sb.append(olapTable.getUseLightSchemaChange()).append("\"");
+ if (olapTable.getEnableLightSchemaChange()) {
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE).append("\"
= \"");
+ sb.append(olapTable.getEnableLightSchemaChange()).append("\"");
}
// storage policy
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
index 8e600c5991..dd8ef85c6c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
@@ -33,6 +33,8 @@ import org.apache.doris.thrift.TStorageType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
@@ -59,6 +61,12 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
private KeysType keysType;
@SerializedName(value = "defineStmt")
private OriginStatement defineStmt;
+ //for light schema change
+ @SerializedName(value = "maxColUniqueId")
+ private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+
+ private static final Logger LOG =
LogManager.getLogger(MaterializedIndexMeta.class);
+
public MaterializedIndexMeta(long indexId, List<Column> schema, int
schemaVersion, int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType
keysType, OriginStatement defineStmt) {
@@ -179,6 +187,9 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
if (indexMeta.keysType != this.keysType) {
return false;
}
+ if (maxColUniqueId != maxColUniqueId) {
+ return false;
+ }
return true;
}
@@ -212,4 +223,26 @@ public class MaterializedIndexMeta implements Writable,
GsonPostProcessable {
}
}
+ //take care: only use when creating MaterializedIndexMeta's schema.
+ public int incAndGetMaxColUniqueId() {
+ this.maxColUniqueId++;
+ return this.maxColUniqueId;
+ }
+
+ public int getMaxColUniqueId() {
+ return this.maxColUniqueId;
+ }
+
+ public void setMaxColUniqueId(int maxColUniqueId) {
+ this.maxColUniqueId = maxColUniqueId;
+ }
+
+ public void initSchemaColumnUniqueId() {
+ maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
+ this.schema.stream().forEach(column -> {
+ column.setUniqueId(incAndGetMaxColUniqueId());
+ LOG.debug("indexId: {}, column:{}, uniqueId:{}",
+ indexId, column, column.getUniqueId());
+ });
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ebae05c885..19466c5dc8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -38,7 +38,6 @@ import org.apache.doris.clone.TabletScheduler;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
-import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.DeepCopy;
@@ -142,8 +141,6 @@ public class OlapTable extends Table {
private TableProperty tableProperty;
- private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
-
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -194,20 +191,6 @@ public class OlapTable extends Table {
return this.tableProperty;
}
- //take care: only use at create olap table.
- public int incAndGetMaxColUniqueId() {
- this.maxColUniqueId++;
- return this.maxColUniqueId;
- }
-
- public int getMaxColUniqueId() {
- return this.maxColUniqueId;
- }
-
- public void setMaxColUniqueId(int maxColUniqueId) {
- this.maxColUniqueId = maxColUniqueId;
- }
-
public boolean dynamicPartitionExists() {
return tableProperty != null
&& tableProperty.getDynamicPartitionProperty() != null
@@ -324,6 +307,7 @@ public class OlapTable extends Table {
MaterializedIndexMeta indexMeta = new MaterializedIndexMeta(indexId,
schema, schemaVersion,
schemaHash, shortKeyColumnCount, storageType, keysType,
origStmt);
+
indexIdToMeta.put(indexId, indexMeta);
indexNameToId.put(indexName, indexId);
}
@@ -1161,7 +1145,6 @@ public class OlapTable extends Table {
}
tempPartitions.write(out);
- out.writeInt(maxColUniqueId);
}
@Override
@@ -1254,9 +1237,6 @@ public class OlapTable extends Table {
}
tempPartitions.unsetPartitionInfo();
- if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_112) {
- maxColUniqueId = in.readInt();
- }
// In the present, the fullSchema could be rebuilt by schema change
while the properties is changed by MV.
// After that, some properties of fullSchema and nameToColumn may be
not same as properties of base columns.
// So, here we need to rebuild the fullSchema to ensure the
correctness of the properties.
@@ -1562,7 +1542,7 @@ public class OlapTable extends Table {
tableProperty.buildInMemory();
}
- public Boolean getUseLightSchemaChange() {
+ public boolean getEnableLightSchemaChange() {
if (tableProperty != null) {
return tableProperty.getUseSchemaLightChange();
}
@@ -1570,13 +1550,13 @@ public class OlapTable extends Table {
return false;
}
- public void setUseLightSchemaChange(boolean useLightSchemaChange) {
+ public void setEnableLightSchemaChange(boolean enableLightSchemaChange) {
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());
}
-
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE,
- Boolean.valueOf(useLightSchemaChange).toString());
- tableProperty.buildUseLightSchemaChange();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE,
+ Boolean.valueOf(enableLightSchemaChange).toString());
+ tableProperty.buildEnableLightSchemaChange();
}
public void setStoragePolicy(String storagePolicy) {
@@ -1879,4 +1859,15 @@ public class OlapTable extends Table {
}
tableProperty.buildReplicaAllocation();
}
+
+ //for light schema change
+ public void initSchemaColumnUniqueId() {
+ if (!getEnableLightSchemaChange()) {
+ return;
+ }
+
+ for (MaterializedIndexMeta indexMeta : indexIdToMeta.values()) {
+ indexMeta.initSchemaColumnUniqueId();
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 3fffca9bee..a355248cd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -73,7 +73,7 @@ public class TableProperty implements Writable {
private TCompressionType compressionType = TCompressionType.LZ4F;
- private Boolean useSchemaLightChange;
+ private boolean enableLightSchemaChange = false;
private DataSortInfo dataSortInfo = new DataSortInfo();
@@ -146,9 +146,9 @@ public class TableProperty implements Writable {
return this;
}
- public TableProperty buildUseLightSchemaChange() {
- useSchemaLightChange = Boolean.parseBoolean(
-
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE,
"false"));
+ public TableProperty buildEnableLightSchemaChange() {
+ enableLightSchemaChange = Boolean.parseBoolean(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE,
"false"));
return this;
}
@@ -261,8 +261,8 @@ public class TableProperty implements Writable {
return compressionType;
}
- public Boolean getUseSchemaLightChange() {
- return useSchemaLightChange;
+ public boolean getUseSchemaLightChange() {
+ return enableLightSchemaChange;
}
public void setEnableUniqueKeyMergeOnWrite(boolean enable) {
@@ -301,7 +301,7 @@ public class TableProperty implements Writable {
.buildRemoteStoragePolicy()
.buildCompressionType()
.buildStoragePolicy()
- .buildUseLightSchemaChange();
+ .buildEnableLightSchemaChange();
if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_105) {
// get replica num from property map and create replica allocation
String repNum =
tableProperty.properties.remove(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 865e4a2c6c..37cbf8e9c9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -79,7 +79,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_TIMEOUT = "timeout";
public static final String PROPERTIES_COMPRESSION = "compression";
- public static final String PROPERTIES_USE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
+ public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE =
"distribution_type";
public static final String PROPERTIES_SEND_CLEAR_ALTER_TASK =
"send_clear_alter_tasks";
@@ -457,18 +457,18 @@ public class PropertyAnalyzer {
if (properties == null || properties.isEmpty()) {
return false;
}
- String value = properties.get(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
+ String value = properties.get(PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE);
// set light schema change false by default
if (null == value) {
return false;
}
- properties.remove(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
+ properties.remove(PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE);
if (value.equalsIgnoreCase("true")) {
return true;
} else if (value.equalsIgnoreCase("false")) {
return false;
}
- throw new AnalysisException(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE
+ throw new AnalysisException(PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE
+ " must be `true` or `false`");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
index 27ceedcc8f..ab9805981d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
@@ -1727,23 +1727,14 @@ public class InternalDataSource implements
DataSourceIf<Database> {
Map<String, String> properties = stmt.getProperties();
// get use light schema change
- Boolean useLightSchemaChange = false;
+ Boolean enableLightSchemaChange = false;
try {
- useLightSchemaChange =
PropertyAnalyzer.analyzeUseLightSchemaChange(properties);
+ enableLightSchemaChange =
PropertyAnalyzer.analyzeUseLightSchemaChange(properties);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
// use light schema change optimization
- olapTable.setUseLightSchemaChange(useLightSchemaChange);
- if (useLightSchemaChange) {
- for (Column column : baseSchema) {
- column.setUniqueId(olapTable.incAndGetMaxColUniqueId());
- LOG.debug("table: {}, newColumn: {}, uniqueId: {}",
olapTable.getName(), column.getName(),
- column.getUniqueId());
- }
- } else {
- LOG.debug("table: {} doesn't use light schema change",
olapTable.getName());
- }
+ olapTable.setEnableLightSchemaChange(enableLightSchemaChange);
// get storage format
TStorageFormat storageFormat = TStorageFormat.V2; // default is
segment v2
@@ -1923,6 +1914,9 @@ public class InternalDataSource implements
DataSourceIf<Database> {
throw new DdlException(e.getMessage());
}
+ olapTable.initSchemaColumnUniqueId();
+ olapTable.rebuildFullSchema();
+
// analyze version info
Long versionInfo = null;
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 1f41caa8b5..5bc2d6834e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -323,6 +323,7 @@ public class OlapScanNode extends ScanNode {
if (update) {
this.selectedIndexId = selectedIndexId;
+ updateSlotUniqueId();
setIsPreAggregation(isPreAggregation, reasonOfDisable);
updateColumnType();
if (LOG.isDebugEnabled()) {
@@ -368,6 +369,25 @@ public class OlapScanNode extends ScanNode {
}
}
+ /**
+ * In some situation, we need use mv col unique id , because mv col unique
and
+ * base col unique id is different.
+ * For example: select count(*) from table (table has a mv named mv1)
+ * if Optimizer deceide use mv1, we need updateSlotUniqueId.
+ */
+ private void updateSlotUniqueId() {
+ if (!olapTable.getEnableLightSchemaChange() || selectedIndexId ==
olapTable.getBaseIndexId()) {
+ return;
+ }
+ MaterializedIndexMeta meta =
olapTable.getIndexMetaByIndexId(selectedIndexId);
+ for (SlotDescriptor slotDescriptor : desc.getSlots()) {
+ Column baseColumn = slotDescriptor.getColumn();
+ Column mvColumn = meta.getColumnByName(baseColumn.getName());
+ slotDescriptor.setColumn(mvColumn);
+ }
+ LOG.debug("updateSlotUniqueId() slots: {}", desc.getSlots());
+ }
+
public OlapTable getOlapTable() {
return olapTable;
}
@@ -696,6 +716,7 @@ public class OlapScanNode extends ScanNode {
}
final RollupSelector rollupSelector = new RollupSelector(analyzer,
desc, olapTable);
selectedIndexId =
rollupSelector.selectBestRollup(selectedPartitionIds, conjuncts,
isPreAggregation);
+ updateSlotUniqueId();
LOG.debug("select best roll up cost: {} ms, best index id: {}",
(System.currentTimeMillis() - start), selectedIndexId);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
index 8a44fd7e96..c90d7ee394 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
@@ -123,6 +123,8 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
Assertions.assertEquals(baseIndexName, tbl.getName());
MaterializedIndexMeta indexMeta =
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId());
Assertions.assertNotNull(indexMeta);
+ //col_unique_id 0-9
+ Assertions.assertEquals(9, indexMeta.getMaxColUniqueId());
} finally {
tbl.readUnlock();
}
@@ -228,7 +230,7 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
Assertions.assertEquals(baseIndexName, tbl.getName());
MaterializedIndexMeta indexMeta =
tbl.getIndexMetaByIndexId(tbl.getBaseIndexId());
Assertions.assertNotNull(indexMeta);
- Assertions.assertEquals(12, tbl.getMaxColUniqueId());
+ Assertions.assertEquals(12, indexMeta.getMaxColUniqueId());
} finally {
tbl.readUnlock();
}
@@ -365,7 +367,7 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
try {
Deencapsulation.invoke(schemaChangeHandler, "addColumnInternal",
olapTable, newColumn, columnPosition,
- new Long(2), new Long(1), Maps.newHashMap(),
Sets.newHashSet(), false);
+ new Long(2), new Long(1), Maps.newHashMap(),
Sets.newHashSet(), false, Maps.newHashMap());
Assert.fail();
} catch (Exception e) {
System.out.println(e.getMessage());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
