This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ba3e0b3f96 [feature](compaction) allow to set disable_auto_compaction
for tables (#11743)
ba3e0b3f96 is described below
commit ba3e0b3f9627639da1cf8cbe9d0cd4fa2b611c57
Author: Gabriel <[email protected]>
AuthorDate: Wed Aug 17 11:05:47 2022 +0800
[feature](compaction) allow to set disable_auto_compaction for tables
(#11743)
---
be/src/olap/olap_server.cpp | 8 +++-
be/src/olap/tablet_meta.cpp | 4 ++
be/src/olap/tablet_schema.cpp | 4 ++
be/src/olap/tablet_schema.h | 5 +++
be/test/olap/test_data/header_without_inc_rs.txt | 3 +-
.../Create/CREATE-TABLE.md | 8 ++++
.../Create/CREATE-TABLE.md | 8 ++++
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 4 ++
.../java/org/apache/doris/catalog/OlapTable.java | 16 ++++++++
.../org/apache/doris/catalog/TableProperty.java | 12 ++++++
.../apache/doris/common/util/PropertyAnalyzer.java | 21 ++++++++++
.../apache/doris/datasource/InternalCatalog.java | 27 ++++++++++---
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../org/apache/doris/task/CreateReplicaTask.java | 6 ++-
.../analysis/CreateTableAsSelectStmtTest.java | 47 ++++++++++++++--------
.../java/org/apache/doris/task/AgentTaskTest.java | 8 ++--
gensrc/proto/olap_file.proto | 1 +
gensrc/thrift/AgentService.thrift | 1 +
21 files changed, 161 insertions(+), 34 deletions(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index cb080a0e70..68c24d212a 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -523,11 +523,17 @@ std::vector<TabletSharedPtr>
StorageEngine::_generate_compaction_tasks(
? copied_cumu_map[data_dir]
: copied_base_map[data_dir],
&disk_max_score, _cumulative_compaction_policy);
- if (tablet != nullptr) {
+ if (tablet != nullptr &&
+
!tablet->tablet_meta()->tablet_schema().disable_auto_compaction()) {
if (need_pick_tablet) {
tablets_compaction.emplace_back(tablet);
}
max_compaction_score = std::max(max_compaction_score,
disk_max_score);
+ } else if (tablet != nullptr &&
+
tablet->tablet_meta()->tablet_schema().disable_auto_compaction()) {
+ LOG(INFO) << "Tablet " << tablet->full_name()
+ << " will be ignored by automatic compaction tasks
since it's set to "
+ << "disabled automatic compaction.";
}
}
}
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index b2dd33b281..786fe2674e 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -183,6 +183,10 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
schema->set_is_in_memory(tablet_schema.is_in_memory);
}
+ if (tablet_schema.__isset.disable_auto_compaction) {
+
schema->set_disable_auto_compaction(tablet_schema.disable_auto_compaction);
+ }
+
if (tablet_schema.__isset.delete_sign_idx) {
schema->set_delete_sign_idx(tablet_schema.delete_sign_idx);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index d3668c838a..e691cc2a4f 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -514,6 +514,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema) {
_bf_fpp = BLOOM_FILTER_DEFAULT_FPP;
}
_is_in_memory = schema.is_in_memory();
+ _disable_auto_compaction = schema.disable_auto_compaction();
_delete_sign_idx = schema.delete_sign_idx();
_sequence_col_idx = schema.sequence_col_idx();
_sort_type = schema.sort_type();
@@ -546,6 +547,7 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
// todo(yixiu): unique_id
_next_column_unique_id = ori_tablet_schema.next_column_unique_id();
_is_in_memory = ori_tablet_schema.is_in_memory();
+ _disable_auto_compaction = ori_tablet_schema.disable_auto_compaction();
_delete_sign_idx = ori_tablet_schema.delete_sign_idx();
_sequence_col_idx = ori_tablet_schema.sequence_col_idx();
_sort_type = ori_tablet_schema.sort_type();
@@ -602,6 +604,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
}
tablet_schema_pb->set_next_column_unique_id(_next_column_unique_id);
tablet_schema_pb->set_is_in_memory(_is_in_memory);
+ tablet_schema_pb->set_disable_auto_compaction(_disable_auto_compaction);
tablet_schema_pb->set_delete_sign_idx(_delete_sign_idx);
tablet_schema_pb->set_sequence_col_idx(_sequence_col_idx);
tablet_schema_pb->set_sort_type(_sort_type);
@@ -730,6 +733,7 @@ bool operator==(const TabletSchema& a, const TabletSchema&
b) {
}
if (a._is_in_memory != b._is_in_memory) return false;
if (a._delete_sign_idx != b._delete_sign_idx) return false;
+ if (a._disable_auto_compaction != b._disable_auto_compaction) return false;
return true;
}
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index cea9a46bb8..8271b2eafd 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -149,6 +149,10 @@ public:
double bloom_filter_fpp() const { return _bf_fpp; }
bool is_in_memory() const { return _is_in_memory; }
void set_is_in_memory(bool is_in_memory) { _is_in_memory = is_in_memory; }
+ void set_disable_auto_compaction(bool disable_auto_compaction) {
+ _disable_auto_compaction = disable_auto_compaction;
+ }
+ bool disable_auto_compaction() const { return _disable_auto_compaction; }
int32_t delete_sign_idx() const { return _delete_sign_idx; }
void set_delete_sign_idx(int32_t delete_sign_idx) { _delete_sign_idx =
delete_sign_idx; }
bool has_sequence_col() const { return _sequence_col_idx != -1; }
@@ -195,6 +199,7 @@ private:
int32_t _delete_sign_idx = -1;
int32_t _sequence_col_idx = -1;
int32_t _schema_version = -1;
+ bool _disable_auto_compaction = false;
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
diff --git a/be/test/olap/test_data/header_without_inc_rs.txt
b/be/test/olap/test_data/header_without_inc_rs.txt
index d927865fae..fc2a5cb1e6 100644
--- a/be/test/olap/test_data/header_without_inc_rs.txt
+++ b/be/test/olap/test_data/header_without_inc_rs.txt
@@ -55,7 +55,8 @@
"sort_type": "LEXICAL",
"sort_col_num": 0,
"compression_type": "LZ4F",
- "schema_version": 0
+ "schema_version": 0,
+ "disable_auto_compaction": false
},
"rs_metas": [
{
diff --git
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
index 0e99f10128..2b7dcbe51b 100644
---
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
+++
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -318,6 +318,14 @@ distribution_info
`"light_schema_change"="true"`
+ * `disable_auto_compaction`
+
+ Whether to disable automatic compaction for this table.
+
+ If this property is set to 'true', the background automatic compaction
process will skip all the tables of this table.
+
+ `"disable_auto_compaction" = "false"`
+
* Dynamic partition related
The relevant parameters of dynamic partition are as follows:
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
index 362bedbf2c..0086b4c631 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -318,6 +318,14 @@ distribution_info
Doris默认不使用light schema change优化。如果想使用该优化需要指定为true。
`"light_schema_change" = 'true'`
+
+ * `disable_auto_compaction`
+
+ 是否对这个表禁用自动compaction。
+
+ 如果这个属性设置成 `true`, 后台的自动compaction进程会跳过这个表的所有tablet。
+
+ `"disable_auto_compaction" = "false"`
* 动态分区相关
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index baea44c646..8d75cd774d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -245,7 +245,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
tabletType,
null,
tbl.getCompressionType(),
- tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.getStoragePolicy());
+ tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.getStoragePolicy(),
+ tbl.disableAutoCompaction());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 3793a45ea1..605e335153 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -265,7 +265,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.getPartitionInfo().getTabletType(partitionId),
null,
tbl.getCompressionType(),
- tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.getStoragePolicy());
+ tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.getStoragePolicy(),
+ tbl.disableAutoCompaction());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index d3cac57003..bcd06071e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -970,7 +970,8 @@ public class RestoreJob extends AbstractJob {
localTbl.getPartitionInfo().getTabletType(restorePart.getId()),
null,
localTbl.getCompressionType(),
- localTbl.getEnableUniqueKeyMergeOnWrite(),
localTbl.getStoragePolicy());
+ localTbl.getEnableUniqueKeyMergeOnWrite(),
localTbl.getStoragePolicy(),
+ localTbl.disableAutoCompaction());
task.setInRestoreMode(true);
batchTask.addTask(task);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index cb1646fa27..53ddeab48d 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -2915,6 +2915,10 @@ public class Env {
sb.append(olapTable.getSequenceType().toString()).append("\"");
}
+ // disable auto compaction
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).append("\"
= \"");
+ sb.append(olapTable.disableAutoCompaction()).append("\"");
+
sb.append("\n)");
} else if (table.getType() == TableType.MYSQL) {
MysqlTable mysqlTable = (MysqlTable) table;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 8143cbb359..dff5a790e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1582,6 +1582,22 @@ public class OlapTable extends Table {
return "";
}
+ public void setDisableAutoCompaction(boolean disableAutoCompaction) {
+ if (tableProperty == null) {
+ tableProperty = new TableProperty(new HashMap<>());
+ }
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION,
+ Boolean.valueOf(disableAutoCompaction).toString());
+ tableProperty.buildDisableAutoCompaction();
+ }
+
+ public Boolean disableAutoCompaction() {
+ if (tableProperty != null) {
+ return tableProperty.disableAutoCompaction();
+ }
+ return false;
+ }
+
public void setDataSortInfo(DataSortInfo dataSortInfo) {
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index a355248cd3..8ff2962686 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -75,6 +75,8 @@ public class TableProperty implements Writable {
private boolean enableLightSchemaChange = false;
+ private Boolean disableAutoCompaction;
+
private DataSortInfo dataSortInfo = new DataSortInfo();
// remote storage policy, for cold data
@@ -152,6 +154,16 @@ public class TableProperty implements Writable {
return this;
}
+ public TableProperty buildDisableAutoCompaction() {
+ disableAutoCompaction = Boolean.parseBoolean(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION,
"false"));
+ return this;
+ }
+
+ public boolean disableAutoCompaction() {
+ return disableAutoCompaction;
+ }
+
public TableProperty buildStoragePolicy() {
storagePolicy =
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, "");
return this;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 37cbf8e9c9..52631edd33 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -114,6 +114,8 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_STORAGE_POLICY = "storage_policy";
+ public static final String PROPERTIES_DISABLE_AUTO_COMPACTION =
"disable_auto_compaction";
+
private static final Logger LOG =
LogManager.getLogger(PropertyAnalyzer.class);
private static final String COMMA_SEPARATOR = ",";
private static final double MAX_FPP = 0.05;
@@ -472,6 +474,25 @@ public class PropertyAnalyzer {
+ " must be `true` or `false`");
}
+ public static Boolean analyzeDisableAutoCompaction(Map<String, String>
properties) throws AnalysisException {
+ if (properties == null || properties.isEmpty()) {
+ return false;
+ }
+ String value = properties.get(PROPERTIES_DISABLE_AUTO_COMPACTION);
+ // set light schema change false by default
+ if (null == value) {
+ return false;
+ }
+ properties.remove(PROPERTIES_DISABLE_AUTO_COMPACTION);
+ if (value.equalsIgnoreCase("true")) {
+ return true;
+ } else if (value.equalsIgnoreCase("false")) {
+ return false;
+ }
+ throw new AnalysisException(PROPERTIES_DISABLE_AUTO_COMPACTION
+ + " must be `true` or `false`");
+ }
+
// analyzeCompressionType will parse the compression type from properties
public static TCompressionType analyzeCompressionType(Map<String, String>
properties) throws AnalysisException {
String compressionType = "";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 433fcd2054..7a4c75faab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1247,6 +1247,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY))
{
properties.put(PropertyAnalyzer.PROPERTIES_INMEMORY,
olapTable.isInMemory().toString());
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)) {
+
properties.put(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION,
+ olapTable.disableAutoCompaction().toString());
+ }
singlePartitionDesc.analyze(partitionInfo.getPartitionColumns().size(),
properties);
partitionInfo.createAndCheckPartitionItem(singlePartitionDesc,
isTempPartition);
@@ -1325,7 +1329,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
singlePartitionDesc.getVersionInfo(), bfColumns,
olapTable.getBfFpp(), tabletIdSet,
olapTable.getCopiedIndexes(),
singlePartitionDesc.isInMemory(), olapTable.getStorageFormat(),
singlePartitionDesc.getTabletType(),
olapTable.getCompressionType(), olapTable.getDataSortInfo(),
- olapTable.getEnableUniqueKeyMergeOnWrite(),
olapTable.getStoragePolicy(), idGeneratorBuffer);
+ olapTable.getEnableUniqueKeyMergeOnWrite(),
olapTable.getStoragePolicy(), idGeneratorBuffer,
+ olapTable.disableAutoCompaction());
// check again
table = db.getOlapTableOrDdlException(tableName);
@@ -1547,7 +1552,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
Long versionInfo, Set<String> bfColumns, double bfFpp, Set<Long>
tabletIdSet, List<Index> indexes,
boolean isInMemory, TStorageFormat storageFormat, TTabletType
tabletType, TCompressionType compressionType,
DataSortInfo dataSortInfo, boolean enableUniqueKeyMergeOnWrite,
String storagePolicy,
- IdGeneratorBuffer idGeneratorBuffer) throws DdlException {
+ IdGeneratorBuffer idGeneratorBuffer, boolean
disableAutoCompaction) throws DdlException {
// create base index first.
Preconditions.checkArgument(baseIndexId != -1);
MaterializedIndex baseIndex = new MaterializedIndex(baseIndexId,
IndexState.NORMAL);
@@ -1607,7 +1612,8 @@ public class InternalCatalog implements
CatalogIf<Database> {
CreateReplicaTask task = new CreateReplicaTask(backendId,
dbId, tableId, partitionId, indexId,
tabletId, replicaId, shortKeyColumnCount,
schemaHash, version, keysType, storageType,
storageMedium, schema, bfColumns, bfFpp,
countDownLatch, indexes, isInMemory, tabletType,
- dataSortInfo, compressionType,
enableUniqueKeyMergeOnWrite, storagePolicy);
+ dataSortInfo, compressionType,
enableUniqueKeyMergeOnWrite, storagePolicy,
+ disableAutoCompaction);
task.setStorageFormat(storageFormat);
batchTask.addTask(task);
@@ -1736,6 +1742,15 @@ public class InternalCatalog implements
CatalogIf<Database> {
// use light schema change optimization
olapTable.setEnableLightSchemaChange(enableLightSchemaChange);
+ boolean disableAutoCompaction = false;
+ try {
+ disableAutoCompaction =
PropertyAnalyzer.analyzeDisableAutoCompaction(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ // use light schema change optimization
+ olapTable.setDisableAutoCompaction(disableAutoCompaction);
+
// get storage format
TStorageFormat storageFormat = TStorageFormat.V2; // default is
segment v2
try {
@@ -1950,7 +1965,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
partitionInfo.getReplicaAllocation(partitionId),
versionInfo, bfColumns, bfFpp, tabletIdSet,
olapTable.getCopiedIndexes(), isInMemory,
storageFormat, tabletType, compressionType,
olapTable.getDataSortInfo(),
olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy,
- idGeneratorBuffer);
+ idGeneratorBuffer, olapTable.disableAutoCompaction());
olapTable.addPartition(partition);
} else if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
@@ -2008,7 +2023,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
tabletIdSet, olapTable.getCopiedIndexes(),
isInMemory, storageFormat,
partitionInfo.getTabletType(entry.getValue()),
compressionType,
olapTable.getDataSortInfo(),
olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy,
- idGeneratorBuffer);
+ idGeneratorBuffer,
olapTable.disableAutoCompaction());
olapTable.addPartition(partition);
}
} else {
@@ -2385,7 +2400,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
copiedTbl.isInMemory(), copiedTbl.getStorageFormat(),
copiedTbl.getPartitionInfo().getTabletType(oldPartitionId),
copiedTbl.getCompressionType(),
copiedTbl.getDataSortInfo(),
copiedTbl.getEnableUniqueKeyMergeOnWrite(),
- olapTable.getStoragePolicy(), idGeneratorBuffer);
+ olapTable.getStoragePolicy(), idGeneratorBuffer,
olapTable.disableAutoCompaction());
newPartitions.add(newPartition);
}
} catch (DdlException e) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 91ebddf0f8..c8e03e4d6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -607,7 +607,8 @@ public class ReportHandler extends Daemon {
olapTable.getPartitionInfo().getTabletType(partitionId),
null,
olapTable.getCompressionType(),
-
olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getStoragePolicy());
+
olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getStoragePolicy(),
+ olapTable.disableAutoCompaction());
createReplicaTask.setIsRecoverTask(true);
createReplicaBatchTask.addTask(createReplicaTask);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 9b71369276..2da8f6dda3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -92,6 +92,8 @@ public class CreateReplicaTask extends AgentTask {
private boolean enableUniqueKeyMergeOnWrite;
+ private boolean disableAutoCompaction;
+
public CreateReplicaTask(long backendId, long dbId, long tableId, long
partitionId, long indexId, long tabletId,
long replicaId, short shortKeyColumnCount, int
schemaHash, long version,
KeysType keysType, TStorageType storageType,
@@ -103,7 +105,7 @@ public class CreateReplicaTask extends AgentTask {
DataSortInfo dataSortInfo,
TCompressionType compressionType,
boolean enableUniqueKeyMergeOnWrite,
- String storagePolicy) {
+ String storagePolicy, boolean
disableAutoCompaction) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
this.replicaId = replicaId;
@@ -130,6 +132,7 @@ public class CreateReplicaTask extends AgentTask {
this.dataSortInfo = dataSortInfo;
this.enableUniqueKeyMergeOnWrite = (keysType == KeysType.UNIQUE_KEYS
&& enableUniqueKeyMergeOnWrite);
this.storagePolicy = storagePolicy;
+ this.disableAutoCompaction = disableAutoCompaction;
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -228,6 +231,7 @@ public class CreateReplicaTask extends AgentTask {
tSchema.setBloomFilterFpp(bfFpp);
}
tSchema.setIsInMemory(isInMemory);
+ tSchema.setDisableAutoCompaction(disableAutoCompaction);
createTabletReq.setTabletSchema(tSchema);
createTabletReq.setVersion(version);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index bb6dcc1497..75355d6c69 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -40,7 +40,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ ") ENGINE = OLAP unique KEY(`userId`)\n" + "COMMENT
'varchar_table'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES
(\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")";
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")";
createTable(varcharTable);
String decimalTable = "CREATE TABLE `test`.`decimal_table`\n" + "(\n"
+ " `userId` varchar(255) NOT NULL,\n" + "
`amount_decimal` decimal(10, 2) NOT NULL\n"
@@ -67,7 +67,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\","
+ + "\n\"disable_auto_compaction\" = \"false\"\n" + ")",
showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1));
String selectFromDecimal1 =
"create table `test`.`select_decimal_table_1`
PROPERTIES(\"replication_num\" = \"1\") "
@@ -78,14 +79,16 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
"CREATE TABLE `select_decimal_table_1` (\n" + " `_col0`
decimal(38, 2) NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\","
+ + "\n\"disable_auto_compaction\" = \"false\"\n" +
")",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
} else {
Assertions.assertEquals(
"CREATE TABLE `select_decimal_table_1` (\n" + " `_col0`
decimal(27, 9) NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\","
+ + "\n\"disable_auto_compaction\" = \"false\"\n" +
")",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
}
}
@@ -110,7 +113,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ " `username` varchar(255) REPLACE NOT NULL\n" + ")
ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS
10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet.getResultRows().get(0).get(1));
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")",
+ showResultSet.getResultRows().get(0).get(1));
}
@Test
@@ -123,7 +127,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
"CREATE TABLE `select_function_1` (\n" + " `_col0` bigint(20)
NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n" +
"DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\""
+ + ",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
showResultSet1.getResultRows().get(0).get(1));
String selectFromFunction2 = "create table `test`.`select_function_2`
PROPERTIES(\"replication_num\" = \"1\") "
@@ -138,7 +143,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "DUPLICATE KEY(`_col0`, `_col1`, `_col2`)\n" +
"COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" +
"PROPERTIES (\n"
+ "\"replication_allocation\" =
\"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet2.getResultRows().get(0).get(1));
+ + "\"storage_format\" =
\"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ showResultSet2.getResultRows().get(0).get(1));
}
@Test
@@ -151,7 +157,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "DUPLICATE KEY(`amount`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\","
+ + "\n\"disable_auto_compaction\" = \"false\"\n" + ")",
showResultSet1.getResultRows().get(0).get(1));
String selectAlias2 = "create table `test`.`select_alias_2`
PROPERTIES(\"replication_num\" = \"1\") "
+ "as select userId as alias_name, username from
`test`.`varchar_table`";
@@ -161,7 +168,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ " `username` varchar(255) REPLACE NOT NULL\n" + ")
ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`alias_name`)
BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet2.getResultRows().get(0).get(1));
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")",
+ showResultSet2.getResultRows().get(0).get(1));
}
@Test
@@ -176,7 +184,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT
'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES
(\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet.getResultRows().get(0).get(1));
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")",
+ showResultSet.getResultRows().get(0).get(1));
String selectFromJoin1 = "create table `test`.`select_join1`
PROPERTIES(\"replication_num\" = \"1\") "
+ "as select vt.userId as userId1, jt.userId as userId2,
vt.username, jt.status "
+ "from `test`.`varchar_table` vt " + "join
`test`.`join_table` jt on vt.userId=jt.userId";
@@ -187,7 +196,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ " `status` int(11) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n"
+ "AGGREGATE KEY(`userId1`, `userId2`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId1`) BUCKETS
10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet1.getResultRows().get(0).get(1));
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")",
+ showResultSet1.getResultRows().get(0).get(1));
}
@Test
@@ -203,7 +213,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`user`)\n" + "COMMENT
'OLAP'\n"
+ "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet.getResultRows().get(0).get(1));
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")",
+ showResultSet.getResultRows().get(0).get(1));
}
@Test
@@ -216,7 +227,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\",\n\"disable_auto_compaction\""
+ + " = \"false\"\n" + ")",
showResultSet.getResultRows().get(0).get(1));
}
@@ -231,7 +243,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" +
"PROPERTIES (\n"
+ "\"replication_allocation\" =
\"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet.getResultRows().get(0).get(1));
+ + "\"storage_format\" =
\"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ showResultSet.getResultRows().get(0).get(1));
String selectFromCteAndUnion = "create table `test`.`select_cte_union`
PROPERTIES(\"replication_num\" = \"1\")"
+ "as with source_data as (select 1 as id union all select 2
as id) select * from source_data;";
createTableAsSelect(selectFromCteAndUnion);
@@ -240,7 +253,8 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
"CREATE TABLE `select_cte_union` (\n" + " `id` tinyint(4)
NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`id`)\n" + "COMMENT 'OLAP'\n" +
"DISTRIBUTED BY HASH(`id`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" =
\"tag.location.default: 1\",\n"
- + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\"\n" + ")",
+ + "\"in_memory\" = \"false\",\n" + "\"storage_format\"
= \"V2\","
+ + "\n\"disable_auto_compaction\" = \"false\"\n" + ")",
showResultSet1.getResultRows().get(0).get(1));
}
@@ -257,6 +271,7 @@ public class CreateTableAsSelectStmtTest extends
TestWithFeService {
+ "(PARTITION p1 VALUES IN (\"CA\",\"GB\",\"US\",\"ZH\"))\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES
(\n"
+ "\"replication_allocation\" = \"tag.location.default:
1\",\n" + "\"in_memory\" = \"false\",\n"
- + "\"storage_format\" = \"V2\"\n" + ")",
showResultSet.getResultRows().get(0).get(1));
+ + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" =
\"false\"\n" + ")",
+ showResultSet.getResultRows().get(0).get(1));
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index 4ec82e30a8..f951853d01 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -105,11 +105,9 @@ public class AgentTaskTest {
// create
createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId,
partitionId,
- indexId1, tabletId1,
replicaId1, shortKeyNum, schemaHash1,
- version, KeysType.AGG_KEYS,
- storageType,
TStorageMedium.SSD,
- columns, null, 0, latch,
null,
- false,
TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "");
+ indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
+ TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
+ TCompressionType.LZ4F, false, "", false);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 19bc56823a..020be00ead 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -221,6 +221,7 @@ message TabletSchemaPB {
optional int32 sort_col_num = 12;
optional segment_v2.CompressionTypePB compression_type = 13 [default=LZ4F];
optional int32 schema_version = 14;
+ optional bool disable_auto_compaction = 15 [default=false];
}
enum TabletStatePB {
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 156b3e70a7..d3192f082b 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -38,6 +38,7 @@ struct TTabletSchema {
10: optional i32 sequence_col_idx = -1
11: optional Types.TSortType sort_type
12: optional i32 sort_col_num
+ 13: optional bool disable_auto_compaction
}
// this enum stands for different storage format in src_backends
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
