This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch release-2.0.2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/release-2.0.2.1 by this push:
new 60c8c73ee88 [improvement](meta) allow to ignore unknown image module
(#25450) (#25968)
60c8c73ee88 is described below
commit 60c8c73ee881cb5c46acbae43b795e9662aae742
Author: Mingyu Chen <[email protected]>
AuthorDate: Thu Oct 26 19:54:48 2023 +0800
[improvement](meta) allow to ignore unknown image module (#25450) (#25968)
Add new FE config `ignore_unknown_metadata_module`. Default is false.
If set to true, when reading metadata image file, and there are unknown
modules, these modules
will be ignored and skipped.
This is mainly used in downgrade operation, old version can be compatible
with new version Image file.
---
.../src/main/java/org/apache/doris/common/Config.java | 14 ++++++++++++++
.../java/org/apache/doris/persist/meta/MetaReader.java | 16 +++++++++++++---
.../apache/doris/persist/meta/PersistMetaModules.java | 2 +-
3 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 27c70ee2cad..ba04e21029b 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2161,4 +2161,18 @@ public class Config extends ConfigBase {
"min buckets of auto bucket"
})
public static int autobucket_min_buckets = 1;
+
+ @ConfField(description = {
+ "是否忽略 Image 文件中未知的模块。如果为 true，不在 PersistMetaModules.MODULE_NAMES
中的元数据模块将被忽略并跳过。"
+ + "默认为 false，如果 Image 文件中包含未知的模块，Doris 将会抛出异常。"
+ + "该参数主要用于降级操作中，老版本可以兼容新版本的 Image 文件。",
+ "Whether to ignore unknown modules in Image file. "
+ + "If true, metadata modules not in
PersistMetaModules.MODULE_NAMES "
+ + "will be ignored and skipped. Default is false, if Image
file contains unknown modules, "
+ + "Doris will throw exception. "
+ + "This parameter is mainly used in downgrade operation, "
+ + "old version can be compatible with new version Image
file."
+ })
+ public static boolean ignore_unknown_metadata_module = false;
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
index 62e02bde429..8024105fe26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
@@ -18,6 +18,7 @@
package org.apache.doris.persist.meta;
import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import com.google.common.base.Preconditions;
@@ -101,7 +102,7 @@ public class MetaReader {
}
// skip deprecated modules
if
(PersistMetaModules.DEPRECATED_MODULE_NAMES.contains(metaIndex.name)) {
- LOG.warn("meta modules {} is deprecated, ignore and skip
it");
+ LOG.warn("meta modules {} is deprecated, ignore and skip
it", metaIndex.name);
// If this is the last module, nothing need to do.
if (i < metaFooter.metaIndices.size() - 1) {
IOUtils.skipFully(dis, metaFooter.metaIndices.get(i +
1).offset - metaIndex.offset);
@@ -110,8 +111,17 @@ public class MetaReader {
}
MetaPersistMethod persistMethod =
PersistMetaModules.MODULES_MAP.get(metaIndex.name);
if (persistMethod == null) {
- throw new IOException("Unknown meta module: " +
metaIndex.name + ". Known modules: "
- + PersistMetaModules.MODULE_NAMES);
+ if (Config.ignore_unknown_metadata_module) {
+ LOG.warn("meta modules {} is unknown, ignore and skip
it", metaIndex.name);
+ // If this is the last module, nothing need to do.
+ if (i < metaFooter.metaIndices.size() - 1) {
+ IOUtils.skipFully(dis,
metaFooter.metaIndices.get(i + 1).offset - metaIndex.offset);
+ }
+ continue;
+ } else {
+ throw new IOException("Unknown meta module: " +
metaIndex.name + ". Known modules: "
+ + PersistMetaModules.MODULE_NAMES);
+ }
}
checksum = (long) persistMethod.readMethod.invoke(env, dis,
checksum);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
index 6e99a6757fe..753411e0aff 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
@@ -43,7 +43,7 @@ public class PersistMetaModules {
// Modules in this list is deprecated and will not be saved in meta file.
(also should not be in MODULE_NAMES)
public static final ImmutableList<String> DEPRECATED_MODULE_NAMES =
ImmutableList.of(
- "loadJob", "cooldownJob");
+ "loadJob", "cooldownJob", "AnalysisMgrV2");
static {
MODULES_MAP = Maps.newHashMap();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
