This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6912b20dadcc682f7369a60b340413e7fe38dd14
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Thu Feb 2 09:30:25 2023 +0800
[fix](iceberg) fix meta persist bug of iceberg catalog (#16344)
This PR #16082 forgot to update the GsonUtil for Iceberg
Catalog/Database/Table
---
.../doris/catalog/external/IcebergExternalTable.java | 7 ++-----
.../main/java/org/apache/doris/persist/gson/GsonUtils.java | 14 +++++++++++---
.../planner/external/iceberg/IcebergScanProvider.java | 10 +++++-----
3 files changed, 18 insertions(+), 13 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
index 389ba8dc55..7653efcefb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
@@ -29,15 +29,12 @@ import java.util.List;
public class IcebergExternalTable extends ExternalTable {
- IcebergExternalCatalog icebergCatalog;
-
public IcebergExternalTable(long id, String name, String dbName,
IcebergExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
- icebergCatalog = catalog;
}
public String getIcebergCatalogType() {
- return icebergCatalog.getIcebergCatalogType();
+ return ((IcebergExternalCatalog) catalog).getIcebergCatalogType();
}
protected synchronized void makeSureInitialized() {
@@ -49,7 +46,7 @@ public class IcebergExternalTable extends ExternalTable {
@Override
public TTableDescriptor toThrift() {
List<Column> schema = getFullSchema();
- if (icebergCatalog.getIcebergCatalogType().equals("hms")) {
+ if (getIcebergCatalogType().equals("hms")) {
THiveTable tHiveTable = new THiveTable(dbName, name, new
HashMap<>());
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(),
TTableType.HIVE_TABLE, schema.size(), 0,
getName(), dbName);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index f690ded337..f613d6f986 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -43,6 +43,8 @@ import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalDatabase;
import org.apache.doris.catalog.external.HMSExternalTable;
+import org.apache.doris.catalog.external.IcebergExternalDatabase;
+import org.apache.doris.catalog.external.IcebergExternalTable;
import org.apache.doris.catalog.external.JdbcExternalDatabase;
import org.apache.doris.catalog.external.JdbcExternalTable;
import org.apache.doris.datasource.CatalogIf;
@@ -50,6 +52,8 @@ import org.apache.doris.datasource.EsExternalCatalog;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.JdbcExternalCatalog;
+import org.apache.doris.datasource.iceberg.IcebergHMSExternalCatalog;
+import org.apache.doris.datasource.iceberg.IcebergRestExternalCatalog;
import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo;
import org.apache.doris.load.loadv2.SparkLoadJob.SparkLoadJobStateUpdateInfo;
import org.apache.doris.load.sync.SyncJob;
@@ -167,21 +171,25 @@ public class GsonUtils {
.registerSubtype(InternalCatalog.class,
InternalCatalog.class.getSimpleName())
.registerSubtype(HMSExternalCatalog.class,
HMSExternalCatalog.class.getSimpleName())
.registerSubtype(EsExternalCatalog.class,
EsExternalCatalog.class.getSimpleName())
- .registerSubtype(JdbcExternalCatalog.class,
JdbcExternalCatalog.class.getSimpleName());
+ .registerSubtype(JdbcExternalCatalog.class,
JdbcExternalCatalog.class.getSimpleName())
+ .registerSubtype(IcebergHMSExternalCatalog.class,
IcebergHMSExternalCatalog.class.getSimpleName())
+ .registerSubtype(IcebergRestExternalCatalog.class,
IcebergRestExternalCatalog.class.getSimpleName());
private static RuntimeTypeAdapterFactory<DatabaseIf> dbTypeAdapterFactory
= RuntimeTypeAdapterFactory.of(
DatabaseIf.class, "clazz")
.registerSubtype(ExternalDatabase.class,
ExternalDatabase.class.getSimpleName())
.registerSubtype(EsExternalDatabase.class,
EsExternalDatabase.class.getSimpleName())
.registerSubtype(HMSExternalDatabase.class,
HMSExternalDatabase.class.getSimpleName())
- .registerSubtype(JdbcExternalDatabase.class,
JdbcExternalDatabase.class.getSimpleName());
+ .registerSubtype(JdbcExternalDatabase.class,
JdbcExternalDatabase.class.getSimpleName())
+ .registerSubtype(IcebergExternalDatabase.class,
IcebergExternalDatabase.class.getSimpleName());
private static RuntimeTypeAdapterFactory<TableIf> tblTypeAdapterFactory =
RuntimeTypeAdapterFactory.of(
TableIf.class, "clazz")
.registerSubtype(ExternalTable.class,
ExternalTable.class.getSimpleName())
.registerSubtype(EsExternalTable.class,
EsExternalTable.class.getSimpleName())
.registerSubtype(HMSExternalTable.class,
HMSExternalTable.class.getSimpleName())
- .registerSubtype(JdbcExternalTable.class,
JdbcExternalTable.class.getSimpleName());
+ .registerSubtype(JdbcExternalTable.class,
JdbcExternalTable.class.getSimpleName())
+ .registerSubtype(IcebergExternalTable.class,
IcebergExternalTable.class.getSimpleName());
// the builder of GSON instance.
// Add any other adapters if necessary.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
index 550151d2cf..6c4fb85055 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
@@ -170,13 +170,13 @@ public class IcebergScanProvider extends
QueryScanProvider {
List<InputSplit> splits = new ArrayList<>();
int formatVersion = ((BaseTable)
table).operations().current().formatVersion();
for (FileScanTask task : scan.planFiles()) {
- for (FileScanTask spitTask : task.split(128 * 1024 * 1024)) {
- String dataFilePath = spitTask.file().path().toString();
- IcebergSplit split = new IcebergSplit(new Path(dataFilePath),
spitTask.start(),
- spitTask.length(), new String[0]);
+ for (FileScanTask splitTask : task.split(128 * 1024 * 1024)) {
+ String dataFilePath = splitTask.file().path().toString();
+ IcebergSplit split = new IcebergSplit(new Path(dataFilePath),
splitTask.start(),
+ splitTask.length(), new String[0]);
split.setFormatVersion(formatVersion);
if (formatVersion >= MIN_DELETE_FILE_SUPPORT_VERSION) {
- split.setDeleteFileFilters(getDeleteFileFilters(spitTask));
+
split.setDeleteFileFilters(getDeleteFileFilters(splitTask));
}
split.setTableFormatType(TableFormatType.ICEBERG);
split.setAnalyzer(analyzer);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
