This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 7c1a7b21aa7 [fix](snapshot) Restore convert rowset meta tablet id
(#29478) (#29750)
7c1a7b21aa7 is described below
commit 7c1a7b21aa709e49684f22562abdaffe8177d5e9
Author: Kang <[email protected]>
AuthorDate: Wed Jan 10 00:34:10 2024 +0800
[fix](snapshot) Restore convert rowset meta tablet id (#29478) (#29750)
---
be/src/olap/snapshot_manager.cpp | 11 ++
..._backup_restore_with_two_phase_fetch_opt.groovy | 115 +++++++++++++++++++++
2 files changed, 126 insertions(+)
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index eb0559927d0..e8135a42025 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -190,6 +190,12 @@ Status SnapshotManager::convert_rowset_ids(const
std::string& clone_dir, int64_t
// remote rowset
*rowset_meta = visible_rowset;
}
+
+ rowset_meta->set_tablet_id(tablet_id);
+ if (partition_id != -1) {
+ rowset_meta->set_partition_id(partition_id);
+ }
+
Version rowset_version = {visible_rowset.start_version(),
visible_rowset.end_version()};
rs_version_map[rowset_version] = rowset_meta;
}
@@ -218,6 +224,11 @@ Status SnapshotManager::convert_rowset_ids(const
std::string& clone_dir, int64_t
// remote rowset
*rowset_meta = stale_rowset;
}
+
+ rowset_meta->set_tablet_id(tablet_id);
+ if (partition_id != -1) {
+ rowset_meta->set_partition_id(partition_id);
+ }
}
if (!rowset_id_mapping.empty() &&
cloned_tablet_meta_pb.has_delete_bitmap()) {
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_with_two_phase_fetch_opt.groovy
b/regression-test/suites/backup_restore/test_backup_restore_with_two_phase_fetch_opt.groovy
new file mode 100644
index 00000000000..49c3c89b351
--- /dev/null
+++
b/regression-test/suites/backup_restore/test_backup_restore_with_two_phase_fetch_opt.groovy
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_with_two_phase_fetch_opt", "backup_restore") {
+ String suiteName = "test_backup_restore_with_two_phase_fetch_opt"
+ String repoName = "${suiteName}_repo"
+ String dbName = "${suiteName}_db"
+ String tableName = "${suiteName}_table"
+ String snapshotName = "${suiteName}_snapshot"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `distinct_id` varchar(255) NULL,
+ `dt` date NULL,
+ `device_id` text NULL,
+ `ep_user_id` bigint(20) NULL,
+ `ep_corp_name` text NULL,
+ `ep_product_sid` bigint(20) NULL,
+ `supply_id` bigint(20) NULL,
+ `ep_supply_type` text NULL,
+ `ep_split_code` text NULL,
+ `ep_symbol_name` text NULL,
+ `ep_manufactor_short_name` text NULL,
+ `event_type` text NULL,
+ `platform_type` text NULL,
+ `is_first` text NULL,
+ `os` text NULL,
+ `area_address` text NULL,
+ `ts` text NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`distinct_id`, `dt`)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE (dt)(
+ FROM ("2023-12-01") TO ("2023-12-25") INTERVAL 1 DAY
+ )
+ DISTRIBUTED BY HASH(`distinct_id`) BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.prefix" = "p"
+ );
+ """
+
+ sql """
+ INSERT INTO ${dbName}.${tableName} VALUES
+ ("1231", "2023-12-01", "device_id", 123123, "corp_name", 123321, 1231,
"supply_type", "split_code", "symbol_name", "short_name", "event_type",
"platform_type", "is_first", "os", "area_address", "ts"),
+ ("1232", "2023-12-01", "device_id", 123123, "corp_name", 123321, 1231,
"supply_type", "split_code", "symbol_name", "short_name", "event_type",
"platform_type", "is_first", "os", "area_address", "ts"),
+ ("1233", "2023-12-02", "device_id", 123123, "corp_name", 123321, 1231,
"supply_type", "split_code", "symbol_name", "short_name", "event_type",
"platform_type", "is_first", "os", "area_address", "ts"),
+ ("1234", "2023-12-03", "device_id", 123123, "corp_name", 123321, 1231,
"supply_type", "split_code", "symbol_name", "short_name", "event_type",
"platform_type", "is_first", "os", "area_address", "ts"),
+ ("1235", "2023-12-04", "device_id", 123123, "corp_name", 123321, 1231,
"supply_type", "split_code", "symbol_name", "short_name", "event_type",
"platform_type", "is_first", "os", "area_address", "ts"),
+ ("1236", "2023-12-04", "device_id", 123123, "corp_name", 123321, 1231,
"supply_type", "split_code", "symbol_name", "short_name", "event_type",
"platform_type", "is_first", "os", "area_address", "ts");
+ """
+ def result = sql "SELECT * FROM ${dbName}.${tableName} ORDER BY dt DESC
LIMIT 5"
+ assertEquals(result.size(), 5)
+
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ """
+
+ while (!syncer.checkSnapshotFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON ( `${tableName}`)
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_dynamic_partition_enable"="true",
+ "reserve_replica" = "true"
+ )
+ """
+
+ while (!syncer.checkAllRestoreFinish(dbName)) {
+ Thread.sleep(3000)
+ }
+
+ result = sql "SELECT * FROM ${dbName}.${tableName} ORDER BY dt DESC LIMIT
5"
+ assertEquals(result.size(), 5)
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
