This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 80e8815 [improvement](restore) allow query on part of partitions when
others are in RESTORE (#8245)
80e8815 is described below
commit 80e88159d9550179be6329985bcb648967647096
Author: GoGoWen <82132356+gogo...@users.noreply.github.com>
AuthorDate: Thu Mar 3 22:34:10 2022 +0800
[improvement](restore) allow query on part of partitions when others are in
RESTORE (#8245)
---
.../java/org/apache/doris/analysis/Analyzer.java | 14 ++++++++-
.../java/org/apache/doris/backup/RestoreJob.java | 34 ++++++++++++++++++++++
.../java/org/apache/doris/catalog/Partition.java | 3 +-
.../org/apache/doris/planner/OlapScanNode.java | 8 +++++
4 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index db6e6c5..9bcb5a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Type;
@@ -81,6 +82,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Repository of analysis state for single select block.
@@ -585,7 +587,17 @@ public class Analyzer {
if (table.getType() == TableType.OLAP && (((OlapTable)
table).getState() == OlapTableState.RESTORE
|| ((OlapTable) table).getState() ==
OlapTableState.RESTORE_WITH_LOAD)) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_STATE,
"RESTORING");
+ Boolean isNotRestoring = ((OlapTable)
table).getPartitions().stream().filter(
+ partition -> partition.getState() == PartitionState.RESTORE
+ ).collect(Collectors.toList()).isEmpty();
+
+ if(!isNotRestoring){
+ // if doing restore with partitions, the status check push
down to OlapScanNode::computePartitionInfo to
+ // support query that partitions is not restoring.
+ } else {
+ // if doing restore with table, throw exception here
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_STATE, "RESTORING");
+ }
}
// tableName.getTbl() stores the table name specified by the user in
the from statement.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 005bb9c..f6f1830 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -36,6 +36,7 @@ import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
@@ -469,6 +470,16 @@ public class RestoreJob extends AbstractJob {
}
olapTbl.setState(OlapTableState.RESTORE);
+ // set restore status for partitions
+ BackupOlapTableInfo tblInfo =
jobInfo.backupOlapTableObjects.get(tableName);
+ for (Map.Entry<String, BackupPartitionInfo> partitionEntry :
tblInfo.partitions.entrySet()) {
+ String partitionName = partitionEntry.getKey();
+ Partition partition = olapTbl.getPartition(partitionName);
+ if (partition == null) {
+ continue;
+ }
+ partition.setState(PartitionState.RESTORE);
+ }
} finally {
olapTbl.writeUnlock();
}
@@ -578,6 +589,7 @@ public class RestoreJob extends AbstractJob {
return;
}
}
+
} else {
// partitions does not exist
PartitionInfo localPartitionInfo =
localOlapTbl.getPartitionInfo();
@@ -1076,6 +1088,16 @@ public class RestoreJob extends AbstractJob {
tbl.writeLock();
try {
olapTbl.setState(OlapTableState.RESTORE);
+ // set restore status for partitions
+ BackupOlapTableInfo tblInfo =
jobInfo.backupOlapTableObjects.get(tableName);
+ for (Map.Entry<String, BackupPartitionInfo> partitionEntry :
tblInfo.partitions.entrySet()) {
+ String partitionName = partitionEntry.getKey();
+ Partition partition = olapTbl.getPartition(partitionName);
+ if (partition == null) {
+ continue;
+ }
+ partition.setState(PartitionState.RESTORE);
+ }
} finally {
tbl.writeUnlock();
}
@@ -1650,6 +1672,18 @@ public class RestoreJob extends AbstractJob {
|| olapTbl.getState() ==
OlapTableState.RESTORE_WITH_LOAD) {
olapTbl.setState(OlapTableState.NORMAL);
}
+
+ BackupOlapTableInfo tblInfo =
jobInfo.backupOlapTableObjects.get(tableName);
+ for (Map.Entry<String, BackupPartitionInfo> partitionEntry :
tblInfo.partitions.entrySet()) {
+ String partitionName = partitionEntry.getKey();
+ Partition partition = olapTbl.getPartition(partitionName);
+ if (partition == null) {
+ continue;
+ }
+ if (partition.getState() == PartitionState.RESTORE) {
+ partition.setState(PartitionState.NORMAL);
+ }
+ }
} finally {
tbl.writeUnlock();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
index 3120c64..8b94e28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
@@ -52,7 +52,8 @@ public class Partition extends MetaObject implements Writable
{
@Deprecated
ROLLUP,
@Deprecated
- SCHEMA_CHANGE
+ SCHEMA_CHANGE,
+ RESTORE
}
@SerializedName(value = "id")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index ab0e3b2..5ef9ed6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -39,6 +39,7 @@ import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
@@ -597,6 +598,13 @@ public class OlapScanNode extends ScanNode {
.collect(Collectors.toList());
}
selectedPartitionNum = selectedPartitionIds.size();
+
+ for(long id : selectedPartitionIds){
+ Partition partition = olapTable.getPartition(id);
+ if(partition.getState() == PartitionState.RESTORE){
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_PARTITION_STATE,
partition.getName(), "RESTORING");
+ }
+ }
LOG.debug("partition prune cost: {} ms, partitions: {}",
(System.currentTimeMillis() - start), selectedPartitionIds);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
