morningman commented on a change in pull request #3081: [Dynamic Partition]
Support for automatically drop partitions
URL: https://github.com/apache/incubator-doris/pull/3081#discussion_r396185388
##########
File path:
fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -168,98 +281,62 @@ private void dynamicAddPartition() {
try {
partitionFormat =
DynamicPartitionUtil.getPartitionFormat(partitionColumn);
} catch (DdlException e) {
- recordFailedMsg(olapTable.getName(), e.getMessage());
+ recordCreatePartitionFailedMsg(olapTable.getName(),
e.getMessage());
continue;
}
- Calendar calendar = Calendar.getInstance();
- TableProperty tableProperty = olapTable.getTableProperty();
- DynamicPartitionProperty dynamicPartitionProperty =
tableProperty.getDynamicPartitionProperty();
-
- for (int i = 0; i <= dynamicPartitionProperty.getEnd(); i++) {
- String dynamicPartitionPrefix =
dynamicPartitionProperty.getPrefix();
- String prevBorder =
DynamicPartitionUtil.getPartitionRange(dynamicPartitionProperty.getTimeUnit(),
- i, (Calendar) calendar.clone(), partitionFormat);
- String partitionName = dynamicPartitionPrefix +
DynamicPartitionUtil.getFormattedPartitionName(prevBorder);
-
- // continue if partition already exists
- String nextBorder =
DynamicPartitionUtil.getPartitionRange(dynamicPartitionProperty.getTimeUnit(),
- i + 1, (Calendar) calendar.clone(),
partitionFormat);
- PartitionValue lowerValue = new PartitionValue(prevBorder);
- PartitionValue upperValue = new PartitionValue(nextBorder);
- PartitionInfo partitionInfo = olapTable.getPartitionInfo();
- RangePartitionInfo info = (RangePartitionInfo)
(partitionInfo);
- boolean isPartitionExists = false;
- Range<PartitionKey> addPartitionKeyRange = null;
- try {
- PartitionKey lowerBound =
PartitionKey.createPartitionKey(Collections.singletonList(lowerValue),
Collections.singletonList(partitionColumn));
- PartitionKey upperBound =
PartitionKey.createPartitionKey(Collections.singletonList(upperValue),
Collections.singletonList(partitionColumn));
- addPartitionKeyRange = Range.closedOpen(lowerBound,
upperBound);
- } catch (AnalysisException e) {
- // keys.size is always equal to column.size, cannot
reach this exception
- LOG.error("Keys size is not equl to column size.");
- continue;
- }
- for (Range<PartitionKey> partitionKeyRange :
info.getIdToRange().values()) {
- // only support single column partition now
- try {
- RangeUtils.checkRangeIntersect(partitionKeyRange,
addPartitionKeyRange);
- } catch (DdlException e) {
- isPartitionExists = true;
- if
(addPartitionKeyRange.equals(partitionKeyRange)) {
- clearFailedMsg(olapTable.getName());
- } else {
- recordFailedMsg(olapTable.getName(),
e.getMessage());
- }
- break;
- }
- }
- if (isPartitionExists) {
- continue;
- }
-
- // construct partition desc
- PartitionKeyDesc partitionKeyDesc = new
PartitionKeyDesc(Collections.singletonList(lowerValue),
Collections.singletonList(upperValue));
- HashMap<String, String> partitionProperties = new
HashMap<>(1);
- partitionProperties.put("replication_num",
String.valueOf(DynamicPartitionUtil.estimateReplicateNum(olapTable)));
- SingleRangePartitionDesc rangePartitionDesc = new
SingleRangePartitionDesc(true, partitionName,
- partitionKeyDesc, partitionProperties);
-
- // construct distribution desc
- HashDistributionInfo hashDistributionInfo =
(HashDistributionInfo) olapTable.getDefaultDistributionInfo();
- List<String> distColumnNames = new ArrayList<>();
- for (Column distributionColumn :
hashDistributionInfo.getDistributionColumns()) {
- distColumnNames.add(distributionColumn.getName());
- }
- DistributionDesc distributionDesc = new
HashDistributionDesc(dynamicPartitionProperty.getBuckets(), distColumnNames);
-
- // add partition according to partition desc and
distribution desc
- addPartitionClauses.add(new
AddPartitionClause(rangePartitionDesc, distributionDesc, null, false));
- }
+ addPartitionClauses = getAddPartitionClause(olapTable,
partitionColumn, partitionFormat);
Review comment:
No need to get add partition clause is `skipAddPartition` is true.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
