marising commented on a change in pull request #4330: URL: https://github.com/apache/incubator-doris/pull/4330#discussion_r472611199
########## File path: fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java ########## @@ -0,0 +1,596 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe.cache; + +import org.apache.doris.analysis.CompoundPredicate; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.DateLiteral; +import org.apache.doris.analysis.InPredicate; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.Config; +import org.apache.doris.planner.PartitionColumnFilter; + +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.Lists; +import com.google.common.collect.Range; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * Convert the range of the partition to the list + * all partition by day/week/month split to day list + */ +public class PartitionRange { + private static final Logger LOG = LogManager.getLogger(PartitionRange.class); + + public class PartitionSingle { + private Partition partition; + private PartitionKey partitionKey; + private long partitionId; + private PartitionKeyType cacheKey; + private boolean fromCache; + private boolean tooNew; + + public Partition getPartition() { + return partition; + } + + public void setPartition(Partition partition) { + this.partition = partition; + } + + public PartitionKey getPartitionKey() { + return partitionKey; + } + + public void setPartitionKey(PartitionKey key) { + this.partitionKey = key; + } + + public long getPartitionId() { + return partitionId; + } + + public void setPartitionId(long partitionId) { + this.partitionId = partitionId; + } + + public PartitionKeyType getCacheKey() { + return cacheKey; + } + + public void setCacheKey(PartitionKeyType cacheKey) { + this.cacheKey.clone(cacheKey); + } + + public boolean isFromCache() { + return fromCache; + } + + public void setFromCache(boolean fromCache) { + this.fromCache = fromCache; + } + + public boolean isTooNew() { + return tooNew; + } + + public void setTooNew(boolean tooNew) { + this.tooNew = tooNew; + } + + public PartitionSingle() { + this.partitionId = 0; + this.cacheKey = new PartitionKeyType(); + this.fromCache = false; + this.tooNew = false; + } + + public void Debug() { + if (partition != null) { + LOG.info("partition id {}, cacheKey {}, version {}, time {}, fromCache {}, tooNew {} ", + partitionId, cacheKey.realValue(), + partition.getVisibleVersion(), partition.getVisibleVersionTime(), + fromCache, tooNew); + } else { + LOG.info("partition id {}, cacheKey {}, fromCache {}, tooNew {} ", partitionId, + cacheKey.realValue(), fromCache, tooNew); + } + } + } + + public enum KeyType { + DEFAULT, + LONG, + DATE, + DATETIME, + TIME + } + + public static class PartitionKeyType { + private SimpleDateFormat df8 = new SimpleDateFormat("yyyyMMdd"); + private SimpleDateFormat df10 = new SimpleDateFormat("yyyy-MM-dd"); + + public KeyType keyType = KeyType.DEFAULT; + public long value; + public Date date; + + public boolean init(Type type, String str) { + if (type.getPrimitiveType() == PrimitiveType.DATE) { + try { + date = df10.parse(str); + } catch (Exception e) { + LOG.warn("parse error str{}.", str); + return false; + } + keyType = KeyType.DATE; + } else { + value = Long.valueOf(str); + keyType = KeyType.LONG; + } + return true; + } + + public boolean init(Type type, LiteralExpr expr) { + switch (type.getPrimitiveType()) { + case BOOLEAN: + case TIME: + case DATETIME: + case FLOAT: + case DOUBLE: + case DECIMAL: + case DECIMALV2: + case CHAR: + case VARCHAR: + case LARGEINT: + LOG.info("PartitionCache not support such key type {}", type.toSql()); + return false; + case DATE: + date = getDateValue(expr); + keyType = KeyType.DATE; + break; + case TINYINT: + case SMALLINT: + case INT: + case BIGINT: + value = expr.getLongValue(); + keyType = KeyType.LONG; + break; + } + return true; + } + + public void clone(PartitionKeyType key) { + keyType = key.keyType; + value = key.value; + date = key.date; + } + + public boolean equals(PartitionKeyType key) { + return realValue() == key.realValue(); + } + + public void add(int num) { + if (keyType == KeyType.DATE) { + date = new Date(date.getTime() + num * 3600 * 24 * 1000); + } else { + value += num; + } + } + + public String toString() { + if (keyType == KeyType.DEFAULT) { + return ""; + } else if (keyType == KeyType.DATE) { + return df10.format(date); + } else { + return String.valueOf(value); + } + } + + public long realValue() { + if (keyType == KeyType.DATE) { + return Long.parseLong(df8.format(date)); + } else { + return value; + } + } + + private Date getDateValue(LiteralExpr expr) { + value = expr.getLongValue() / 1000000; + Date dt = null; + try { + dt = df8.parse(String.valueOf(value)); + } catch (Exception e) { + } + return dt; + } + } + + private CompoundPredicate partitionKeyPredicate; + private OlapTable olapTable; + private RangePartitionInfo rangePartitionInfo; + private Column partitionColumn; + private List<PartitionSingle> partitionSingleList; + + public CompoundPredicate getPartitionKeyPredicate() { + return partitionKeyPredicate; + } + + public void setPartitionKeyPredicate(CompoundPredicate partitionKeyPredicate) { + this.partitionKeyPredicate = partitionKeyPredicate; + } + + public RangePartitionInfo getRangePartitionInfo() { + return rangePartitionInfo; + } + + public void setRangePartitionInfo(RangePartitionInfo rangePartitionInfo) { + this.rangePartitionInfo = rangePartitionInfo; + } + + public Column getPartitionColumn() { + return partitionColumn; + } + + public void setPartitionColumn(Column partitionColumn) { + this.partitionColumn = partitionColumn; + } + + public List<PartitionSingle> getPartitionSingleList() { + return partitionSingleList; + } + + public PartitionRange() { + } + + public PartitionRange(CompoundPredicate partitionKeyPredicate, OlapTable olapTable, + RangePartitionInfo rangePartitionInfo) { + this.partitionKeyPredicate = partitionKeyPredicate; + this.olapTable = olapTable; + this.rangePartitionInfo = rangePartitionInfo; + this.partitionSingleList = Lists.newArrayList(); + } + + /** + * analytics PartitionKey and PartitionInfo + * + * @return + */ + public boolean analytics() { + if (rangePartitionInfo.getPartitionColumns().size() != 1) { + return false; + } + partitionColumn = rangePartitionInfo.getPartitionColumns().get(0); + PartitionColumnFilter filter = createPartitionFilter(this.partitionKeyPredicate, partitionColumn); + try { + if (!buildPartitionKeyRange(filter, partitionColumn)) { + return false; + } + getTablePartitionList(olapTable); + } catch (AnalysisException e) { + LOG.warn("get partition range failed, because:", e); + return false; + } + return true; + } + + public boolean setCacheFlag(long cacheKey) { + boolean find = false; + for (PartitionSingle single : partitionSingleList) { + if (single.getCacheKey().realValue() == cacheKey) { + single.setFromCache(true); + find = true; + break; + } + } + return find; + } + + public boolean setTooNewByID(long partitionId) { + boolean find = false; + for (PartitionSingle single : partitionSingleList) { + if (single.getPartition().getId() == partitionId) { + single.setTooNew(true); + find = true; + break; + } + } + return find; + } + + public boolean setTooNewByKey(long cacheKey) { + boolean find = false; + for (PartitionSingle single : partitionSingleList) { + if (single.getCacheKey().realValue() == cacheKey) { + single.setTooNew(true); + find = true; + break; + } + } + return find; + } + + /** + * Support left or right hit cache, not support middle. + * 20200113-2020115, not support 20200114 + */ + public Cache.HitRange diskPartitionRange(List<PartitionSingle> rangeList) { Review comment: buildDiskPartitionRange(); ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
