924060929 commented on code in PR #69299: URL: https://github.com/apache/doris/pull/69299#discussion_r4321334951
########## fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenPartitionView.java: ########## @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mvcc; + +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition; +import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TScalarType; +import org.apache.doris.thrift.TTypeDesc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** One published partition generation: maps and pruning index share the same read-only partition items. */ +public final class PluginDrivenPartitionView { + private final ImmutableMap<String, PartitionItem> items; + private final ImmutableMap<String, Long> freshness; + private final SortedPartitionRanges<String> sortedRanges; + + PluginDrivenPartitionView(Map<String, PartitionItem> items, Map<String, Long> freshness) { + this.items = ImmutableMap.copyOf(items); + this.freshness = ImmutableMap.copyOf(freshness); + // Build before cache admission, like HivePartitionValues: the cache accounts for the index + // together with its maps, and readers never observe a partially initialized generation. + this.sortedRanges = SortedPartitionRanges.build(this.items); Review Comment: Fixed in 12ba5caef8d92a09526c8172f567d40052d6d5b6. A partition view with heterogeneous LIST tuple arity or primitive types retains its complete partition/freshness maps but does not construct an invalid ordered pruning index. Added coverage for mixed one- and two-column tuples; homogeneous LIST and RANGE indexes remain supported. ########## fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java: ########## @@ -540,6 +552,10 @@ private static class DefaultExternalMetaCache extends ExternalCatalogMetaCache { MetaCacheEntryInvalidation.forTableIdentity( key -> key.getNameMapping().getLocalDbName(), key -> key.getNameMapping().getLocalTblName()))); + registerEntry(MetaCacheEntryDef.contextualOnly( Review Comment: Fixed in 12ba5caef8d92a09526c8172f567d40052d6d5b6. Cache entry definitions now carry an entry-specific size estimator through the catalog runtime. Partition-view admission accounts for every partition with independent per-record traversal bounds and typed literal sizing, avoiding the whole-table 10,000-object reflective cap and Expr supplier traversal. Added 12,000-partition weighted admission/hit/invalidation, RANGE sizing, and actual catalog-runtime estimator propagation tests. Final validation: 210 scoped tests plus an overlapping 21-test cache/runtime pass, and build.sh --fe with Checkstyle. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
