github-actions[bot] commented on code in PR #66717:
URL: https://github.com/apache/doris/pull/66717#discussion_r3797627345
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java:
##########
@@ -2000,7 +2037,8 @@ public static IcebergSnapshotCacheValue
getSnapshotCacheValue(
Optional<TableScanParams> scanParams) {
if (tableSnapshot.isPresent() ||
IcebergUtils.isIcebergBranchOrTag(scanParams)) {
// If a snapshot is specified, use the specified snapshot and the
corresponding schema (not latest).
- Table icebergTable =
IcebergSnapshotCacheValue.retainTableGeneration(getIcebergTable(dorisTable));
+ IcebergExternalMetaCache metaCache =
icebergExternalMetaCache(dorisTable);
+ Table icebergTable =
metaCache.getQueryScopedIcebergTable(dorisTable);
Review Comment:
[P2] Preserve snapshot isolation through this handoff
This now returns a `QueryScopedTable`, but the four-argument
`IcebergSnapshotCacheValue` constructor immediately feeds it through
`retainTableGeneration()`. Because `QueryScopedTableOperations` is not a
`FrozenTableOperations`, that path rebuilds a plain `BaseTable` over the same
admitted `TableMetadata` and loses the query-local snapshot copies. A `FOR
VERSION`/`FOR TIME`/branch/tag scan can then populate the shared `BaseSnapshot`
manifest caches after the table entry's weight was fixed; repeated historical
reads can grow the entry past its table/catalog/global reservation. Please
preserve the query-scoped wrapper (or another isolated snapshot copy) and cover
a weighted v2 historical/ref scan that leaves the cached generation's
`allManifests`/`dataManifests`/`deleteManifests` fields null.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java:
##########
@@ -0,0 +1,450 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
+import org.apache.doris.datasource.metacache.MetaCacheWeightUtils;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.paimon.privilege.PrivilegedFileStoreTable;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FallbackReadFileStoreTable;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.BigIntType;
+import org.apache.paimon.types.BinaryType;
+import org.apache.paimon.types.BlobType;
+import org.apache.paimon.types.BooleanType;
+import org.apache.paimon.types.CharType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DateType;
+import org.apache.paimon.types.DecimalType;
+import org.apache.paimon.types.DoubleType;
+import org.apache.paimon.types.FloatType;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.LocalZonedTimestampType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.MultisetType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.SmallIntType;
+import org.apache.paimon.types.TimeType;
+import org.apache.paimon.types.TimestampType;
+import org.apache.paimon.types.TinyIntType;
+import org.apache.paimon.types.VarBinaryType;
+import org.apache.paimon.types.VarCharType;
+import org.apache.paimon.types.VariantType;
+import org.apache.paimon.types.VectorType;
+
+import java.util.List;
+import java.util.Map;
+
+/** Publication-time retained-weight formula for Paimon snapshot projections.
*/
+final class PaimonCacheSizeEstimator {
+ // Calibrated against JOL retained-graph deltas in
PaimonExternalMetaCacheTest.
+ private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L;
+ private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128;
+ private static final long KEY_BASE_BYTES = objectBytes(128L);
+ private static final long SNAPSHOT_BASE_BYTES = objectBytes(4L * 1024L);
+ private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L);
+ // A top-level DataField, its list slot and shared per-field overhead; the
DataType instance
+ // is accounted separately by addTypePayload.
+ private static final long TABLE_FIELD_BYTES = objectBytes(40L);
+ private static final long TABLE_OPTION_BYTES = objectBytes(44L);
+ private static final long TABLE_KEY_BYTES = objectBytes(128L);
+ // Exact Paimon 1.4.2 layouts, pinned by PAIMON_TYPE_LAYOUT_SUPPORTED.
+ private static final long DATA_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 4L);
+ private static final long ARRAY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L);
+ private static final long VECTOR_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 5L);
+ private static final long MAP_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 1L);
+ private static final long MULTISET_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L);
+ // RowType plus Collections.unmodifiableList(new ArrayList<>(fields)).
+ private static final long ROW_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 1L);
+ private static final long UNMODIFIABLE_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long ARRAY_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L);
+ private static final long HASH_MAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L);
+ private static final long HASH_MAP_NODE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ private static final long INTEGER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final int ROW_TYPE_LAZY_MAP_COUNT = 4;
+ // Accepted leaf DataType implementations and the int fields each adds to
DataType's nullable
+ // flag and type root. Any other class, including a future or third-party
type, rejects
+ // weighted admission instead of being counted as an arbitrary primitive.
+ private static final String[] NO_LEAF_FIELDS = {};
+ private static final String[] LENGTH_LEAF_FIELDS = {"length:int"};
+ private static final String[] PRECISION_LEAF_FIELDS = {"precision:int"};
+ private static final Map<Class<? extends DataType>, String[]>
LEAF_TYPE_FIELDS =
+ ImmutableMap.<Class<? extends DataType>, String[]>builder()
+ .put(CharType.class, LENGTH_LEAF_FIELDS)
+ .put(VarCharType.class, LENGTH_LEAF_FIELDS)
+ .put(BooleanType.class, NO_LEAF_FIELDS)
+ .put(BinaryType.class, LENGTH_LEAF_FIELDS)
+ .put(VarBinaryType.class, LENGTH_LEAF_FIELDS)
+ .put(DecimalType.class, new String[] {"precision:int",
"scale:int"})
+ .put(TinyIntType.class, NO_LEAF_FIELDS)
+ .put(SmallIntType.class, NO_LEAF_FIELDS)
+ .put(IntType.class, NO_LEAF_FIELDS)
+ .put(BigIntType.class, NO_LEAF_FIELDS)
+ .put(FloatType.class, NO_LEAF_FIELDS)
+ .put(DoubleType.class, NO_LEAF_FIELDS)
+ .put(DateType.class, NO_LEAF_FIELDS)
+ .put(TimeType.class, PRECISION_LEAF_FIELDS)
+ .put(TimestampType.class, PRECISION_LEAF_FIELDS)
+ .put(LocalZonedTimestampType.class, PRECISION_LEAF_FIELDS)
+ .put(VariantType.class, NO_LEAF_FIELDS)
+ .put(BlobType.class, NO_LEAF_FIELDS)
+ .build();
+ private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED =
checkPaimonTypeLayout();
+ private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED =
checkPaimonTableLayout();
+ // One Paimon Partition record with its single-column LinkedHashMap spec
plus map entry; extra
+ // columns are charged by PaimonPartitionInfo.
+ private static final long PARTITION_BYTES = objectBytes(272L);
+ private static final long PARTITION_ITEM_BYTES = objectBytes(640L);
+ private static final long WRAPPER_BYTES = objectBytes(512L);
+
+ private PaimonCacheSizeEstimator() {
+ }
+
+ private static long objectBytes(long bytes) {
+ return MetaCacheWeightUtils.estimatedObjectBytes(bytes);
+ }
+
+ /** DataType: typeRoot reference plus the isNullable flag, then the
subclass int fields. */
+ private static long leafTypeBytes(String[] intFields) {
+ return MetaCacheWeightUtils.estimatedObjectLayoutBytes(
+ 1L, 1L + (long) Integer.BYTES * intFields.length);
+ }
+
+ /** Pin the Paimon 1.4.2 DataType/DataField/RowType layouts the formulas
above are built on. */
+ private static boolean checkPaimonTypeLayout() {
+ boolean supported = MetaCacheWeightUtils.hasExpectedInstanceFields(
+ DataType.class, "isNullable:boolean", "typeRoot:DataTypeRoot")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ DataField.class, "id:int", "name:String",
"type:DataType",
+ "description:String", "defaultValue:String")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ RowType.class, "fields:List", "laziedNameToField:Map",
+ "laziedNameToIndex:Map", "laziedFieldIdToField:Map",
+ "laziedFieldIdToIndex:Map")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ ArrayType.class, "elementType:DataType")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ VectorType.class, "elementType:DataType", "length:int")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ MapType.class, "keyType:DataType",
"valueType:DataType")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ MultisetType.class, "elementType:DataType");
+ for (Map.Entry<Class<? extends DataType>, String[]> leaf :
LEAF_TYPE_FIELDS.entrySet()) {
+ supported &= MetaCacheWeightUtils.hasExpectedInstanceFields(
+ leaf.getKey(), leaf.getValue());
+ }
+ return supported;
+ }
+
+ /** Pin TableSchema and the two accepted FileStoreTable implementations. */
+ private static boolean checkPaimonTableLayout() {
+ ClassLoader loader = FileStoreTable.class.getClassLoader();
+ String[] abstractTableFields = {
+ "fileIO:FileIO", "path:Path", "tableSchema:TableSchema",
+ "catalogEnvironment:CatalogEnvironment",
"manifestCache:SegmentsCache",
+ "snapshotCache:Cache", "statsCache:Cache",
"dvmetaCache:DVMetaCache"};
+ return MetaCacheWeightUtils.hasExpectedInstanceFields(
+ TableSchema.class, "version:int", "id:long", "fields:List",
+ "highestFieldId:int", "partitionKeys:List", "primaryKeys:List",
+ "bucketKeys:List", "numBucket:int", "options:Map",
"comment:String",
+ "timeMillis:long")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ "org.apache.paimon.table.AbstractFileStoreTable",
loader,
+ abstractTableFields)
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ "org.apache.paimon.table.AppendOnlyFileStoreTable",
loader,
+ "lazyStore:AppendOnlyFileStore")
+ && MetaCacheWeightUtils.hasExpectedInstanceFields(
+ "org.apache.paimon.table.PrimaryKeyFileStoreTable",
loader,
+ "lazyStore:KeyValueFileStore");
+ }
+
+ static MetaCacheSizeEstimate estimateSnapshotEntry(
+ PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) {
+ if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) {
+ return
MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment");
+ }
+ if (!PAIMON_TYPE_LAYOUT_SUPPORTED || !PAIMON_TABLE_LAYOUT_SUPPORTED) {
+ return
MetaCacheSizeEstimate.incomplete("unsupported_paimon_layout");
+ }
+ Table table = value.getSnapshot().getTable();
+ if (!isSupportedTable(table)) {
+ return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:"
+ + (table == null ? "null" : table.getClass().getName()));
+ }
+
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ KEY_BASE_BYTES,
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_BYTES);
+ bytes = addCount(bytes,
value.getPartitionInfo().getNameToPartition().size(), PARTITION_BYTES);
+ bytes = addCount(bytes,
value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getPartitionInfo().getRetainedPayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedTablePayloadBytes());
+ return MetaCacheSizeEstimate.complete(
+ MetaCacheWeightUtils.saturatedAdd(bytes,
estimateTable(table)));
Review Comment:
[P2] Reserve the retained Paimon store graph
This estimate never charges `FileStoreTable.lazyStore`. Partition loading
calls `newReadBuilder().newScan()` on this exact retained table before
publication, while an unpartitioned table reaches the same initialization
during scan planning after admission. Paimon 1.4.2's store owns copied
`CoreOptions`/options state and logical `RowType`/field/type trees, so the
omission grows with supported schema width and is not covered by the 16 KiB
fixed base (a few hundred primitive fields already exceed that slack). Please
reserve the store-side graph without opening remote metadata, and add wide
append-only/primary-key JOL checks before and after
`newReadBuilder().newScan()`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
