CalvinKirs commented on code in PR #66717: URL: https://github.com/apache/doris/pull/66717#discussion_r3797374355
########## fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java: ########## @@ -0,0 +1,1194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.iceberg.BlobMetadata; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.UnknownTransform; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Publication-time retained-weight formulas for Iceberg cache entries. */ +final class IcebergCacheSizeEstimator { + // Calibrated against JOL retained-graph deltas in IcebergExternalMetaCacheTest. + // Every metadata element visited (field, type, snapshot, summary entry, ...) costs a few + // reads; the bound only guards against pathological metadata and is far above real tables + // (a 10,000-snapshot history with 15 summary keys each is 160,000 elements). Exceeding it + // rejects weighted admission, so it must not be reachable by ordinary long-lived tables. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L; + // Total name characters the estimator may lower-case while reserving case-insensitive indexes. + private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // TableMetadata-side share of one schema version: schemas list slot and schemasById entry, + // including the growth of both from their singleton to their regular immutable shapes. + private static final long SCHEMA_VERSION_BYTES = objectBytes(128L); + private static final long PARTITION_SPEC_BYTES = objectBytes(256L); + // Exact active-layout sizes of the Iceberg/Guava objects that lazy partition, sort and + // schema state allocates. Iceberg 1.10.1 field layouts are pinned by ICEBERG_LAZY_LAYOUT_SUPPORTED. + private static final long PARTITION_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L); + private static final long SORT_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + // Identity/Bucket/Truncate transforms are allocated per parsed field; time transforms are enums. + private static final long TRANSFORM_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long NESTED_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L); + private static final long STRUCT_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L); + private static final long SCHEMA_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L); + private static final long IMMUTABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long IMMUTABLE_MAP_KEY_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SINGLETON_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long REGULAR_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long LONG_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; + // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). + private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L), + MetaCacheWeightUtils.estimatedIntArrayBytes(1L)); + private static final long LIST_MULTIMAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L); + private static final long CAPTURING_SUPPLIER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long POSITION_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L); + // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor. Required ancestors + // collapse into a single Position2/3Accessor that replaces the inner accessor, which retains + // less than this per-level reservation. + private static final long WRAPPED_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L); + private static final long LIST_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L); + private static final long DECIMAL_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final long FIXED_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long GEOMETRY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long GEOGRAPHY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SORT_ORDER_BYTES = objectBytes(256L); + private static final long TABLE_PROPERTY_BYTES = objectBytes(40L); + private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L); + private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L); + private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L); + private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L); + private static final long SNAPSHOT_REF_BYTES = objectBytes(128L); + private static final long STATISTICS_FILE_BYTES = objectBytes(512L); + private static final long BLOB_METADATA_BYTES = objectBytes(128L); + private static final long BLOB_FIELD_BYTES = objectBytes(32L); + private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); + private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); + // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a + // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. + private static final long PARTITION_BYTES = objectBytes(680L); + private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); + private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); + private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); + private static final long DATA_FILE_BYTES = objectBytes(896L); + private static final long DELETE_FILE_BYTES = objectBytes(1024L); + private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L); + private static final String BASE_SNAPSHOT_CLASS_NAME = "org.apache.iceberg.BaseSnapshot"; + private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS = + loadBaseSnapshotRetainedCacheFields(); + // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog supplier + // (REST snapshot-loading-mode=refs). Publication must not perform that IO. + private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD = + loadTableMetadataField("snapshotsLoaded", boolean.class); + private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD = + loadTableMetadataField("snapshotsSupplier", null); + // The formulas above are built on the Iceberg 1.10.1 instance-field layouts of the classes a + // cached table retains. Every non-static field is pinned, not only the transient lazy ones: a + // library upgrade that adds a retained reference makes weighted admission fail closed. + private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED = checkIcebergLayout(); + + private IcebergCacheSizeEstimator() { + } + + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } + Table table = value.getRetainedIcebergTable(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } + long bytes = KEY_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation())); + + IcebergPartitionInfo partitionInfo = value.getPartitionInfo(); + bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); + bytes = addCount(bytes, value.getNameMapping().map(Map::size).orElse(0), + NAME_MAPPING_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedNameMappingPayloadBytes()); + + if (value.getRetainedIcebergTable().isPresent()) { + Table table = value.getRetainedIcebergTable().get(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); Review Comment: Kept intentionally and documented in code: the snapshot projection holds its own strong reference to the frozen table generation, and the table entry and the projection have independent lifetimes (TTL, weight eviction, soft collection). If only the projection charged the delta, an evicted table entry would leave the whole retained table graph uncharged until the projection expires. Charging it once per dependent entry keeps every admitted entry able to carry its own retained graph; the budget should be sized with table metadata counted once per dependent entry. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
