github-actions[bot] commented on code in PR #66717:
URL: https://github.com/apache/doris/pull/66717#discussion_r3797186561
##########
be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp:
##########
@@ -53,6 +53,16 @@ std::vector<SchemaScanner::ColumnDesc>
SchemaCatalogMetaCacheStatsScanner::_s_tb
{"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true},
{"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true},
{"LAST_ERROR", TYPE_STRING, sizeof(StringRef), true},
+ {"WEIGHT_BOUNDED", TYPE_BOOLEAN, sizeof(bool), true},
Review Comment:
[P2] Preserve this RPC across a rolling upgrade
A new BE sends every local column name to fetchSchemaTableData, but an old
FE's META_CACHE_STATS_COLUMN_TO_INDEX has none of this appended suffix; its
filterColumns() looks up a null index and throws. The inverse pair also cannot
execute a new-column projection on an old BE descriptor. Please add a
version-gated legacy request/fallback that fills unsupported cells with
NULL/defaults (and gate new-column planning as needed), with mixed-version
tests in both directions.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java:
##########
@@ -0,0 +1,1194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.datasource.NameMapping;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
+import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
+import org.apache.doris.datasource.metacache.MetaCacheWeightUtils;
+
+import org.apache.iceberg.BlobMetadata;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StatisticsFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.UnknownTransform;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Publication-time retained-weight formulas for Iceberg cache entries. */
+final class IcebergCacheSizeEstimator {
+ // Calibrated against JOL retained-graph deltas in
IcebergExternalMetaCacheTest.
+ // Every metadata element visited (field, type, snapshot, summary entry,
...) costs a few
+ // reads; the bound only guards against pathological metadata and is far
above real tables
+ // (a 10,000-snapshot history with 15 summary keys each is 160,000
elements). Exceeding it
+ // rejects weighted admission, so it must not be reachable by ordinary
long-lived tables.
+ private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L;
+ // Total name characters the estimator may lower-case while reserving
case-insensitive indexes.
+ private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L;
+ private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128;
+ private static final long KEY_BASE_BYTES = objectBytes(128L);
+ private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L);
+ // TableMetadata-side share of one schema version: schemas list slot and
schemasById entry,
+ // including the growth of both from their singleton to their regular
immutable shapes.
+ private static final long SCHEMA_VERSION_BYTES = objectBytes(128L);
+ private static final long PARTITION_SPEC_BYTES = objectBytes(256L);
+ // Exact active-layout sizes of the Iceberg/Guava objects that lazy
partition, sort and
+ // schema state allocates. Iceberg 1.10.1 field layouts are pinned by
ICEBERG_LAZY_LAYOUT_SUPPORTED.
+ private static final long PARTITION_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L);
+ private static final long SORT_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ // Identity/Bucket/Truncate transforms are allocated per parsed field;
time transforms are enums.
+ private static final long TRANSFORM_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long NESTED_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L);
+ private static final long STRUCT_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L);
+ private static final long SCHEMA_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L);
+ private static final long IMMUTABLE_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long IMMUTABLE_MAP_KEY_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SINGLETON_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long REGULAR_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L);
+ private static final long ARRAY_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L);
+ private static final long HASH_MAP_NODE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ private static final long HASH_MAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L);
+ private static final long INTEGER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long LONG_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate[";
+ // Truncate on a decimal source retains a BigInteger width (object plus
one-int magnitude).
+ private static final long TRUNCATE_WIDTH_BYTES =
MetaCacheWeightUtils.saturatedAdd(
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L),
+ MetaCacheWeightUtils.estimatedIntArrayBytes(1L));
+ private static final long LIST_MULTIMAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L);
+ private static final long CAPTURING_SUPPLIER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long POSITION_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L);
+ // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor.
Required ancestors
+ // collapse into a single Position2/3Accessor that replaces the inner
accessor, which retains
+ // less than this per-level reservation.
+ private static final long WRAPPED_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L);
+ private static final long LIST_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long MAP_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L);
+ private static final long DECIMAL_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final long FIXED_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long GEOMETRY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long GEOGRAPHY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SORT_ORDER_BYTES = objectBytes(256L);
+ private static final long TABLE_PROPERTY_BYTES = objectBytes(40L);
+ private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L);
+ private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L);
+ private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L);
+ private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L);
+ private static final long SNAPSHOT_REF_BYTES = objectBytes(128L);
+ private static final long STATISTICS_FILE_BYTES = objectBytes(512L);
+ private static final long BLOB_METADATA_BYTES = objectBytes(128L);
+ private static final long BLOB_FIELD_BYTES = objectBytes(32L);
+ private static final long PARTITION_STATISTICS_FILE_BYTES =
objectBytes(256L);
+ private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L);
+ // One retained IcebergPartition (value/transform ArrayLists) or one
RangePartitionItem with a
+ // single partition column plus its map entry; extra columns are charged
by IcebergPartitionInfo.
+ private static final long PARTITION_BYTES = objectBytes(680L);
+ private static final long PARTITION_ALIAS_BYTES = objectBytes(256L);
+ private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L);
+ private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L);
+ private static final long DATA_FILE_BYTES = objectBytes(896L);
+ private static final long DELETE_FILE_BYTES = objectBytes(1024L);
+ private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L);
+ private static final String BASE_SNAPSHOT_CLASS_NAME =
"org.apache.iceberg.BaseSnapshot";
+ private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS =
+ loadBaseSnapshotRetainedCacheFields();
+ // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog
supplier
+ // (REST snapshot-loading-mode=refs). Publication must not perform that IO.
+ private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD =
+ loadTableMetadataField("snapshotsLoaded", boolean.class);
+ private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD =
+ loadTableMetadataField("snapshotsSupplier", null);
+ // The formulas above are built on the Iceberg 1.10.1 instance-field
layouts of the classes a
+ // cached table retains. Every non-static field is pinned, not only the
transient lazy ones: a
+ // library upgrade that adds a retained reference makes weighted admission
fail closed.
+ private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED =
checkIcebergLayout();
+
+ private IcebergCacheSizeEstimator() {
+ }
+
+ private static long objectBytes(long bytes) {
+ return MetaCacheWeightUtils.estimatedObjectBytes(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key,
IcebergTableCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ Table table = value.getRetainedIcebergTable();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ KEY_BASE_BYTES,
MetaCacheWeightUtils.estimatedNameMappingBytes(key));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedTablePayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedCurrentSnapshotPayloadBytes());
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateSnapshotEntry(
+ IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ long bytes = KEY_BASE_BYTES;
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation()));
+
+ IcebergPartitionInfo partitionInfo = value.getPartitionInfo();
+ bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(),
PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES);
Review Comment:
[P2] Account for every merged partition alias
`mergeOverlapPartitions()` can put many physical month/day/hour partition
names into one surviving range's `HashSet`, but this term scales only with the
outer map size. The physical-partition payload charge reaches the strings, not
the retained set's backing table and one node per alias, so a large overlap
group is increasingly underweighted. Please charge the sum of the set
cardinalities (including table capacity) and add a one-to-many overlap JOL
calibration; the current snapshot fixture always uses an empty alias map.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java:
##########
@@ -0,0 +1,1194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.datasource.NameMapping;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
+import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
+import org.apache.doris.datasource.metacache.MetaCacheWeightUtils;
+
+import org.apache.iceberg.BlobMetadata;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StatisticsFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.UnknownTransform;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Publication-time retained-weight formulas for Iceberg cache entries. */
+final class IcebergCacheSizeEstimator {
+ // Calibrated against JOL retained-graph deltas in
IcebergExternalMetaCacheTest.
+ // Every metadata element visited (field, type, snapshot, summary entry,
...) costs a few
+ // reads; the bound only guards against pathological metadata and is far
above real tables
+ // (a 10,000-snapshot history with 15 summary keys each is 160,000
elements). Exceeding it
+ // rejects weighted admission, so it must not be reachable by ordinary
long-lived tables.
+ private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L;
+ // Total name characters the estimator may lower-case while reserving
case-insensitive indexes.
+ private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L;
+ private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128;
+ private static final long KEY_BASE_BYTES = objectBytes(128L);
+ private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L);
+ // TableMetadata-side share of one schema version: schemas list slot and
schemasById entry,
+ // including the growth of both from their singleton to their regular
immutable shapes.
+ private static final long SCHEMA_VERSION_BYTES = objectBytes(128L);
+ private static final long PARTITION_SPEC_BYTES = objectBytes(256L);
+ // Exact active-layout sizes of the Iceberg/Guava objects that lazy
partition, sort and
+ // schema state allocates. Iceberg 1.10.1 field layouts are pinned by
ICEBERG_LAZY_LAYOUT_SUPPORTED.
+ private static final long PARTITION_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L);
+ private static final long SORT_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ // Identity/Bucket/Truncate transforms are allocated per parsed field;
time transforms are enums.
+ private static final long TRANSFORM_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long NESTED_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L);
+ private static final long STRUCT_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L);
+ private static final long SCHEMA_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L);
+ private static final long IMMUTABLE_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long IMMUTABLE_MAP_KEY_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SINGLETON_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long REGULAR_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L);
+ private static final long ARRAY_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L);
+ private static final long HASH_MAP_NODE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ private static final long HASH_MAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L);
+ private static final long INTEGER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long LONG_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate[";
+ // Truncate on a decimal source retains a BigInteger width (object plus
one-int magnitude).
+ private static final long TRUNCATE_WIDTH_BYTES =
MetaCacheWeightUtils.saturatedAdd(
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L),
+ MetaCacheWeightUtils.estimatedIntArrayBytes(1L));
+ private static final long LIST_MULTIMAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L);
+ private static final long CAPTURING_SUPPLIER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long POSITION_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L);
+ // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor.
Required ancestors
+ // collapse into a single Position2/3Accessor that replaces the inner
accessor, which retains
+ // less than this per-level reservation.
+ private static final long WRAPPED_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L);
+ private static final long LIST_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long MAP_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L);
+ private static final long DECIMAL_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final long FIXED_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long GEOMETRY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long GEOGRAPHY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SORT_ORDER_BYTES = objectBytes(256L);
+ private static final long TABLE_PROPERTY_BYTES = objectBytes(40L);
+ private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L);
+ private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L);
+ private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L);
+ private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L);
+ private static final long SNAPSHOT_REF_BYTES = objectBytes(128L);
+ private static final long STATISTICS_FILE_BYTES = objectBytes(512L);
+ private static final long BLOB_METADATA_BYTES = objectBytes(128L);
+ private static final long BLOB_FIELD_BYTES = objectBytes(32L);
+ private static final long PARTITION_STATISTICS_FILE_BYTES =
objectBytes(256L);
+ private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L);
+ // One retained IcebergPartition (value/transform ArrayLists) or one
RangePartitionItem with a
+ // single partition column plus its map entry; extra columns are charged
by IcebergPartitionInfo.
+ private static final long PARTITION_BYTES = objectBytes(680L);
+ private static final long PARTITION_ALIAS_BYTES = objectBytes(256L);
+ private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L);
+ private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L);
+ private static final long DATA_FILE_BYTES = objectBytes(896L);
+ private static final long DELETE_FILE_BYTES = objectBytes(1024L);
+ private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L);
+ private static final String BASE_SNAPSHOT_CLASS_NAME =
"org.apache.iceberg.BaseSnapshot";
+ private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS =
+ loadBaseSnapshotRetainedCacheFields();
+ // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog
supplier
+ // (REST snapshot-loading-mode=refs). Publication must not perform that IO.
+ private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD =
+ loadTableMetadataField("snapshotsLoaded", boolean.class);
+ private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD =
+ loadTableMetadataField("snapshotsSupplier", null);
+ // The formulas above are built on the Iceberg 1.10.1 instance-field
layouts of the classes a
+ // cached table retains. Every non-static field is pinned, not only the
transient lazy ones: a
+ // library upgrade that adds a retained reference makes weighted admission
fail closed.
+ private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED =
checkIcebergLayout();
+
+ private IcebergCacheSizeEstimator() {
+ }
+
+ private static long objectBytes(long bytes) {
+ return MetaCacheWeightUtils.estimatedObjectBytes(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key,
IcebergTableCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ Table table = value.getRetainedIcebergTable();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ KEY_BASE_BYTES,
MetaCacheWeightUtils.estimatedNameMappingBytes(key));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedTablePayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedCurrentSnapshotPayloadBytes());
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateSnapshotEntry(
+ IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ long bytes = KEY_BASE_BYTES;
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation()));
+
+ IcebergPartitionInfo partitionInfo = value.getPartitionInfo();
+ bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(),
PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
partitionInfo.getRetainedPayloadBytes());
+ bytes = addCount(bytes,
value.getNameMapping().map(Map::size).orElse(0),
+ NAME_MAPPING_ENTRY_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedNameMappingPayloadBytes());
+
+ if (value.getRetainedIcebergTable().isPresent()) {
+ Table table = value.getRetainedIcebergTable().get();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
estimateTable(table));
Review Comment:
[P2] Avoid reserving the shared table graph twice
On the weighted path, the table entry has already retained and charged this
non-growing table plus its current-snapshot JSON. getSnapshotCache() passes
those same object references into the snapshot value, yet this branch charges
the full graph again; the two reservations are summed in one catalog/global
budget. A cap that fits the actual table graph and partition projection can
therefore reject the projection and reload its partitions on every miss. Please
use shared/handoff accounting (or couple the entry lifetimes) so the payload
stays charged while either owner survives but only once while both coexist,
with a simultaneous-admission/independent-eviction test.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java:
##########
@@ -0,0 +1,1194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.datasource.NameMapping;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
+import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
+import org.apache.doris.datasource.metacache.MetaCacheWeightUtils;
+
+import org.apache.iceberg.BlobMetadata;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StatisticsFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.UnknownTransform;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Publication-time retained-weight formulas for Iceberg cache entries. */
+final class IcebergCacheSizeEstimator {
+ // Calibrated against JOL retained-graph deltas in
IcebergExternalMetaCacheTest.
+ // Every metadata element visited (field, type, snapshot, summary entry,
...) costs a few
+ // reads; the bound only guards against pathological metadata and is far
above real tables
+ // (a 10,000-snapshot history with 15 summary keys each is 160,000
elements). Exceeding it
+ // rejects weighted admission, so it must not be reachable by ordinary
long-lived tables.
+ private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L;
+ // Total name characters the estimator may lower-case while reserving
case-insensitive indexes.
+ private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L;
+ private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128;
+ private static final long KEY_BASE_BYTES = objectBytes(128L);
+ private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L);
+ // TableMetadata-side share of one schema version: schemas list slot and
schemasById entry,
+ // including the growth of both from their singleton to their regular
immutable shapes.
+ private static final long SCHEMA_VERSION_BYTES = objectBytes(128L);
+ private static final long PARTITION_SPEC_BYTES = objectBytes(256L);
+ // Exact active-layout sizes of the Iceberg/Guava objects that lazy
partition, sort and
+ // schema state allocates. Iceberg 1.10.1 field layouts are pinned by
ICEBERG_LAZY_LAYOUT_SUPPORTED.
+ private static final long PARTITION_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L);
+ private static final long SORT_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ // Identity/Bucket/Truncate transforms are allocated per parsed field;
time transforms are enums.
+ private static final long TRANSFORM_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long NESTED_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L);
+ private static final long STRUCT_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L);
+ private static final long SCHEMA_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L);
+ private static final long IMMUTABLE_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long IMMUTABLE_MAP_KEY_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SINGLETON_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long REGULAR_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L);
+ private static final long ARRAY_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L);
+ private static final long HASH_MAP_NODE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ private static final long HASH_MAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L);
+ private static final long INTEGER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long LONG_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate[";
+ // Truncate on a decimal source retains a BigInteger width (object plus
one-int magnitude).
+ private static final long TRUNCATE_WIDTH_BYTES =
MetaCacheWeightUtils.saturatedAdd(
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L),
+ MetaCacheWeightUtils.estimatedIntArrayBytes(1L));
+ private static final long LIST_MULTIMAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L);
+ private static final long CAPTURING_SUPPLIER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long POSITION_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L);
+ // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor.
Required ancestors
+ // collapse into a single Position2/3Accessor that replaces the inner
accessor, which retains
+ // less than this per-level reservation.
+ private static final long WRAPPED_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L);
+ private static final long LIST_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long MAP_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L);
+ private static final long DECIMAL_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final long FIXED_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long GEOMETRY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long GEOGRAPHY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SORT_ORDER_BYTES = objectBytes(256L);
+ private static final long TABLE_PROPERTY_BYTES = objectBytes(40L);
+ private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L);
+ private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L);
+ private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L);
+ private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L);
+ private static final long SNAPSHOT_REF_BYTES = objectBytes(128L);
+ private static final long STATISTICS_FILE_BYTES = objectBytes(512L);
+ private static final long BLOB_METADATA_BYTES = objectBytes(128L);
+ private static final long BLOB_FIELD_BYTES = objectBytes(32L);
+ private static final long PARTITION_STATISTICS_FILE_BYTES =
objectBytes(256L);
+ private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L);
+ // One retained IcebergPartition (value/transform ArrayLists) or one
RangePartitionItem with a
+ // single partition column plus its map entry; extra columns are charged
by IcebergPartitionInfo.
+ private static final long PARTITION_BYTES = objectBytes(680L);
+ private static final long PARTITION_ALIAS_BYTES = objectBytes(256L);
+ private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L);
+ private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L);
+ private static final long DATA_FILE_BYTES = objectBytes(896L);
+ private static final long DELETE_FILE_BYTES = objectBytes(1024L);
+ private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L);
+ private static final String BASE_SNAPSHOT_CLASS_NAME =
"org.apache.iceberg.BaseSnapshot";
+ private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS =
+ loadBaseSnapshotRetainedCacheFields();
+ // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog
supplier
+ // (REST snapshot-loading-mode=refs). Publication must not perform that IO.
+ private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD =
+ loadTableMetadataField("snapshotsLoaded", boolean.class);
+ private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD =
+ loadTableMetadataField("snapshotsSupplier", null);
+ // The formulas above are built on the Iceberg 1.10.1 instance-field
layouts of the classes a
+ // cached table retains. Every non-static field is pinned, not only the
transient lazy ones: a
+ // library upgrade that adds a retained reference makes weighted admission
fail closed.
+ private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED =
checkIcebergLayout();
+
+ private IcebergCacheSizeEstimator() {
+ }
+
+ private static long objectBytes(long bytes) {
+ return MetaCacheWeightUtils.estimatedObjectBytes(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key,
IcebergTableCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ Table table = value.getRetainedIcebergTable();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ KEY_BASE_BYTES,
MetaCacheWeightUtils.estimatedNameMappingBytes(key));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedTablePayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedCurrentSnapshotPayloadBytes());
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateSnapshotEntry(
+ IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ long bytes = KEY_BASE_BYTES;
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation()));
+
+ IcebergPartitionInfo partitionInfo = value.getPartitionInfo();
+ bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(),
PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
partitionInfo.getRetainedPayloadBytes());
+ bytes = addCount(bytes,
value.getNameMapping().map(Map::size).orElse(0),
Review Comment:
[P2] Scale name-mapping weight with aliases per field
The snapshot copy retains an ImmutableList of every MappedField.names()
value for each field ID, but this term scales only with the number of IDs.
retainedNameMappingPayloadBytes covers the alias strings, not the list's
element array/slots, so one field with many historical names grows the retained
graph while its structural weight stays fixed. Please sum the inner-list
cardinalities/capacities and add a fixed-field-count JOL delta that increases
aliases per field.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java:
##########
@@ -0,0 +1,1194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.datasource.NameMapping;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
+import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
+import org.apache.doris.datasource.metacache.MetaCacheWeightUtils;
+
+import org.apache.iceberg.BlobMetadata;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StatisticsFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.UnknownTransform;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Publication-time retained-weight formulas for Iceberg cache entries. */
+final class IcebergCacheSizeEstimator {
+ // Calibrated against JOL retained-graph deltas in
IcebergExternalMetaCacheTest.
+ // Every metadata element visited (field, type, snapshot, summary entry,
...) costs a few
+ // reads; the bound only guards against pathological metadata and is far
above real tables
+ // (a 10,000-snapshot history with 15 summary keys each is 160,000
elements). Exceeding it
+ // rejects weighted admission, so it must not be reachable by ordinary
long-lived tables.
+ private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L;
+ // Total name characters the estimator may lower-case while reserving
case-insensitive indexes.
+ private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L;
+ private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128;
+ private static final long KEY_BASE_BYTES = objectBytes(128L);
+ private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L);
+ // TableMetadata-side share of one schema version: schemas list slot and
schemasById entry,
+ // including the growth of both from their singleton to their regular
immutable shapes.
+ private static final long SCHEMA_VERSION_BYTES = objectBytes(128L);
+ private static final long PARTITION_SPEC_BYTES = objectBytes(256L);
+ // Exact active-layout sizes of the Iceberg/Guava objects that lazy
partition, sort and
+ // schema state allocates. Iceberg 1.10.1 field layouts are pinned by
ICEBERG_LAZY_LAYOUT_SUPPORTED.
+ private static final long PARTITION_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L);
+ private static final long SORT_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ // Identity/Bucket/Truncate transforms are allocated per parsed field;
time transforms are enums.
+ private static final long TRANSFORM_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long NESTED_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L);
+ private static final long STRUCT_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L);
+ private static final long SCHEMA_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L);
+ private static final long IMMUTABLE_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long IMMUTABLE_MAP_KEY_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SINGLETON_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long REGULAR_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L);
+ private static final long ARRAY_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L);
+ private static final long HASH_MAP_NODE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ private static final long HASH_MAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L);
+ private static final long INTEGER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long LONG_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate[";
+ // Truncate on a decimal source retains a BigInteger width (object plus
one-int magnitude).
+ private static final long TRUNCATE_WIDTH_BYTES =
MetaCacheWeightUtils.saturatedAdd(
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L),
+ MetaCacheWeightUtils.estimatedIntArrayBytes(1L));
+ private static final long LIST_MULTIMAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L);
+ private static final long CAPTURING_SUPPLIER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long POSITION_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L);
+ // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor.
Required ancestors
+ // collapse into a single Position2/3Accessor that replaces the inner
accessor, which retains
+ // less than this per-level reservation.
+ private static final long WRAPPED_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L);
+ private static final long LIST_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long MAP_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L);
+ private static final long DECIMAL_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final long FIXED_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long GEOMETRY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long GEOGRAPHY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SORT_ORDER_BYTES = objectBytes(256L);
+ private static final long TABLE_PROPERTY_BYTES = objectBytes(40L);
+ private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L);
+ private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L);
+ private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L);
+ private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L);
+ private static final long SNAPSHOT_REF_BYTES = objectBytes(128L);
+ private static final long STATISTICS_FILE_BYTES = objectBytes(512L);
+ private static final long BLOB_METADATA_BYTES = objectBytes(128L);
+ private static final long BLOB_FIELD_BYTES = objectBytes(32L);
+ private static final long PARTITION_STATISTICS_FILE_BYTES =
objectBytes(256L);
+ private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L);
+ // One retained IcebergPartition (value/transform ArrayLists) or one
RangePartitionItem with a
+ // single partition column plus its map entry; extra columns are charged
by IcebergPartitionInfo.
+ private static final long PARTITION_BYTES = objectBytes(680L);
+ private static final long PARTITION_ALIAS_BYTES = objectBytes(256L);
+ private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L);
+ private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L);
+ private static final long DATA_FILE_BYTES = objectBytes(896L);
+ private static final long DELETE_FILE_BYTES = objectBytes(1024L);
+ private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L);
+ private static final String BASE_SNAPSHOT_CLASS_NAME =
"org.apache.iceberg.BaseSnapshot";
+ private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS =
+ loadBaseSnapshotRetainedCacheFields();
+ // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog
supplier
+ // (REST snapshot-loading-mode=refs). Publication must not perform that IO.
+ private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD =
+ loadTableMetadataField("snapshotsLoaded", boolean.class);
+ private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD =
+ loadTableMetadataField("snapshotsSupplier", null);
+ // The formulas above are built on the Iceberg 1.10.1 instance-field
layouts of the classes a
+ // cached table retains. Every non-static field is pinned, not only the
transient lazy ones: a
+ // library upgrade that adds a retained reference makes weighted admission
fail closed.
+ private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED =
checkIcebergLayout();
+
+ private IcebergCacheSizeEstimator() {
+ }
+
+ private static long objectBytes(long bytes) {
+ return MetaCacheWeightUtils.estimatedObjectBytes(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key,
IcebergTableCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ Table table = value.getRetainedIcebergTable();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ KEY_BASE_BYTES,
MetaCacheWeightUtils.estimatedNameMappingBytes(key));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedTablePayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedCurrentSnapshotPayloadBytes());
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateSnapshotEntry(
+ IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ long bytes = KEY_BASE_BYTES;
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation()));
+
+ IcebergPartitionInfo partitionInfo = value.getPartitionInfo();
+ bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(),
PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES);
+ bytes = addCount(bytes,
partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
partitionInfo.getRetainedPayloadBytes());
+ bytes = addCount(bytes,
value.getNameMapping().map(Map::size).orElse(0),
+ NAME_MAPPING_ENTRY_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedNameMappingPayloadBytes());
+
+ if (value.getRetainedIcebergTable().isPresent()) {
+ Table table = value.getRetainedIcebergTable().get();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
estimateTable(table));
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedTablePayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedCurrentSnapshotPayloadBytes());
+ }
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateManifestEntry(
+ IcebergManifestEntryKey key, ManifestCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ if (!value.isAccountingComplete()) {
+ return
MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete");
+ }
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ MANIFEST_ENTRY_BASE_BYTES,
+
MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath()));
+ bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_BYTES);
+ bytes = addCount(bytes, value.getDeleteFiles().size(),
DELETE_FILE_BYTES);
+ bytes = addCount(bytes, value.getDataFileMetricEntryCount(),
FILE_METRIC_ENTRY_BYTES);
+ bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(),
FILE_METRIC_ENTRY_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedPayloadBytes());
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ private static MetaCacheSizeEstimate checkJvmObjectLayout() {
+ return MetaCacheWeightUtils.isSupportedJvmObjectLayout()
+ ? MetaCacheSizeEstimate.complete(1L)
+ :
MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment");
+ }
+
+ private static MetaCacheSizeEstimate checkSupportedTable(Table table) {
+ if (!ICEBERG_LAZY_LAYOUT_SUPPORTED) {
+ return
MetaCacheSizeEstimate.incomplete("unsupported_iceberg_lazy_layout");
+ }
+ if (table == null) {
+ return MetaCacheSizeEstimate.incomplete("missing_iceberg_table");
+ }
+ if (!(table instanceof HasTableOperations)) {
+ return MetaCacheSizeEstimate.incomplete(
+ "unsupported_iceberg_table:" + table.getClass().getName());
+ }
+ TableMetadata metadata = ((HasTableOperations)
table).operations().current();
+ if (metadata == null) {
+ return
MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata");
+ }
+ if (!areSnapshotsLoaded(metadata)) {
+ return
MetaCacheSizeEstimate.incomplete("iceberg_snapshots_not_loaded");
+ }
+ if (metadata.metadataFileLocation() == null
+ || metadata.metadataFileLocation().isEmpty()) {
+ return
MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location");
+ }
+ return MetaCacheSizeEstimate.complete(1L);
+ }
+
+ /** Reads only metadata collection sizes and a constant number of strings;
no FileIO is used. */
+ private static long estimateTable(Table table) {
+ TableMetadata metadata = ((HasTableOperations)
table).operations().current();
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ TABLE_BASE_BYTES,
MetaCacheWeightUtils.estimatedStringBytes(table.name()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(metadata.location()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(metadata.metadataFileLocation()));
+
+ bytes = addCount(bytes, metadata.properties().size(),
TABLE_PROPERTY_BYTES);
+ if (metadata.currentSnapshot() != null) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
CURRENT_SNAPSHOT_BYTES);
+ }
+ return bytes;
+ }
+
+ /**
+ * Fully accounts variable payload with a bounded amount of
publication-time work. Only
+ * already-parsed metadata is read; the SDK state it touches on the way
(StructType, ListType
+ * and MapType fieldList copies, the identifier field set) is small,
accounted and O(N).
+ */
+ static long retainedTablePayloadBytes(Table table) {
+ if (!(table instanceof HasTableOperations)) {
+ return 0L;
+ }
+ TableMetadata metadata = ((HasTableOperations)
table).operations().current();
+ if (metadata == null) {
+ return 0L;
+ }
+ if (!areSnapshotsLoaded(metadata)) {
+ // snapshots()/refs() would call the catalog's lazy snapshot
supplier: fail closed.
+ throw new IllegalStateException("Iceberg table snapshots are not
loaded");
+ }
+
+ long bytes = 0L;
+ AccountingBudget budget = new AccountingBudget(
+ MAX_TABLE_ACCOUNTING_ELEMENTS,
MAX_TABLE_ACCOUNTING_CHARACTERS);
+ for (PartitionSpec spec : metadata.specs()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
partitionSpecBytes(spec, budget));
+ }
+ for (SortOrder sortOrder : metadata.sortOrders()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
sortOrderBytes(sortOrder, budget));
+ }
+ for (Schema schema : metadata.schemas()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
schemaBytes(schema, budget));
+ }
+ budget.chargeElements(metadata.properties().size());
+ for (Map.Entry<String, String> property :
metadata.properties().entrySet()) {
+ bytes = addString(bytes, property.getKey());
+ bytes = addString(bytes, property.getValue());
+ }
+ for (Snapshot snapshot : metadata.snapshots()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
snapshotBytes(snapshot, budget));
+ }
+ budget.chargeElements(metadata.snapshotLog().size());
+ bytes = addCount(bytes, metadata.snapshotLog().size(),
SNAPSHOT_LOG_ENTRY_BYTES);
+ budget.chargeElements(metadata.previousFiles().size());
+ for (TableMetadata.MetadataLogEntry previousFile :
metadata.previousFiles()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
METADATA_LOG_ENTRY_BYTES);
+ bytes = addString(bytes, previousFile.file());
+ }
+ budget.chargeElements(metadata.refs().size());
+ bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES);
+ for (String refName : metadata.refs().keySet()) {
+ bytes = addString(bytes, refName);
+ }
+ budget.chargeElements(metadata.statisticsFiles().size());
+ for (StatisticsFile statisticsFile : metadata.statisticsFiles()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
STATISTICS_FILE_BYTES);
+ bytes = addString(bytes, statisticsFile.path());
+ for (BlobMetadata blob : statisticsFile.blobMetadata()) {
+ budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L,
+ MetaCacheWeightUtils.saturatedAdd(
+ blob.fields().size(),
blob.properties().size())));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
BLOB_METADATA_BYTES);
+ bytes = addString(bytes, blob.type());
+ bytes = addCount(bytes, blob.fields().size(),
BLOB_FIELD_BYTES);
+ bytes = addStringMap(bytes, blob.properties(),
TABLE_PROPERTY_BYTES);
+ }
+ }
+ budget.chargeElements(metadata.partitionStatisticsFiles().size());
+ for (PartitionStatisticsFile statisticsFile :
metadata.partitionStatisticsFiles()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
PARTITION_STATISTICS_FILE_BYTES);
+ bytes = addString(bytes, statisticsFile.path());
+ }
+ budget.chargeElements(metadata.encryptionKeys().size());
+ for (EncryptedKey encryptedKey : metadata.encryptionKeys()) {
+ budget.chargeElements(encryptedKey.properties().size());
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
ENCRYPTED_KEY_BYTES);
+ bytes = addString(bytes, encryptedKey.keyId());
+ bytes = addString(bytes, encryptedKey.encryptedById());
+ bytes = addBufferPayload(bytes,
encryptedKey.encryptedKeyMetadata());
+ bytes = addStringMap(bytes, encryptedKey.properties(),
TABLE_PROPERTY_BYTES);
+ }
+ bytes = addString(bytes, metadata.uuid());
+ return bytes;
+ }
+
+ /**
+ * Account a PartitionSpec together with the lazy state that a normal scan
materializes after
+ * admission: fieldList, javaClasses, partitionType() with its StructType
indexes, the secondary
+ * Schema/Binder graph behind partitionType().asSchema() and
fieldsBySourceId. Iceberg 1.10.1
+ * allocates one Object[fieldCount] per distinct source id inside
fieldsBySourceId, so that
+ * retained graph is O(distinctSourceIds * fieldCount); it is reserved
here in O(fieldCount)
+ * publication work without materializing any of it.
+ */
+ private static long partitionSpecBytes(PartitionSpec spec,
AccountingBudget budget) {
+ List<PartitionField> fields = spec.fields();
+ long fieldCount = fields.size();
+ budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L,
fieldCount));
+ long bytes = PARTITION_SPEC_BYTES;
+ if (fieldCount == 0L) {
+ return bytes;
+ }
+ Set<Integer> distinctSourceIds = new HashSet<>();
+ long uncachedSourceIds = 0L;
+ long uncachedFieldIds = 0L;
+ long lowerCaseNameBytes = 0L;
+ for (PartitionField field : fields) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
PARTITION_FIELD_BYTES);
+ bytes = addTransformPayload(bytes, field.transform());
+ bytes = addString(bytes, field.name());
+ lowerCaseNameBytes = MetaCacheWeightUtils.saturatedAdd(
+ lowerCaseNameBytes,
generatedLowerCaseNameBytes(field.name(), budget));
+ if (isUncachedInteger(field.fieldId())) {
+ uncachedFieldIds++;
+ }
+ if (distinctSourceIds.add(field.sourceId()) &&
isUncachedInteger(field.sourceId())) {
+ uncachedSourceIds++;
+ }
+ }
+ // Eager PartitionField[] plus lazy fieldList and javaClasses.
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
objectArrayGrowthBytes(fieldCount));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
immutableListBytes(fieldCount));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
objectArrayGrowthBytes(fieldCount));
+ // partitionType(): the StructType itself also exists for an
unpartitioned spec.
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
objectArrayGrowthBytes(fieldCount));
+ bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ structTypeIndexBytes(fieldCount, uncachedFieldIds,
lowerCaseNameBytes));
+ if (!spec.schema().idsToOriginal().isEmpty()) {
+ // rawPartitionType() rebuilds the struct with original ids.
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
STRUCT_TYPE_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes,
MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount));
+ bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES);
+ }
+ // A partition filter binds against partitionType().asSchema().
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes(
+ SchemaShape.flat(fieldCount, uncachedFieldIds,
lowerCaseNameBytes)));
+ // fieldsBySourceId: HashMap<Integer, ArrayList(capacity =
fieldCount)>.
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LIST_MULTIMAP_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
CAPTURING_SUPPLIER_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ hashIdMapBytes(distinctSourceIds.size(), uncachedSourceIds));
+ return addCount(bytes, distinctSourceIds.size(),
+ MetaCacheWeightUtils.saturatedAdd(ARRAY_LIST_BYTES,
+
MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)));
+ }
+
+ /** Account a SortOrder: SortField[] with per-field transforms plus the
lazy fieldList copy. */
+ private static long sortOrderBytes(SortOrder sortOrder, AccountingBudget
budget) {
+ long fieldCount = sortOrder.fields().size();
+ budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L,
fieldCount));
+ long bytes = SORT_ORDER_BYTES;
+ if (fieldCount == 0L) {
+ return bytes;
+ }
+ for (SortField field : sortOrder.fields()) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_FIELD_BYTES);
+ bytes = addTransformPayload(bytes, field.transform());
+ }
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
objectArrayGrowthBytes(fieldCount));
+ return MetaCacheWeightUtils.saturatedAdd(bytes,
immutableListBytes(fieldCount));
+ }
+
+ /** Transform instance plus the payload only some transforms retain. */
+ private static long addTransformPayload(long bytes, Transform<?, ?>
transform) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TRANSFORM_BYTES);
+ if (transform instanceof UnknownTransform) {
+ return addString(bytes, transform.toString());
+ }
+ if (transform.toString().startsWith(TRUNCATE_TRANSFORM_PREFIX)) {
+ // Truncate is package-private; its serialized name is the SPI
contract.
+ return MetaCacheWeightUtils.saturatedAdd(bytes,
TRUNCATE_WIDTH_BYTES);
+ }
+ return bytes;
+ }
+
+ /** Lazy StructType indexes: fieldList, fieldsByName,
fieldsByLowerCaseName and fieldsById. */
+ private static long structTypeIndexBytes(
+ long fieldCount, long uncachedFieldIds, long lowerCaseNameBytes) {
+ long bytes = immutableListBytes(fieldCount);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ immutableNameMapBytes(fieldCount, 0L, 0L));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ immutableNameMapBytes(fieldCount, 0L, lowerCaseNameBytes));
+ return MetaCacheWeightUtils.saturatedAdd(bytes,
+ immutableNameMapBytes(fieldCount, uncachedFieldIds, 0L));
+ }
+
+ /**
+ * The Schema created by StructType.asSchema(): its constructor
materializes idToName and two
+ * empty id maps; Binder and projection paths add nameToId,
lowerCaseNameToId, idToField and
+ * idToAccessor; its own StructType copy grows the same lazy indexes as
the root struct.
+ */
+ private static long secondarySchemaBytes(SchemaShape shape) {
+ long bytes = schemaObjectBytes(shape);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
schemaLookupBytes(shape));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
schemaLazyIndexBytes(shape));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount));
+ return MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes(
+ shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount,
+ shape.topLevelLowerCaseStringBytes));
+ }
+
+ /** Schema object, empty identifier int[], the two empty id maps and the
eager idToName keySet. */
+ private static long schemaObjectBytes(SchemaShape shape) {
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ SCHEMA_BYTES, MetaCacheWeightUtils.estimatedIntArrayBytes(0L));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES);
+ if (shape.fieldCount == 1L) {
+ return MetaCacheWeightUtils.saturatedAdd(bytes,
SINGLETON_IMMUTABLE_SET_BYTES);
+ }
+ return shape.fieldCount > 1L
+ ? MetaCacheWeightUtils.saturatedAdd(bytes,
IMMUTABLE_MAP_KEY_SET_BYTES) : bytes;
+ }
+
+ /**
+ * idToName (eager in the constructor), nameToId and idToField. Every map
boxes uncached ids
+ * itself; idToName and nameToId each retain their own copy of every
nested canonical name and
+ * nameToId also retains the short aliases.
+ */
+ private static long schemaLookupBytes(SchemaShape shape) {
+ long bytes = immutableNameMapBytes(
+ shape.fieldCount, shape.uncachedFieldIdCount,
shape.pathStringBytes);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableNameMapBytes(
+ shape.nameEntryCount, shape.uncachedNameIdCount,
+ MetaCacheWeightUtils.saturatedAdd(
+ shape.pathStringBytes, shape.aliasStringBytes)));
+ return MetaCacheWeightUtils.saturatedAdd(bytes,
+ hashIdMapBytes(shape.fieldCount, shape.uncachedFieldIdCount));
+ }
+
+ /** lowerCaseNameToId and idToAccessor, materialized by case-insensitive
lookups and Binder. */
+ private static long schemaLazyIndexBytes(SchemaShape shape) {
+ long bytes = immutableNameMapBytes(
+ shape.nameEntryCount, shape.uncachedNameIdCount,
shape.lowerCaseStringBytes);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ hashIdMapBytes(shape.accessorFieldCount,
shape.uncachedAccessorIdCount));
+ bytes = addCount(bytes, shape.accessorFieldCount,
POSITION_ACCESSOR_BYTES);
+ return addCount(bytes, shape.wrappedAccessorCount,
WRAPPED_ACCESSOR_BYTES);
+ }
+
+ /**
+ * One table schema version with every index a normal scan can materialize
afterwards. Only
+ * metadata already parsed is read; nothing lazy is touched, and each
field is visited once.
+ */
+ private static long schemaBytes(Schema schema, AccountingBudget budget) {
+ budget.chargeElements(1L);
+ SchemaShape shape = new SchemaShape();
+ long bytes = SCHEMA_VERSION_BYTES;
+ for (Types.NestedField field : schema.columns()) {
+ bytes = addFieldPayload(
+ bytes, field, PathState.ROOT, FieldKind.STRUCT_FIELD,
budget, shape);
+ }
+ Set<Integer> identifierFieldIds = schema.identifierFieldIds();
+ budget.chargeElements(identifierFieldIds.size());
+ bytes = addIdentifierFieldPayload(bytes, identifierFieldIds);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
shape.typeObjectBytes);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
schemaObjectBytes(shape));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount));
+ if (shape.fieldCount == 0L) {
+ // Nothing can be looked up in an empty schema; its indexes stay
shared singletons.
+ return bytes;
+ }
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
schemaLookupBytes(shape));
+ // Future lazy growth: main lookups, root struct indexes and the
asSchema() secondary graph.
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
schemaLazyIndexBytes(shape));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes(
+ shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount,
+ shape.topLevelLowerCaseStringBytes));
+ return MetaCacheWeightUtils.saturatedAdd(bytes,
secondarySchemaBytes(shape));
+ }
+
+ /** ImmutableList.copyOf(array): shared empty, singleton, or a list object
plus copied array. */
+ private static long immutableListBytes(long elementCount) {
+ if (elementCount <= 0L) {
+ return 0L;
+ }
+ if (elementCount == 1L) {
+ return IMMUTABLE_LIST_BYTES;
+ }
+ return MetaCacheWeightUtils.saturatedAdd(IMMUTABLE_LIST_BYTES,
+ MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount));
+ }
+
+ /** Growth of a reference array that replaces an empty array retained by
the empty shape. */
+ private static long objectArrayGrowthBytes(long elementCount) {
+ long populated =
MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount);
+ long empty = MetaCacheWeightUtils.estimatedObjectArrayBytes(0L);
+ return populated == Long.MAX_VALUE ? populated : populated - empty;
+ }
+
+ /** Boxed Integer keys outside the JVM Integer cache are retained per
lookup map. */
+ private static boolean isUncachedInteger(int value) {
+ return value < -128 || value > 127;
+ }
+
+ /**
+ * Retained bytes of one lower-cased copy of a name, or 0 when the name is
already lower case
+ * and the index reuses it. Every case-insensitive index (partition
StructType, secondary
+ * Schema and secondary StructType) allocates its own copy, so callers add
this per index.
+ */
+ private static long generatedLowerCaseNameBytes(String name,
AccountingBudget budget) {
+ budget.chargeCharacters(name.length());
+ String lowerName = name.toLowerCase(Locale.ROOT);
+ if (lowerName.equals(name)) {
+ return 0L;
+ }
+ return MetaCacheWeightUtils.estimatedGeneratedStringBytes(
+ lowerName.length(),
MetaCacheWeightUtils.isLatin1String(lowerName));
+ }
+
+ private static long hashIdMapBytes(long entryCount, long uncachedIds) {
+ long bytes = HASH_MAP_BYTES;
+ if (entryCount <= 0L) {
+ // HashMap allocates its table on the first put.
+ return bytes;
+ }
+ bytes = addCount(bytes, entryCount, HASH_MAP_NODE_BYTES);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedObjectArrayBytes(
+ hashMapCapacity(entryCount)));
+ return addCount(bytes, uncachedIds, INTEGER_BYTES);
+ }
+
+ private static long immutableNameMapBytes(
+ long entryCount, long uncachedIds, long generatedStringBytes) {
+ long bytes = 0L;
+ if (entryCount == 1L) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(8L, 0L));
+ } else if (entryCount > 1L) {
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 4L));
+ bytes = addCount(bytes, entryCount,
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L));
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes,
MetaCacheWeightUtils.estimatedObjectArrayBytes(entryCount));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedObjectArrayBytes(
+ immutableMapTableCapacity(entryCount)));
+ }
+ bytes = addCount(bytes, uncachedIds, INTEGER_BYTES);
+ return MetaCacheWeightUtils.saturatedAdd(bytes, generatedStringBytes);
+ }
+
+ private static long snapshotBytes(Snapshot snapshot, AccountingBudget
budget) {
+ rejectMaterializedSnapshotPayload(snapshot);
+ Map<String, String> summary = snapshot.summary();
+ budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(
+ 1L, summary == null ? 0L : summary.size()));
+ long bytes = HISTORICAL_SNAPSHOT_BYTES;
+ // The parsed parent id is never inside the Long cache; the row-id
fields are boxed too
+ // and only tiny values would share a cached instance, so each present
field is charged.
+ bytes = addBoxedLong(bytes, snapshot.parentId());
+ bytes = addBoxedLong(bytes, snapshot.firstRowId());
+ bytes = addBoxedLong(bytes, snapshot.addedRows());
+ bytes = addString(bytes, snapshot.operation());
+ String manifestListLocation = snapshot.manifestListLocation();
+ if (manifestListLocation == null) {
+ // A snapshot serialized with an inline "manifests" array (legacy
writers) retains a
+ // String[] of manifest locations that is only exposed through
ManifestFile wrappers.
+ // Reject weighted admission instead of doing IO or admitting an
underestimate.
+ throw new IllegalStateException(
+ "Iceberg snapshot with inline manifest list is
unsupported");
+ }
+ bytes = addString(bytes, manifestListLocation);
+ bytes = addString(bytes, snapshot.keyId());
+ return addStringMap(bytes, summary, TABLE_PROPERTY_BYTES);
+ }
+
+ private static void rejectMaterializedSnapshotPayload(Snapshot snapshot) {
+ if (!BASE_SNAPSHOT_CLASS_NAME.equals(snapshot.getClass().getName())) {
+ throw new IllegalStateException(
+ "Unsupported Iceberg snapshot implementation: "
+ + snapshot.getClass().getName());
+ }
+ if (BASE_SNAPSHOT_RETAINED_CACHE_FIELDS == null) {
+ throw new IllegalStateException(
+ "Iceberg BaseSnapshot retained-cache inspection is
unavailable");
+ }
+ try {
+ // The field list is resolved once per process. Publication only
performs a bounded
+ // number of O(1) reads and never walks a retained manifest/file
graph.
+ for (Field retainedCacheField :
BASE_SNAPSHOT_RETAINED_CACHE_FIELDS) {
+ if (retainedCacheField.get(snapshot) != null) {
+ throw new IllegalStateException(
+ "Iceberg snapshot has materialized retained
payload: "
+ + retainedCacheField.getName());
+ }
+ }
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(
+ "Cannot inspect Iceberg BaseSnapshot retained payload", e);
+ }
+ }
+
+ /** Iceberg marks snapshots loaded at construction unless a lazy supplier
was configured. */
+ private static boolean areSnapshotsLoaded(TableMetadata metadata) {
+ if (TABLE_METADATA_SNAPSHOTS_LOADED_FIELD == null
+ || TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD == null) {
+ return false;
+ }
+ try {
+ return TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD.get(metadata) ==
null
+ ||
TABLE_METADATA_SNAPSHOTS_LOADED_FIELD.getBoolean(metadata);
+ } catch (IllegalAccessException | RuntimeException e) {
+ return false;
+ }
+ }
+
+ private static Field loadTableMetadataField(String name, Class<?>
expectedType) {
+ try {
+ Field field = TableMetadata.class.getDeclaredField(name);
+ if ((expectedType != null && field.getType() != expectedType)
+ || Modifier.isStatic(field.getModifiers())) {
+ return null;
+ }
+ field.setAccessible(true);
+ return field;
+ } catch (ReflectiveOperationException | RuntimeException e) {
+ return null;
+ }
+ }
+
+ private static Field[] loadBaseSnapshotRetainedCacheFields() {
+ try {
+ Class<?> snapshotClass = Class.forName(
+ BASE_SNAPSHOT_CLASS_NAME, false,
Snapshot.class.getClassLoader());
+ List<Field> retainedCacheFields = new ArrayList<>();
+ for (Field field : snapshotClass.getDeclaredFields()) {
+ int modifiers = field.getModifiers();
+ if (Modifier.isTransient(modifiers) &&
!Modifier.isStatic(modifiers)
+ && !field.getType().isPrimitive()) {
+ field.setAccessible(true);
+ retainedCacheFields.add(field);
+ }
+ }
+ return retainedCacheFields.isEmpty()
+ ? null : retainedCacheFields.toArray(new Field[0]);
+ } catch (ReflectiveOperationException | RuntimeException e) {
+ return null;
+ }
+ }
+
+ private static boolean checkIcebergLayout() {
+ ClassLoader loader = Snapshot.class.getClassLoader();
+ return MetaCacheWeightUtils.hasExpectedInstanceFields(Schema.class,
+ "struct:StructType", "schemaId:int",
"identifierFieldIds:int[]",
+ "highestFieldId:int", "aliasToId:BiMap", "idToField:Map",
"nameToId:Map",
+ "lowerCaseNameToId:Map", "idToAccessor:Map", "idToName:Map",
+ "identifierFieldIdSet:Set", "idsToReassigned:Map",
"idsToOriginal:Map")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionSpec.class,
+ "schema:Schema", "specId:int",
"fields:PartitionField[]",
+ "fieldsBySourceId:ListMultimap",
"lazyJavaClasses:Class[]",
+ "lazyPartitionType:StructType",
"lazyRawPartitionType:StructType",
+ "fieldList:List", "lastAssignedFieldId:int")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionField.class,
+ "sourceId:int", "fieldId:int", "name:String",
"transform:Transform")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(SortOrder.class,
+ "schema:Schema", "orderId:int", "fields:SortField[]",
"fieldList:List")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(SortField.class,
+ "transform:Transform", "sourceId:int",
"direction:SortDirection",
+ "nullOrder:NullOrder")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(Types.StructType.class,
+ "fields:NestedField[]", "schema:Schema",
"fieldList:List",
+ "fieldsByName:Map", "fieldsByLowerCaseName:Map",
"fieldsById:Map")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(Types.ListType.class,
+ "elementField:NestedField", "fields:List")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(Types.MapType.class,
+ "keyField:NestedField", "valueField:NestedField",
"fields:List")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(Types.NestedField.class,
+ "isOptional:boolean", "id:int", "name:String",
"type:Type",
+ "doc:String", "initialDefault:Literal",
"writeDefault:Literal")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(TableMetadata.class,
+ "metadataFileLocation:String", "formatVersion:int",
"uuid:String",
+ "location:String", "lastSequenceNumber:long",
"lastUpdatedMillis:long",
+ "lastColumnId:int", "currentSchemaId:int",
"schemas:List",
+ "defaultSpecId:int", "specs:List",
"lastAssignedPartitionId:int",
+ "defaultSortOrderId:int", "sortOrders:List",
"properties:Map",
+ "currentSnapshotId:long", "schemasById:Map",
"specsById:Map",
+ "sortOrdersById:Map", "snapshotLog:List",
"previousFiles:List",
+ "statisticsFiles:List",
"partitionStatisticsFiles:List", "changes:List",
+ "nextRowId:long", "encryptionKeys:List",
+ "snapshotsSupplier:SerializableSupplier",
"snapshots:List",
+ "snapshotsById:Map", "refs:Map",
"snapshotsLoaded:boolean")
+ &&
MetaCacheWeightUtils.hasExpectedInstanceFields(BASE_SNAPSHOT_CLASS_NAME, loader,
+ "snapshotId:long", "parentId:Long",
"sequenceNumber:long",
+ "timestampMillis:long", "manifestListLocation:String",
+ "operation:String", "summary:Map", "schemaId:Integer",
+ "v1ManifestLocations:String[]", "firstRowId:Long",
"addedRows:Long",
+ "keyId:String", "allManifests:List",
"dataManifests:List",
+ "deleteManifests:List", "addedDataFiles:List",
"removedDataFiles:List",
+ "addedDeleteFiles:List", "removedDeleteFiles:List");
+ }
+
+ private static long addBoxedLong(long bytes, Long value) {
+ return value == null ? bytes :
MetaCacheWeightUtils.saturatedAdd(bytes, LONG_BYTES);
+ }
+
+ private static long addBufferPayload(long bytes, ByteBuffer buffer) {
+ return buffer == null ? bytes :
MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity());
+ }
+
+ /**
+ * Account one NestedField, its owned strings and its type subtree, and
record the shape data
+ * the lookup-map formulas need. Canonical and short names follow
Iceberg's IndexByName: a
+ * nested name joins its ancestors with '.', a struct-typed list element
or map value is left
+ * out of its children's short names (which then become aliases), and
every lower-case index
+ * lower-cases each entry.
+ */
+ private static long addFieldPayload(
+ long bytes, Types.NestedField field, PathState ancestors,
FieldKind kind,
+ AccountingBudget budget, SchemaShape shape) {
+ budget.chargeElements(1L);
+ budget.chargeCharacters(field.name().length());
+ String name = field.name();
+ String lowerName = name.toLowerCase(Locale.ROOT);
+ boolean nameLatin1 = MetaCacheWeightUtils.isLatin1String(name);
+ boolean lowerLatin1 = MetaCacheWeightUtils.isLatin1String(lowerName);
+ shape.addField(field.fieldId(), ancestors, name, nameLatin1,
lowerName, lowerLatin1);
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES);
+ if (kind == FieldKind.STRUCT_FIELD) {
+ // List element and map key/value fields are named by shared
"element"/"key"/"value"
+ // literals inside Iceberg's type constructors.
+ bytes = addString(bytes, name);
+ }
+ bytes = addString(bytes, field.doc());
+ bytes = addDefaultPayload(bytes, field.initialDefault());
Review Comment:
[P2] Account the retained field-default literals
NestedField retains initialDefault and writeDefault as Iceberg Literal<?>
objects (as the layout check above confirms), but these accessors unwrap them
to their values. This helper then handles only strings/binary payloads, so
every literal wrapper is omitted and decimal/UUID/numeric defaults contribute
no value object at all; the gap multiplies across defaulted fields and
historical schemas. Please account the
initialDefaultLiteral()/writeDefaultLiteral() implementations and supported
value layouts (or fail closed), and add JOL deltas with defaulted columns and
schema history.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java:
##########
@@ -198,62 +499,621 @@ public MetaCacheEntryStats stats() {
failureCount,
totalLoadTime,
totalLoadCount == 0 ? 0D : (double) totalLoadTime /
totalLoadCount,
- cacheStats.evictionCount(),
+ MetaCacheWeightUtils.saturatedAdd(
+ cacheStats.evictionCount(), localEvictionCount.get()),
invalidateCount.get(),
lastLoadSuccessTimeMs.get(),
lastLoadFailureTimeMs.get(),
- lastError.get());
+ lastError.get(),
+ weightBounded,
+ weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L,
+ weightBounded ? entryBudget.getUsedWeight() : -1L,
+ weightBounded ? MetaCacheWeightUtils.saturatedAdd(
+ cacheStats.evictionWeight(),
localEvictionWeight.get()) : -1L,
+ weightBounded ? weightAdmissionRejectedCount.get() : -1L,
+ weightBounded ? entryBudget.getCatalogMaxWeight() : -1L,
+ weightBounded ? entryBudget.getCatalogUsedWeight() : -1L,
+ weightBounded ? entryBudget.getGlobalMaxWeight() : -1L,
+ weightBounded ? entryBudget.getGlobalUsedWeight() : -1L,
+ weightBounded ? lastWeightRejectReason.get() : "");
+ }
+
+ public boolean isWeightBounded() {
+ return weightBounded;
+ }
+
+ private AdmissionResult admitWeightedValue(
+ K key, V value, @Nullable V expectedCurrent, boolean
requireExpected,
+ @Nullable KeyMutationToken expectedMutation, long
expectedReservationGeneration,
+ boolean advanceMutationOnAdmission) {
+ if (closed.get()) {
+ return AdmissionResult.DISABLED;
+ }
+ MetaCacheSizeEstimate estimate;
+ try {
+ estimate = Objects.requireNonNull(sizeEstimator.estimate(key,
value), "size estimate");
+ } catch (IllegalArgumentException e) {
+ throw e;
+ } catch (RuntimeException e) {
+ rejectWeight("invalid_estimate");
+ return AdmissionResult.REJECTED;
+ }
+ if (!estimate.isComplete()) {
+ rejectWeight(estimate.getIncompleteReason());
+ return AdmissionResult.REJECTED;
+ }
+
+ long estimatedPayloadBytes = estimate.getBytes();
+ // A retained non-null key/value plus Caffeine node can never consume
zero bytes. Treat a
+ // complete zero as an estimator contract violation so an omitted
formula cannot bypass
+ // every quota and admit an unbounded number of zero-weight entries.
+ if (estimatedPayloadBytes == 0L) {
+ rejectWeight("invalid_estimate");
+ return AdmissionResult.REJECTED;
+ }
+ long newWeight = MetaCacheWeightUtils.saturatedAdd(
+ estimatedPayloadBytes, FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES);
+ synchronized (admissionLock) {
+ if (closed.get()) {
+ return AdmissionResult.DISABLED;
+ }
+ if (expectedMutation != null && !isKeyMutationCurrent(key,
expectedMutation)) {
+ return AdmissionResult.NOT_CURRENT;
+ }
+ V oldValue = data.asMap().get(key);
+ ReservationRecord record = reservations.get(key);
+ if (expectedReservationGeneration >= 0L
+ && (record == null || record.generation !=
expectedReservationGeneration)) {
+ return AdmissionResult.NOT_CURRENT;
+ }
+ if (requireExpected && oldValue != expectedCurrent) {
+ return AdmissionResult.NOT_CURRENT;
+ }
+ if (oldValue == null && record != null) {
+ reservations.remove(key, record);
+ record.reservation.release();
+ record = null;
+ }
+ if (oldValue != null && (record == null || !record.published)) {
+ rejectWeight("missing_reservation");
+ return AdmissionResult.REJECTED;
+ }
+
+ if (record == null) {
+ Optional<AdmissionReservation> reservation =
reserveWithLocalEviction(key, newWeight);
+ if (!reservation.isPresent()) {
+ rejectWeight("budget_exceeded");
+ return AdmissionResult.REJECTED;
+ }
+ ReservationRecord newRecord = new ReservationRecord(
+ newWeight, reservation.get(),
nextReservationGeneration());
+ if (advanceMutationOnAdmission) {
+ advanceKeyMutation(key);
+ }
+ reservations.put(key, newRecord);
+ try {
+ beforeWeightedCachePutForTest(key, value);
+ data.put(key, value);
+ if (reservations.get(key) == newRecord &&
data.asMap().get(key) == value) {
+ newRecord.published = true;
+ notifyReplacement(key, null, value);
+ }
+ return AdmissionResult.ADMITTED;
+ } catch (RuntimeException | Error e) {
+ reservations.remove(key, newRecord);
+ newRecord.reservation.release();
+ throw e;
+ }
+ }
+
+ ReservationRecord previousRecord = record;
+ long reservedWeight = Math.max(previousRecord.weight, newWeight);
+ if (!resizeWithLocalEviction(key, previousRecord.reservation,
reservedWeight)) {
+ rejectWeight("budget_exceeded");
+ return AdmissionResult.REJECTED;
+ }
+ if (advanceMutationOnAdmission) {
+ advanceKeyMutation(key);
+ }
+ ReservationRecord newRecord = new ReservationRecord(
+ newWeight, previousRecord.reservation,
nextReservationGeneration());
+ reservations.put(key, newRecord);
+ try {
+ beforeWeightedCachePutForTest(key, value);
+ data.put(key, value);
+ boolean retained = reservations.get(key) == newRecord &&
data.asMap().get(key) == value;
+ if (retained) {
+ newRecord.published = true;
+ }
+ if (retained && reservedWeight != newWeight &&
!newRecord.reservation.tryResize(newWeight)) {
+ throw new IllegalStateException("failed to release cache
replacement reservation delta");
+ }
+ if (retained) {
+ notifyReplacement(key, oldValue, value);
+ }
+ return AdmissionResult.ADMITTED;
+ } catch (RuntimeException | Error e) {
+ if (reservations.replace(key, newRecord, previousRecord)) {
+ if (data.asMap().get(key) == null) {
+ reservations.remove(key, previousRecord);
+ previousRecord.reservation.release();
+ } else if
(!previousRecord.reservation.tryResize(previousRecord.weight)) {
+ throw new IllegalStateException("failed to roll back
cache replacement reservation", e);
+ }
+ }
+ throw e;
+ }
+ }
+ }
+
+ private Optional<AdmissionReservation> reserveWithLocalEviction(K
incomingKey, long bytes) {
+ if (bytes > entryBudget.getEffectiveMaxWeight()) {
+ return Optional.empty();
+ }
+ Optional<AdmissionReservation> reservation =
entryBudget.tryReserve(bytes);
+ while (!reservation.isPresent()) {
+ int evicted = evictLocalColdest(incomingKey,
LOCAL_EVICTION_BATCH_SIZE);
+ if (evicted == 0) {
+ entryBudget.requestPeerReclaim(bytes);
+ break;
+ }
+ reservation = entryBudget.tryReserve(bytes);
+ }
+ return reservation;
+ }
+
+ private boolean resizeWithLocalEviction(K incomingKey,
AdmissionReservation reservation, long newBytes) {
+ if (newBytes > entryBudget.getEffectiveMaxWeight()) {
+ return false;
+ }
+ if (reservation.tryResize(newBytes)) {
+ return true;
+ }
+ while (true) {
+ int evicted = evictLocalColdest(incomingKey,
LOCAL_EVICTION_BATCH_SIZE);
+ if (evicted == 0) {
+ entryBudget.requestPeerReclaim(Math.max(0L, newBytes -
reservation.getBytes()));
+ return false;
+ }
+ if (reservation.tryResize(newBytes)) {
+ return true;
+ }
+ }
+ }
+
+ private int evictLocalColdest(K incomingKey, int limit) {
+ if (!data.policy().eviction().isPresent()) {
+ return 0;
+ }
+ Map<K, V> coldest = data.policy().eviction().get().coldest(limit);
+ int evicted = 0;
+ for (Map.Entry<K, V> candidate : coldest.entrySet()) {
+ if (Objects.equals(candidate.getKey(), incomingKey)) {
+ continue;
+ }
+ V current = data.asMap().get(candidate.getKey());
+ ReservationRecord record = reservations.get(candidate.getKey());
+ long evictedWeight = record != null && record.published && current
!= null ? record.weight : 0L;
+ if (current == candidate.getValue() &&
data.asMap().remove(candidate.getKey(), current)) {
+ if (record != null) {
+ releaseReservation(candidate.getKey(), record.generation);
+ }
+ localEvictionCount.incrementAndGet();
+ localEvictionWeight.accumulateAndGet(evictedWeight,
MetaCacheWeightUtils::saturatedAdd);
+ evicted++;
+ }
+ }
+ return evicted;
+ }
+
+ private long reclaimForPeer(long targetBytes) {
+ if (targetBytes <= 0L || closed.get()) {
+ return 0L;
+ }
+ synchronized (admissionLock) {
+ long before = entryBudget.getUsedWeight();
+ long reclaimed = 0L;
+ while (reclaimed < targetBytes
+ && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE) > 0)
{
+ reclaimed = Math.max(0L, before - entryBudget.getUsedWeight());
+ }
+ return reclaimed;
+ }
+ }
+
+ private int weigh(K key, V value) {
+ ReservationRecord record = reservations.get(key);
+ // Every supported write path installs the reservation record before
calling data.put.
+ // Missing ownership is an invariant violation, so fail closed without
invoking an O(n)
+ // estimator from Caffeine's hot weigher callback.
+ long weight = record == null ? Integer.MAX_VALUE : record.weight;
+ return weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight;
Review Comment:
[P2] Preserve exact weights in eviction telemetry
This clamps any admitted entry above 2 GB to Caffeine's int weigher limit,
but stats() later exposes cacheStats.evictionWeight() alongside the exact long
byte limits. TTL, size-policy, and soft-value evictions therefore report at
most Integer.MAX_VALUE for a larger retained graph; only the explicit
local-reclamation path records the reservation's exact weight. Please retain
enough generation-to-weight history to associate each automatic eviction with
its exact reservation even if a replacement races the asynchronous callback,
and add a forced-eviction test with a synthetic estimate above 2 GB.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
