github-actions[bot] commented on code in PR #66717:
URL: https://github.com/apache/doris/pull/66717#discussion_r3796838713
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java:
##########
@@ -208,8 +401,9 @@ public TableMetadata refresh() {
@Override
public void commit(TableMetadata base, TableMetadata newMetadata) {
- delegate.commit(base, newMetadata);
- currentMetadata = newMetadata;
+ TableMetadata delegateBase = prepareDelegateCommit(delegate, base,
currentMetadata);
Review Comment:
[P1] Keep retry rebinding on the retained table UUID
After a commit conflict, `refresh()` can set `currentMetadata` to a
same-contract recreated table because `isWriterCompatible()` never compares
`TableMetadata.uuid()`. This new delegate preparation then compares that
replacement generation with itself, so the retry can attach files produced for
the dropped table to its replacement. Please require every refreshed UUID to
equal the originally retained UUID before allowing retry, and cover
same-location drop/recreate with restarted schema/spec IDs.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java:
##########
@@ -0,0 +1,1192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.datasource.NameMapping;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
+import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
+import org.apache.doris.datasource.metacache.MetaCacheWeightUtils;
+
+import org.apache.iceberg.BlobMetadata;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionStatisticsFile;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StatisticsFile;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.UnknownTransform;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Publication-time retained-weight formulas for Iceberg cache entries. */
+final class IcebergCacheSizeEstimator {
+ // Calibrated against JOL retained-graph deltas in
IcebergExternalMetaCacheTest.
+ // Every metadata element visited (field, type, snapshot, summary entry,
...) costs a few
+ // reads; the bound only guards against pathological metadata and is far
above real tables
+ // (a 10,000-snapshot history with 15 summary keys each is 160,000
elements). Exceeding it
+ // rejects weighted admission, so it must not be reachable by ordinary
long-lived tables.
+ private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L;
+ // Total name characters the estimator may lower-case while reserving
case-insensitive indexes.
+ private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L;
+ private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128;
+ private static final long KEY_BASE_BYTES = objectBytes(128L);
+ private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L);
+ // TableMetadata-side share of one schema version: schemas list slot and
schemasById entry,
+ // including the growth of both from their singleton to their regular
immutable shapes.
+ private static final long SCHEMA_VERSION_BYTES = objectBytes(128L);
+ private static final long PARTITION_SPEC_BYTES = objectBytes(256L);
+ // Exact active-layout sizes of the Iceberg/Guava objects that lazy
partition, sort and
+ // schema state allocates. Iceberg 1.10.1 field layouts are pinned by
ICEBERG_LAZY_LAYOUT_SUPPORTED.
+ private static final long PARTITION_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L);
+ private static final long SORT_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ // Identity/Bucket/Truncate transforms are allocated per parsed field;
time transforms are enums.
+ private static final long TRANSFORM_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long NESTED_FIELD_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L);
+ private static final long STRUCT_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L);
+ private static final long SCHEMA_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L);
+ private static final long IMMUTABLE_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long IMMUTABLE_MAP_KEY_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SINGLETON_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long REGULAR_IMMUTABLE_SET_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L);
+ private static final long ARRAY_LIST_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L);
+ private static final long HASH_MAP_NODE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L);
+ private static final long HASH_MAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L);
+ private static final long INTEGER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long LONG_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate[";
+ // Truncate on a decimal source retains a BigInteger width (object plus
one-int magnitude).
+ private static final long TRUNCATE_WIDTH_BYTES =
MetaCacheWeightUtils.saturatedAdd(
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L),
+ MetaCacheWeightUtils.estimatedIntArrayBytes(1L));
+ private static final long LIST_MULTIMAP_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L);
+ private static final long CAPTURING_SUPPLIER_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long POSITION_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L);
+ // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor.
Required ancestors
+ // collapse into a single Position2/3Accessor that replaces the inner
accessor, which retains
+ // less than this per-level reservation.
+ private static final long WRAPPED_ACCESSOR_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L);
+ private static final long LIST_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long MAP_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L);
+ private static final long DECIMAL_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L);
+ private static final long FIXED_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L);
+ private static final long GEOMETRY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L);
+ private static final long GEOGRAPHY_TYPE_BYTES =
+ MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L);
+ private static final long SORT_ORDER_BYTES = objectBytes(256L);
+ private static final long TABLE_PROPERTY_BYTES = objectBytes(40L);
+ private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L);
+ private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L);
+ private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L);
+ private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L);
+ private static final long SNAPSHOT_REF_BYTES = objectBytes(128L);
+ private static final long STATISTICS_FILE_BYTES = objectBytes(512L);
+ private static final long BLOB_METADATA_BYTES = objectBytes(128L);
+ private static final long BLOB_FIELD_BYTES = objectBytes(32L);
+ private static final long PARTITION_STATISTICS_FILE_BYTES =
objectBytes(256L);
+ private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L);
+ private static final long PARTITION_BYTES = objectBytes(640L);
+ private static final long PARTITION_ALIAS_BYTES = objectBytes(256L);
+ private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L);
+ private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L);
+ private static final long DATA_FILE_BYTES = objectBytes(896L);
+ private static final long DELETE_FILE_BYTES = objectBytes(1024L);
+ private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L);
+ private static final String BASE_SNAPSHOT_CLASS_NAME =
"org.apache.iceberg.BaseSnapshot";
+ private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS =
+ loadBaseSnapshotRetainedCacheFields();
+ // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog
supplier
+ // (REST snapshot-loading-mode=refs). Publication must not perform that IO.
+ private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD =
+ loadTableMetadataField("snapshotsLoaded", boolean.class);
+ private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD =
+ loadTableMetadataField("snapshotsSupplier", null);
+ // The formulas above are built on the Iceberg 1.10.1 instance-field
layouts of the classes a
+ // cached table retains. Every non-static field is pinned, not only the
transient lazy ones: a
+ // library upgrade that adds a retained reference makes weighted admission
fail closed.
+ private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED =
checkIcebergLayout();
+
+ private IcebergCacheSizeEstimator() {
+ }
+
+ private static long objectBytes(long bytes) {
+ return MetaCacheWeightUtils.estimatedObjectBytes(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key,
IcebergTableCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ Table table = value.getRetainedIcebergTable();
+ MetaCacheSizeEstimate support = checkSupportedTable(table);
+ if (!support.isComplete()) {
+ return support;
+ }
+ long bytes = MetaCacheWeightUtils.saturatedAdd(
+ KEY_BASE_BYTES,
MetaCacheWeightUtils.estimatedNameMappingBytes(key));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
value.getRetainedTablePayloadBytes());
+ bytes = MetaCacheWeightUtils.saturatedAdd(
+ bytes, value.getRetainedCurrentSnapshotPayloadBytes());
+ return MetaCacheSizeEstimate.complete(bytes);
+ }
+
+ static MetaCacheSizeEstimate estimateSnapshotEntry(
+ IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
+ MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout();
+ if (!layoutSupport.isComplete()) {
+ return layoutSupport;
+ }
+ long bytes = KEY_BASE_BYTES;
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+ MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid()));
+ bytes = MetaCacheWeightUtils.saturatedAdd(bytes,
+
MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation()));
+
+ IcebergPartitionInfo partitionInfo = value.getPartitionInfo();
+ bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(),
PARTITION_BYTES);
Review Comment:
[P2] Account for partition width after table refresh
`isValidRelatedTable()` normally limits this projection to one partition
field, but that decision is cached on `IcebergExternalTable` while `tableEntry`
auto-refreshes its physical generation independently. After a table first
validates as single-field and then refreshes to a wider spec,
`loadPartitionInfo()` still retains another slot in each `IcebergPartition`
list plus another literal/type slot in both range endpoints for every added
field. These charges scale only with map sizes and string payload, so the
refreshed graph can be admitted below its real retained weight. Please either
revalidate related-table eligibility per retained generation or carry
partition-field cardinality into the estimate, and cover a one-to-many field
refresh with a JOL delta.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
