924060929 commented on code in PR #67996: URL: https://github.com/apache/doris/pull/67996#discussion_r4059021939
########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCacheSizeEstimator.java: ########## @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.iceberg.IcebergTable; +import org.apache.paimon.table.lance.LanceTable; +import org.apache.paimon.table.object.ObjectTable; + +import java.net.URI; + +/** + * Retained-size formulas for Paimon table-cache entries. + * + * <p>The table's shallow size includes references to FileIO, catalog loaders, and lock factories, + * but their graphs are catalog-scoped executable services rather than entry-owned metadata. Walking + * those graphs both double-counts shared state and reaches strongly encapsulated JDK objects. The + * estimator therefore expands only immutable metadata owned by the entry. + */ +final class PaimonCacheSizeEstimator { + private PaimonCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTable(Identifier key, Table table, long entryOverheadBytes) { + long bytes = add(entryOverheadBytes, ReflectiveObjectSizeEstimator.estimateComplete(key)); + bytes = add(bytes, JvmSizeUtils.instanceSize(table.getClass())); + if (table instanceof FileStoreTable) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(fileStoreTable.schema())); Review Comment: Fixed in 7680bf3ff51. The estimator now walks DelegatedFileStoreTable wrappers, counts both FallbackReadFileStoreTable branches, and deduplicates by object identity. PrivilegedFileStoreTable is rejected from the raw metadata cache because authorization decorators now live outside it. Added weight-governed tests for distinct/shared fallback branches and decorated-value rejection; the full Paimon suite passes (636 tests, 0 failures/errors). ########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java: ########## @@ -0,0 +1,342 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.utils.SegmentsCache; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +/** + * Doris-owned replacement for Paimon's {@code CachingCatalog}. Table and database entries live in + * {@link CatalogMetaCache}, so a Doris catalog/database/table invalidation fences every matching + * in-flight load and cached value. + * + * <p>The user's {@code paimon.cache-enabled} and access/write expiry settings remain authoritative. + * The Paimon SDK wrapper itself is disabled because a second hidden table cache cannot participate + * in Doris invalidation. Under a Doris weight budget, mutable SDK snapshot/stats/manifest caches are + * not attached: their post-publication growth cannot be reweighed by the enclosing budget. + */ +final class PaimonMetaCacheCatalog extends DelegateCatalog { + + private static final int DATABASE_CACHE_CAPACITY = 100; + private static final long TABLE_ENTRY_OVERHEAD_BYTES = JvmSizeUtils.saturatedAdd( + JvmSizeUtils.instanceSize(ExpiringValue.class), JvmSizeUtils.instanceSize(AtomicLong.class)); + + private final CatalogMetaCache metaCache; + private final MetaCache<Identifier, ExpiringValue<Table>> tableCache; + private final MetaCache<String, ExpiringValue<Database>> databaseCache; + private final SegmentsCache<Path> manifestCache; + private final long tableExpireAfterAccessNanos; + private final long databaseExpireAfterAccessNanos; + private final long expireAfterWriteNanos; + private final int snapshotMaxNumPerTable; + private final boolean attachSdkCaches; + private final LongSupplier nanoTime; + + static Catalog tryToCreate(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit) { + return new PaimonMetaCacheCatalog(wrapped, metaCache, tableCacheMaxSize, + tableCacheTtlSecond, catalogOptions, cacheEnabled, hasEnclosingWeightLimit, System::nanoTime); + } + + PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean hasEnclosingWeightLimit, + LongSupplier nanoTime) { + this(wrapped, metaCache, tableCacheMaxSize, tableCacheTtlSecond, catalogOptions, + true, hasEnclosingWeightLimit, nanoTime); + } + + private PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit, LongSupplier nanoTime) { + super(wrapped); + this.metaCache = metaCache; + this.nanoTime = nanoTime; + + Duration expireAfterAccess = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS); + Duration expireAfterWrite = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_WRITE); + if (cacheEnabled) { + requirePositive(expireAfterAccess, CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS.key()); + requirePositive(expireAfterWrite, CatalogOptions.CACHE_EXPIRE_AFTER_WRITE.key()); + } + long paimonAccessNanos = cacheEnabled ? expireAfterAccess.toNanos() : Long.MAX_VALUE; + this.tableExpireAfterAccessNanos = cacheEnabled && tableCacheTtlSecond > 0 + ? Math.min(paimonAccessNanos, Duration.ofSeconds(tableCacheTtlSecond).toNanos()) Review Comment: Fixed in 7680bf3ff51. All accepted access/write/table TTL durations now use a saturated nanosecond conversion, mapping only Duration.toNanos overflow to Long.MAX_VALUE. Added coverage for 9223372037s and a Long.MAX_VALUE connector TTL; catalog creation and cache reuse remain valid. ########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java: ########## @@ -0,0 +1,342 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.utils.SegmentsCache; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +/** + * Doris-owned replacement for Paimon's {@code CachingCatalog}. Table and database entries live in + * {@link CatalogMetaCache}, so a Doris catalog/database/table invalidation fences every matching + * in-flight load and cached value. + * + * <p>The user's {@code paimon.cache-enabled} and access/write expiry settings remain authoritative. + * The Paimon SDK wrapper itself is disabled because a second hidden table cache cannot participate + * in Doris invalidation. Under a Doris weight budget, mutable SDK snapshot/stats/manifest caches are + * not attached: their post-publication growth cannot be reweighed by the enclosing budget. + */ +final class PaimonMetaCacheCatalog extends DelegateCatalog { + + private static final int DATABASE_CACHE_CAPACITY = 100; + private static final long TABLE_ENTRY_OVERHEAD_BYTES = JvmSizeUtils.saturatedAdd( + JvmSizeUtils.instanceSize(ExpiringValue.class), JvmSizeUtils.instanceSize(AtomicLong.class)); + + private final CatalogMetaCache metaCache; + private final MetaCache<Identifier, ExpiringValue<Table>> tableCache; + private final MetaCache<String, ExpiringValue<Database>> databaseCache; + private final SegmentsCache<Path> manifestCache; + private final long tableExpireAfterAccessNanos; + private final long databaseExpireAfterAccessNanos; + private final long expireAfterWriteNanos; + private final int snapshotMaxNumPerTable; + private final boolean attachSdkCaches; + private final LongSupplier nanoTime; + + static Catalog tryToCreate(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit) { + return new PaimonMetaCacheCatalog(wrapped, metaCache, tableCacheMaxSize, + tableCacheTtlSecond, catalogOptions, cacheEnabled, hasEnclosingWeightLimit, System::nanoTime); + } + + PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean hasEnclosingWeightLimit, + LongSupplier nanoTime) { + this(wrapped, metaCache, tableCacheMaxSize, tableCacheTtlSecond, catalogOptions, + true, hasEnclosingWeightLimit, nanoTime); + } + + private PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit, LongSupplier nanoTime) { + super(wrapped); + this.metaCache = metaCache; + this.nanoTime = nanoTime; + + Duration expireAfterAccess = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS); + Duration expireAfterWrite = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_WRITE); + if (cacheEnabled) { + requirePositive(expireAfterAccess, CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS.key()); + requirePositive(expireAfterWrite, CatalogOptions.CACHE_EXPIRE_AFTER_WRITE.key()); + } + long paimonAccessNanos = cacheEnabled ? expireAfterAccess.toNanos() : Long.MAX_VALUE; + this.tableExpireAfterAccessNanos = cacheEnabled && tableCacheTtlSecond > 0 + ? Math.min(paimonAccessNanos, Duration.ofSeconds(tableCacheTtlSecond).toNanos()) + : paimonAccessNanos; + this.databaseExpireAfterAccessNanos = paimonAccessNanos; + this.expireAfterWriteNanos = cacheEnabled ? expireAfterWrite.toNanos() : Long.MAX_VALUE; + + CacheSpec tableSpec = CacheSpec.of(cacheEnabled, + cacheEnabled && tableCacheTtlSecond > 0 + ? CacheSpec.CACHE_NO_TTL : CacheSpec.CACHE_TTL_DISABLE_CACHE, + tableCacheMaxSize); + this.tableCache = metaCache.create(MetaCacheDefinition + .<Identifier, ExpiringValue<Table>>builder("paimon-table", tableSpec, + id -> ScopePath.table(id.getDatabaseName(), id.getTableName())) + .sizeEstimator((id, value) -> PaimonCacheSizeEstimator.estimateTable( + id, value.value, TABLE_ENTRY_OVERHEAD_BYTES)) + .build()); + CacheSpec dbSpec = CacheSpec.of(cacheEnabled, cacheEnabled + ? CacheSpec.CACHE_NO_TTL : CacheSpec.CACHE_TTL_DISABLE_CACHE, DATABASE_CACHE_CAPACITY); + this.databaseCache = metaCache.create(MetaCacheDefinition + .<String, ExpiringValue<Database>>builder("paimon-database", dbSpec, ScopePath::database) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .build()); + + this.attachSdkCaches = cacheEnabled && !hasEnclosingWeightLimit; + this.manifestCache = attachSdkCaches ? buildManifestCache(catalogOptions) : null; + this.snapshotMaxNumPerTable = catalogOptions.get( + CatalogOptions.CACHE_SNAPSHOT_MAX_NUM_PER_TABLE); + } + + @Override + public Table getTable(Identifier identifier) throws TableNotExistException { + if (identifier.isSystemTable()) { + Identifier origin = new Identifier(identifier.getDatabaseName(), identifier.getTableName(), + identifier.getBranchName(), null); + Table originTable = getTable(origin); + if (!(originTable instanceof FileStoreTable)) { + return super.getTable(identifier); + } + Table systemTable = SystemTableLoader.load(identifier.getSystemTableName(), + (FileStoreTable) originTable); + if (systemTable == null) { + throw new TableNotExistException(identifier); + } + return systemTable; + } + + while (true) { + long now = nanoTime.getAsLong(); + ExpiringValue<Table> cached = tableCache.getIfPresent(identifier); + if (cached != null) { + if (cached.tryAccess(now, tableExpireAfterAccessNanos, expireAfterWriteNanos)) { + return cached.value; + } + tableCache.compareAndSet(identifier, cached, null); + continue; + } + try { + return tableCache.get(identifier, ignored -> { + try { + Table loaded = attachPerTableCaches(super.getTable(identifier)); Review Comment: Fixed in 7680bf3ff51. The cache now retains raw Paimon tables and PrivilegedCatalog wraps PaimonMetaCacheCatalog outside the cache, restoring a fresh privilege checker on every lookup. The regression warms one raw table, revokes SELECT, verifies the next scan is denied, and confirms the raw metadata was still loaded only once. ########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java: ########## @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.SegmentsCache; + +import java.time.Duration; +import java.util.Optional; + +/** + * Doris-owned replacement for the Paimon SDK {@code CachingCatalog}. Every catalog-level cache + * ({@code tableCache}, {@code databaseCache}) lives in Doris's {@link CatalogMetaCache} framework + * with a per-catalog scope, so {@code REFRESH TABLE}/{@code REFRESH DATABASE}/ + * {@code REFRESH CATALOG} invalidates them through the same registry path as every other + * connector-owned cache. The per-{@link FileStoreTable} caches ({@code snapshotCache}, + * {@code statsCache}, {@code manifestCache}) are built from the same {@link CatalogOptions} that + * {@code CachingCatalog} reads and attached on {@link #getTable}, preserving scan-time performance. + * + * <p><b>Why not the SDK's CachingCatalog?</b> Its {@code tableCache} freezes schema/snapshot + * state at load time and exposes only per-table {@code invalidateTable(Identifier)} — no + * db/catalog-level eviction. After an external same-name drop/recreate the stale frozen + * {@link Table} survives every Doris-side {@code REFRESH}. + */ +final class PaimonMetaCacheCatalog extends DelegateCatalog { + + private final MetaCache<Identifier, Table> tableCache; + private final MetaCache<String, Database> databaseCache; + private final SegmentsCache<Path> manifestCache; + private final Duration expireAfterAccess; + private final Duration expireAfterWrite; + private final int snapshotMaxNumPerTable; + + PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions) { + super(wrapped); + CacheSpec tableSpec = CacheSpec.ofConnectorTtl(tableCacheTtlSecond, tableCacheMaxSize); + this.tableCache = metaCache.create(MetaCacheDefinition + .<Identifier, Table>builder("paimon-table", tableSpec, + id -> ScopePath.table(id.getDatabaseName(), id.getObjectName())) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .build()); + CacheSpec dbSpec = CacheSpec.ofConnectorTtl(86400L, 100); + this.databaseCache = metaCache.create(MetaCacheDefinition + .<String, Database>builder("paimon-database", dbSpec, + ScopePath::database) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .build()); + + this.manifestCache = buildManifestCache(catalogOptions); + this.expireAfterAccess = catalogOptions.get( + CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS); + this.expireAfterWrite = catalogOptions.get( + CatalogOptions.CACHE_EXPIRE_AFTER_WRITE); + this.snapshotMaxNumPerTable = catalogOptions.get( + CatalogOptions.CACHE_SNAPSHOT_MAX_NUM_PER_TABLE); + } + + @Override + public Table getTable(Identifier identifier) throws TableNotExistException { + try { + return tableCache.get(identifier, ignored -> { + try { + return attachPerTableCaches(super.getTable(identifier)); + } catch (TableNotExistException e) { + throw new RuntimeException(e); + } + }); + } catch (RuntimeException e) { + if (e.getCause() instanceof TableNotExistException) { + throw (TableNotExistException) e.getCause(); + } + throw e; + } + } + + @Override + public Database getDatabase(String name) throws DatabaseNotExistException { + try { + return databaseCache.get(name, ignored -> { + try { + return super.getDatabase(name); + } catch (DatabaseNotExistException e) { + throw new RuntimeException(e); + } + }); + } catch (RuntimeException e) { + if (e.getCause() instanceof DatabaseNotExistException) { + throw (DatabaseNotExistException) e.getCause(); + } + throw e; + } + } + + @Override Review Comment: Fixed in the current head 7680bf3ff51. Successful table mutations invalidate the affected table scope locally even when a later DROP DATABASE FORCE step fails, so already-committed remote drops cannot remain hot until TTL expiry. The partial-drop regression covers failure on a later table. ########## fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java: ########## @@ -0,0 +1,335 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogLoader; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.utils.SegmentsCache; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +/** + * Doris-owned replacement for Paimon's {@code CachingCatalog}. Table and database entries live in + * {@link CatalogMetaCache}, so a Doris catalog/database/table invalidation fences every matching + * in-flight load and cached value. + * + * <p>The user's {@code paimon.cache-enabled} and access/write expiry settings remain authoritative. + * The Paimon SDK wrapper itself is disabled because a second hidden table cache cannot participate + * in Doris invalidation. Under a Doris weight budget, mutable SDK snapshot/stats/manifest caches are + * not attached: their post-publication growth cannot be reweighed by the enclosing budget. + */ +final class PaimonMetaCacheCatalog extends DelegateCatalog { + + private static final int DATABASE_CACHE_CAPACITY = 100; + + private final CatalogMetaCache metaCache; + private final MetaCache<Identifier, ExpiringValue<Table>> tableCache; + private final MetaCache<String, ExpiringValue<Database>> databaseCache; + private final SegmentsCache<Path> manifestCache; + private final long tableExpireAfterAccessNanos; + private final long databaseExpireAfterAccessNanos; + private final long expireAfterWriteNanos; + private final int snapshotMaxNumPerTable; + private final boolean attachSdkCaches; + private final LongSupplier nanoTime; + + static Catalog tryToCreate(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit) { + return new PaimonMetaCacheCatalog(wrapped, metaCache, tableCacheMaxSize, + tableCacheTtlSecond, catalogOptions, cacheEnabled, hasEnclosingWeightLimit, System::nanoTime); + } + + PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean hasEnclosingWeightLimit, + LongSupplier nanoTime) { + this(wrapped, metaCache, tableCacheMaxSize, tableCacheTtlSecond, catalogOptions, + true, hasEnclosingWeightLimit, nanoTime); + } + + private PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int tableCacheMaxSize, + long tableCacheTtlSecond, Options catalogOptions, boolean cacheEnabled, + boolean hasEnclosingWeightLimit, LongSupplier nanoTime) { + super(wrapped); + this.metaCache = metaCache; + this.nanoTime = nanoTime; + + Duration expireAfterAccess = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS); + Duration expireAfterWrite = catalogOptions.get(CatalogOptions.CACHE_EXPIRE_AFTER_WRITE); + if (cacheEnabled) { + requirePositive(expireAfterAccess, CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS.key()); + requirePositive(expireAfterWrite, CatalogOptions.CACHE_EXPIRE_AFTER_WRITE.key()); + } + long paimonAccessNanos = cacheEnabled ? expireAfterAccess.toNanos() : Long.MAX_VALUE; + this.tableExpireAfterAccessNanos = cacheEnabled && tableCacheTtlSecond > 0 + ? Math.min(paimonAccessNanos, Duration.ofSeconds(tableCacheTtlSecond).toNanos()) + : paimonAccessNanos; + this.databaseExpireAfterAccessNanos = paimonAccessNanos; + this.expireAfterWriteNanos = cacheEnabled ? expireAfterWrite.toNanos() : Long.MAX_VALUE; + + CacheSpec tableSpec = CacheSpec.of(cacheEnabled, + cacheEnabled && tableCacheTtlSecond > 0 + ? CacheSpec.CACHE_NO_TTL : CacheSpec.CACHE_TTL_DISABLE_CACHE, + tableCacheMaxSize); + this.tableCache = metaCache.create(MetaCacheDefinition + .<Identifier, ExpiringValue<Table>>builder("paimon-table", tableSpec, + id -> ScopePath.table(id.getDatabaseName(), id.getTableName())) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .build()); + CacheSpec dbSpec = CacheSpec.of(cacheEnabled, cacheEnabled + ? CacheSpec.CACHE_NO_TTL : CacheSpec.CACHE_TTL_DISABLE_CACHE, DATABASE_CACHE_CAPACITY); + this.databaseCache = metaCache.create(MetaCacheDefinition + .<String, ExpiringValue<Database>>builder("paimon-database", dbSpec, ScopePath::database) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .build()); + + this.attachSdkCaches = cacheEnabled && !hasEnclosingWeightLimit; + this.manifestCache = attachSdkCaches ? buildManifestCache(catalogOptions) : null; + this.snapshotMaxNumPerTable = catalogOptions.get( + CatalogOptions.CACHE_SNAPSHOT_MAX_NUM_PER_TABLE); + } + + @Override + public Table getTable(Identifier identifier) throws TableNotExistException { + if (identifier.isSystemTable()) { + Identifier origin = new Identifier(identifier.getDatabaseName(), identifier.getTableName(), + identifier.getBranchName(), null); + Table originTable = getTable(origin); + Table systemTable = SystemTableLoader.load(identifier.getSystemTableName(), + (FileStoreTable) originTable); Review Comment: Fixed in the current head 7680bf3ff51. System-table reconstruction is used only when the cached origin is a FileStoreTable; non-FileStore origins delegate to the wrapped catalog four-argument system identifier path. A regression covers the supported fallback and avoids the ClassCastException. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
