924060929 commented on code in PR #67996:
URL: https://github.com/apache/doris/pull/67996#discussion_r4059025197
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java:
##########
@@ -547,7 +546,9 @@ private Catalog createCatalogFromContext(CatalogContext
catalogContext, String f
? createHmsCatalog(catalogContext, hmsAuth,
catalogProps.getRaw(),
storageHadoopConfig)
: CatalogFactory.createCatalog(catalogContext);
- return catalog;
+ return new PaimonMetaCacheCatalog(catalog, metaCache,
Review Comment:
Fixed in the current head 7680bf3ff51. CatalogBackedPaimonCatalogOps now
resolves REST dispatch through DelegateCatalog.rootCatalog, preserving
authoritative REST partition listing through the metadata and privilege
wrappers. Coverage includes wrapped REST detection plus endpoint success,
forbidden-without-filesystem-fallback, and unsupported-endpoint fallback.
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java:
##########
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.paimon;
+
+import org.apache.doris.connector.cache.CacheSpec;
+import org.apache.doris.connector.cache.CatalogMetaCache;
+import org.apache.doris.connector.cache.MetaCache;
+import org.apache.doris.connector.cache.MetaCacheDefinition;
+import org.apache.doris.connector.cache.MetaCacheSizeEstimators;
+import org.apache.doris.connector.cache.ScopePath;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Database;
+import org.apache.paimon.catalog.DelegateCatalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.MemorySize;
+import org.apache.paimon.options.Options;
+import
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.utils.SegmentsCache;
+
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * Doris-owned replacement for the Paimon SDK {@code CachingCatalog}. Every
catalog-level cache
+ * ({@code tableCache}, {@code databaseCache}) lives in Doris's {@link
CatalogMetaCache} framework
+ * with a per-catalog scope, so {@code REFRESH TABLE}/{@code REFRESH DATABASE}/
+ * {@code REFRESH CATALOG} invalidates them through the same registry path as
every other
+ * connector-owned cache. The per-{@link FileStoreTable} caches ({@code
snapshotCache},
+ * {@code statsCache}, {@code manifestCache}) are built from the same {@link
CatalogOptions} that
+ * {@code CachingCatalog} reads and attached on {@link #getTable}, preserving
scan-time performance.
+ *
+ * <p><b>Why not the SDK's CachingCatalog?</b> Its {@code tableCache} freezes
schema/snapshot
+ * state at load time and exposes only per-table {@code
invalidateTable(Identifier)} — no
+ * db/catalog-level eviction. After an external same-name drop/recreate the
stale frozen
+ * {@link Table} survives every Doris-side {@code REFRESH}.
+ */
+final class PaimonMetaCacheCatalog extends DelegateCatalog {
+
+ private final MetaCache<Identifier, Table> tableCache;
+ private final MetaCache<String, Database> databaseCache;
+ private final SegmentsCache<Path> manifestCache;
+ private final Duration expireAfterAccess;
+ private final Duration expireAfterWrite;
+ private final int snapshotMaxNumPerTable;
+
+ PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int
tableCacheMaxSize,
+ long tableCacheTtlSecond, Options catalogOptions) {
+ super(wrapped);
+ CacheSpec tableSpec = CacheSpec.ofConnectorTtl(tableCacheTtlSecond,
tableCacheMaxSize);
+ this.tableCache = metaCache.create(MetaCacheDefinition
+ .<Identifier, Table>builder("paimon-table", tableSpec,
+ id -> ScopePath.table(id.getDatabaseName(),
id.getObjectName()))
Review Comment:
Fixed in the current head 7680bf3ff51. Cache scope uses
Identifier.getTableName(), so base, branch, and system variants share the
base-table invalidation scope; system tables are rebuilt from the cached
origin. Tests verify base invalidation evicts branch variants and system-table
resolution reloads through the origin.
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java:
##########
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.paimon;
+
+import org.apache.doris.connector.cache.CacheSpec;
+import org.apache.doris.connector.cache.CatalogMetaCache;
+import org.apache.doris.connector.cache.MetaCache;
+import org.apache.doris.connector.cache.MetaCacheDefinition;
+import org.apache.doris.connector.cache.MetaCacheSizeEstimators;
+import org.apache.doris.connector.cache.ScopePath;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Database;
+import org.apache.paimon.catalog.DelegateCatalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.MemorySize;
+import org.apache.paimon.options.Options;
+import
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.utils.SegmentsCache;
+
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * Doris-owned replacement for the Paimon SDK {@code CachingCatalog}. Every
catalog-level cache
+ * ({@code tableCache}, {@code databaseCache}) lives in Doris's {@link
CatalogMetaCache} framework
+ * with a per-catalog scope, so {@code REFRESH TABLE}/{@code REFRESH DATABASE}/
+ * {@code REFRESH CATALOG} invalidates them through the same registry path as
every other
+ * connector-owned cache. The per-{@link FileStoreTable} caches ({@code
snapshotCache},
+ * {@code statsCache}, {@code manifestCache}) are built from the same {@link
CatalogOptions} that
+ * {@code CachingCatalog} reads and attached on {@link #getTable}, preserving
scan-time performance.
+ *
+ * <p><b>Why not the SDK's CachingCatalog?</b> Its {@code tableCache} freezes
schema/snapshot
+ * state at load time and exposes only per-table {@code
invalidateTable(Identifier)} — no
+ * db/catalog-level eviction. After an external same-name drop/recreate the
stale frozen
+ * {@link Table} survives every Doris-side {@code REFRESH}.
+ */
+final class PaimonMetaCacheCatalog extends DelegateCatalog {
+
+ private final MetaCache<Identifier, Table> tableCache;
+ private final MetaCache<String, Database> databaseCache;
+ private final SegmentsCache<Path> manifestCache;
+ private final Duration expireAfterAccess;
+ private final Duration expireAfterWrite;
+ private final int snapshotMaxNumPerTable;
+
+ PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int
tableCacheMaxSize,
+ long tableCacheTtlSecond, Options catalogOptions) {
+ super(wrapped);
+ CacheSpec tableSpec = CacheSpec.ofConnectorTtl(tableCacheTtlSecond,
tableCacheMaxSize);
+ this.tableCache = metaCache.create(MetaCacheDefinition
+ .<Identifier, Table>builder("paimon-table", tableSpec,
+ id -> ScopePath.table(id.getDatabaseName(),
id.getObjectName()))
+ .sizeEstimator(MetaCacheSizeEstimators.reflective())
+ .build());
+ CacheSpec dbSpec = CacheSpec.ofConnectorTtl(86400L, 100);
+ this.databaseCache = metaCache.create(MetaCacheDefinition
+ .<String, Database>builder("paimon-database", dbSpec,
+ ScopePath::database)
+ .sizeEstimator(MetaCacheSizeEstimators.reflective())
+ .build());
+
+ this.manifestCache = buildManifestCache(catalogOptions);
Review Comment:
Fixed in the current head 7680bf3ff51. When CatalogMetaCache has an
enclosing weight limit, the wrapper does not attach Paimon snapshot, stats, or
manifest caches, preventing mutable post-admission growth outside the Doris
budget. The governed/ungoverned attachment test covers both paths.
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonMetaCacheCatalog.java:
##########
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.paimon;
+
+import org.apache.doris.connector.cache.CacheSpec;
+import org.apache.doris.connector.cache.CatalogMetaCache;
+import org.apache.doris.connector.cache.MetaCache;
+import org.apache.doris.connector.cache.MetaCacheDefinition;
+import org.apache.doris.connector.cache.MetaCacheSizeEstimators;
+import org.apache.doris.connector.cache.ScopePath;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Database;
+import org.apache.paimon.catalog.DelegateCatalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.MemorySize;
+import org.apache.paimon.options.Options;
+import
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.utils.SegmentsCache;
+
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * Doris-owned replacement for the Paimon SDK {@code CachingCatalog}. Every
catalog-level cache
+ * ({@code tableCache}, {@code databaseCache}) lives in Doris's {@link
CatalogMetaCache} framework
+ * with a per-catalog scope, so {@code REFRESH TABLE}/{@code REFRESH DATABASE}/
+ * {@code REFRESH CATALOG} invalidates them through the same registry path as
every other
+ * connector-owned cache. The per-{@link FileStoreTable} caches ({@code
snapshotCache},
+ * {@code statsCache}, {@code manifestCache}) are built from the same {@link
CatalogOptions} that
+ * {@code CachingCatalog} reads and attached on {@link #getTable}, preserving
scan-time performance.
+ *
+ * <p><b>Why not the SDK's CachingCatalog?</b> Its {@code tableCache} freezes
schema/snapshot
+ * state at load time and exposes only per-table {@code
invalidateTable(Identifier)} — no
+ * db/catalog-level eviction. After an external same-name drop/recreate the
stale frozen
+ * {@link Table} survives every Doris-side {@code REFRESH}.
+ */
+final class PaimonMetaCacheCatalog extends DelegateCatalog {
+
+ private final MetaCache<Identifier, Table> tableCache;
+ private final MetaCache<String, Database> databaseCache;
+ private final SegmentsCache<Path> manifestCache;
+ private final Duration expireAfterAccess;
+ private final Duration expireAfterWrite;
+ private final int snapshotMaxNumPerTable;
+
+ PaimonMetaCacheCatalog(Catalog wrapped, CatalogMetaCache metaCache, int
tableCacheMaxSize,
+ long tableCacheTtlSecond, Options catalogOptions) {
+ super(wrapped);
+ CacheSpec tableSpec = CacheSpec.ofConnectorTtl(tableCacheTtlSecond,
tableCacheMaxSize);
Review Comment:
Fixed in the current head 7680bf3ff51. The replacement carries
paimon.cache-enabled separately from the forced raw SDK wrapper setting and
applies the effective Paimon access/write durations to Doris Table and Database
entries. Tests cover disabled caching, access/write expiry, and non-monotonic
clock behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
