This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 866428eb6e [#8697] improvement(core): Adjust weight value for entities
with type `metalake` and `catalog` in Caffeine cache weighter. (#8698)
866428eb6e is described below
commit 866428eb6eec0ab689d18eca7ea0efa40ca50035
Author: Mini Yu <[email protected]>
AuthorDate: Tue Sep 30 14:16:41 2025 +0800
[#8697] improvement(core): Adjust weight value for entities with type
`metalake` and `catalog` in Caffeine cache weighter. (#8698)
### What changes were proposed in this pull request?
Adjust the weight value for the entity `metalake` and `catalog` to
prevent it from being evicted easily. Entities with a higher weight
value are more likely to be evicted, so we need to make it smaller for
entities with types `metalake` and `catalog`.
### Why are the changes needed?
Metalakes and catalogs are frequently accessed entities that can always
keep them in memory.
Fix: #8697
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
UTs
---
.../apache/gravitino/cache/EntityCacheWeigher.java | 20 ++---
.../apache/gravitino/cache/TestCacheConfig.java | 93 +++++++++++++++++++++-
2 files changed, 103 insertions(+), 10 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/cache/EntityCacheWeigher.java
b/core/src/main/java/org/apache/gravitino/cache/EntityCacheWeigher.java
index bdf32172b3..768f60eb64 100644
--- a/core/src/main/java/org/apache/gravitino/cache/EntityCacheWeigher.java
+++ b/core/src/main/java/org/apache/gravitino/cache/EntityCacheWeigher.java
@@ -36,17 +36,19 @@ import org.slf4j.LoggerFactory;
* weight is calculated as follows:
*
* <ul>
- * <li>Metalake: 100
- * <li>Catalog: 75
- * <li>Schema: 50
- * <li>Other: 15
+ * <li>Metalake: 0, which means that it will never be evicted from the cache
unless timeout occurs
+ * or manually cleared.
+ * <li>Catalog: 0, which means that it will never be evicted from the cache
unless timeout occurs
+ * or manually cleared.
+ * <li>Schema: 10
+ * <li>Other: 100
* </ul>
*/
public class EntityCacheWeigher implements Weigher<EntityCacheKey,
List<Entity>> {
- public static final int METALAKE_WEIGHT = 100;
- public static final int CATALOG_WEIGHT = 75;
- public static final int SCHEMA_WEIGHT = 50;
- public static final int OTHER_WEIGHT = 15;
+ public static final int METALAKE_WEIGHT = 0;
+ public static final int CATALOG_WEIGHT = 0;
+ public static final int SCHEMA_WEIGHT = 10;
+ public static final int OTHER_WEIGHT = 100;
private static final Logger LOG =
LoggerFactory.getLogger(EntityCacheWeigher.class.getName());
private static final EntityCacheWeigher INSTANCE = new EntityCacheWeigher();
private static final Map<Entity.EntityType, Integer> ENTITY_WEIGHTS =
@@ -54,7 +56,7 @@ public class EntityCacheWeigher implements
Weigher<EntityCacheKey, List<Entity>>
Entity.EntityType.METALAKE, METALAKE_WEIGHT,
Entity.EntityType.CATALOG, CATALOG_WEIGHT,
Entity.EntityType.SCHEMA, SCHEMA_WEIGHT);
- private static long MAX_WEIGHT =
+ private static final long MAX_WEIGHT =
2 * (METALAKE_WEIGHT * 10 + CATALOG_WEIGHT * (10 * 200) + SCHEMA_WEIGHT
* (10 * 200 * 1000));
@VisibleForTesting
diff --git a/core/src/test/java/org/apache/gravitino/cache/TestCacheConfig.java
b/core/src/test/java/org/apache/gravitino/cache/TestCacheConfig.java
index 2320fa36ee..55c62a1ae9 100644
--- a/core/src/test/java/org/apache/gravitino/cache/TestCacheConfig.java
+++ b/core/src/test/java/org/apache/gravitino/cache/TestCacheConfig.java
@@ -19,8 +19,22 @@
package org.apache.gravitino.cache;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import java.time.Duration;
+import java.util.List;
+import org.apache.gravitino.Catalog;
import org.apache.gravitino.Config;
import org.apache.gravitino.Configs;
+import org.apache.gravitino.Entity;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.Namespace;
+import org.apache.gravitino.meta.AuditInfo;
+import org.apache.gravitino.meta.BaseMetalake;
+import org.apache.gravitino.meta.CatalogEntity;
+import org.apache.gravitino.meta.SchemaEntity;
+import org.apache.gravitino.meta.SchemaVersion;
+import org.awaitility.Awaitility;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -33,10 +47,87 @@ public class TestCacheConfig {
Assertions.assertTrue(config.get(Configs.CACHE_WEIGHER_ENABLED));
Assertions.assertEquals(10_000, config.get(Configs.CACHE_MAX_ENTRIES));
Assertions.assertEquals(3_600_000L,
config.get(Configs.CACHE_EXPIRATION_TIME));
- Assertions.assertEquals(200_302_000L, EntityCacheWeigher.getMaxWeight());
+ Assertions.assertEquals(40_000_000L, EntityCacheWeigher.getMaxWeight());
Assertions.assertEquals("caffeine",
config.get(Configs.CACHE_IMPLEMENTATION));
}
+ @Test
+ void testCaffeineCacheWithWeight() throws Exception {
+ Caffeine<Object, Object> builder = Caffeine.newBuilder();
+ builder.maximumWeight(500);
+ builder.weigher(EntityCacheWeigher.getInstance());
+ Cache<EntityCacheRelationKey, List<Entity>> cache = builder.build();
+
+ // Insert 3 metalakes
+ for (int i = 0; i < 3; i++) {
+ BaseMetalake baseMetalake =
+ BaseMetalake.builder()
+ .withName("metalake" + 1)
+ .withId((long) i)
+ .withVersion(SchemaVersion.V_0_1)
+ .withAuditInfo(AuditInfo.EMPTY)
+ .build();
+ cache.put(
+ EntityCacheRelationKey.of(NameIdentifier.of("metalake" + i),
Entity.EntityType.METALAKE),
+ List.of(baseMetalake));
+ }
+
+ // Insert 10 catalogs
+ for (int i = 0; i < 10; i++) {
+ CatalogEntity catalogEntity =
+ CatalogEntity.builder()
+ .withNamespace(Namespace.of("metalake1"))
+ .withName("catalog" + i)
+ .withProvider("provider")
+ .withAuditInfo(AuditInfo.EMPTY)
+ .withId((long) ((i + 1) * 100))
+ .withType(Catalog.Type.RELATIONAL)
+ .build();
+ cache.put(
+ EntityCacheRelationKey.of(
+ NameIdentifier.of("metalake1.catalog" + i),
Entity.EntityType.CATALOG),
+ List.of(catalogEntity));
+ }
+
+ // insert 100 schemas
+ for (int i = 0; i < 100; i++) {
+ SchemaEntity schemaEntity =
+ SchemaEntity.builder()
+ .withNamespace(Namespace.of("metalake1", "catalog1"))
+ .withName("schema" + i)
+ .withAuditInfo(AuditInfo.EMPTY)
+ .withId((long) ((i + 1) * 1000))
+ .build();
+
+ cache.put(
+ EntityCacheRelationKey.of(
+ NameIdentifier.of("metalake1.catalog1.schema" + i),
Entity.EntityType.SCHEMA),
+ List.of(schemaEntity));
+ }
+
+ // Three 3 metalakes still in cache.
+ for (int i = 0; i < 3; i++) {
+ Assertions.assertNotNull(
+ cache.getIfPresent(
+ EntityCacheRelationKey.of(
+ NameIdentifier.of("metalake" + 1),
Entity.EntityType.METALAKE)));
+ }
+
+ // 10 catalogs still in cache.
+ for (int i = 0; i < 10; i++) {
+ Assertions.assertNotNull(
+ cache.getIfPresent(
+ EntityCacheRelationKey.of(
+ NameIdentifier.of("metalake1.catalog" + i),
Entity.EntityType.CATALOG)));
+ }
+
+ // Only some of the 100 schemas are still in the cache, to be exact, 500 /
10 = 50 schemas.
+ Awaitility.await()
+ .atMost(Duration.ofSeconds(5))
+ .pollInterval(Duration.ofMillis(10))
+ .until(() -> cache.asMap().size() == 10 + 3 + 500 / 10);
+ }
+
@Test
void testSetConfigValues() {
Config config = new Config(false) {};
