yuqi1129 commented on code in PR #13106: URL: https://github.com/apache/gravitino/pull/13106#discussion_r4001791962
########## core/src/main/java/org/apache/gravitino/cache/RedisEntityCache.java: ########## @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cache; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Config; +import org.apache.gravitino.Configs; +import org.apache.gravitino.Entity; +import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.utils.HierarchicalSchemaUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.ConnectionPool; +import redis.clients.jedis.DefaultJedisClientConfig; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisClientConfig; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.UnifiedJedis; +import redis.clients.jedis.commands.KeyCommands; +import redis.clients.jedis.commands.SortedSetCommands; +import redis.clients.jedis.exceptions.JedisException; +import redis.clients.jedis.params.ScanParams; +import redis.clients.jedis.resps.ScanResult; + +/** + * An {@link EntityCache} that keeps one copy of every cached entity in Redis, shared by all nodes + * of a Gravitino cluster. It is selected with {@code gravitino.cache.implementation=redis} and + * reports {@link Coherence#SHARED}: a write on any node invalidates the single shared copy, so no + * per-node propagation is needed. + * + * <p>This is a shared cache, not a strongly consistent database and cache pair. The cache is only + * touched after the entity store has committed, and an invalidation deletes the entry rather than + * updating it in place, so a Redis failure degrades to a cache miss and never to a stale hit. What + * remains is the window between a store commit and the invalidation that follows it: a node that + * dies inside that window leaves the entry readable until its TTL expires. Callers that cannot + * tolerate that must read the store directly. + * + * <p><b>Keyspace.</b> See {@link RedisKeyspace}. All keys of one metalake share a Redis Cluster + * hash slot, so every script here is single-slot at any depth; the cost is that one metalake's + * entries are bounded by one cluster node. + * + * <p><b>Container drop.</b> {@link #invalidate(NameIdentifier, Entity.EntityType)} runs one Lua + * script that bumps the version fence of the dropped identifier, deletes its value, and walks the + * lexicographic index range of its descendants deleting each one. The script is atomic, so a + * concurrent reader sees the subtree either complete or gone, never half dropped. + * + * <p><b>Stale-write guard.</b> A read miss records the current fence of the identifier and of each + * of its ancestors for the calling thread. The write that fills the entry afterwards is a Lua + * script that compares those fences again and refuses to write if any has moved. A load that began + * before a drop of the entity, or of any container above it, therefore cannot refill the key once + * the drop has committed, whether or not the entity was indexed at the time. Fences outlive the + * value TTL (see {@code gravitino.cache.redis.fenceTtlMs}), so a slow reader cannot win by + * outliving the value. A write with no recorded fences, such as caching a freshly inserted entity, + * is written unconditionally. + * + * <p><b>Failure policy.</b> Reads and fills are optimizations: a Redis error or timeout makes them + * a miss or a no-op. An invalidation is a correctness obligation: a failure is propagated as a + * {@link RuntimeException} so it is never silently dropped, and the entry expires by TTL at the + * latest. An unreachable Redis at startup fails fast. + */ +public class RedisEntityCache extends BaseEntityCache { Review Comment: Since Redis has strong consistency, I think we can change the logic about `CACHEABLE_TYPES` in `BaseEntityCache`. We only cache some types of Gravitino entities due to different consistency requirements for in-memory cache; however, since Redis does not suffer from the same problem as caffeine cache, so we can call things as long as we make truely make them strong consistency in Redis. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
