Implement RCU variants for existing dst_cache helpers interacting with
dst_entry.
The new helpers avoid referencing the dst_entry, sparing some unnecessary
atomic operations. They should only be used in flows that are already
guaranteed to be inside a RCU read-side critical context.
Signed-off-by: Marek Mietus <mmietu...@yahoo.com>
---
include/net/dst_cache.h | 26 ++++++++++++++
net/core/dst_cache.c | 78 +++++++++++++++++++++++++++++++++++------
2 files changed, 94 insertions(+), 10 deletions(-)
diff --git a/include/net/dst_cache.h b/include/net/dst_cache.h
index 1961699598e2..d3bf616a6e6f 100644
--- a/include/net/dst_cache.h
+++ b/include/net/dst_cache.h
@@ -32,6 +32,21 @@ struct dst_entry *dst_cache_get(struct dst_cache *dst_cache);
*/
struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr);
+/**
+ * dst_cache_get_ip4_rcu - lookup cache and ipv4 source under RCU
+ * @dst_cache: the cache
+ * @saddr: return value for the retrieved source address
+ *
+ * Perform cache lookup and fetch ipv4 source without taking a
+ * reference on the dst.
+ * Must be called with local BH disabled, and within an rcu read side
+ * critical section.
+ *
+ * Return: Pointer to retrieved rtable if cache is initialized and
+ * cached dst is valid, NULL otherwise.
+ */
+struct rtable *dst_cache_get_ip4_rcu(struct dst_cache *dst_cache, __be32
*saddr);
+
/**
* dst_cache_set_ip4 - store the ipv4 dst into the cache
* @dst_cache: the cache
@@ -43,6 +58,17 @@ struct rtable *dst_cache_get_ip4(struct dst_cache
*dst_cache, __be32 *saddr);
void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
__be32 saddr);
+/**
+ * dst_cache_steal_ip4 - store the ipv4 dst into the cache and steal its
+ * reference
+ * @dst_cache: the cache
+ * @dst: the entry to be cached whose reference will be stolen
+ * @saddr: the source address to be stored inside the cache
+ *
+ * local BH must be disabled
+ */
+void dst_cache_steal_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
+ __be32 saddr);
#if IS_ENABLED(CONFIG_IPV6)
/**
diff --git a/net/core/dst_cache.c b/net/core/dst_cache.c
index 9ab4902324e1..f1e3992d8171 100644
--- a/net/core/dst_cache.c
+++ b/net/core/dst_cache.c
@@ -25,20 +25,30 @@ struct dst_cache_pcpu {
};
};
-static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
- struct dst_entry *dst, u32 cookie)
+static void __dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
+ struct dst_entry *dst, u32 cookie)
{
DEBUG_NET_WARN_ON_ONCE(!in_softirq());
dst_release(dst_cache->dst);
- if (dst)
- dst_hold(dst);
dst_cache->cookie = cookie;
dst_cache->dst = dst;
}
-static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
- struct dst_cache_pcpu *idst)
+static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
+ struct dst_entry *dst, u32 cookie)
+{
+ if (dst == dst_cache->dst && cookie == dst_cache->cookie)
+ return;
+
+ if (dst)
+ dst_hold(dst);
+
+ __dst_cache_per_cpu_dst_set(dst_cache, dst, cookie);
+}
+
+static struct dst_entry *__dst_cache_per_cpu_get(struct dst_cache *dst_cache,
+ struct dst_cache_pcpu *idst)
{
struct dst_entry *dst;
@@ -47,14 +57,10 @@ static struct dst_entry *dst_cache_per_cpu_get(struct
dst_cache *dst_cache,
if (!dst)
goto fail;
- /* the cache already hold a dst reference; it can't go away */
- dst_hold(dst);
-
if (unlikely(!time_after(idst->refresh_ts,
READ_ONCE(dst_cache->reset_ts)) ||
(READ_ONCE(dst->obsolete) && !dst->ops->check(dst,
idst->cookie)))) {
dst_cache_per_cpu_dst_set(idst, NULL, 0);
- dst_release(dst);
goto fail;
}
return dst;
@@ -64,6 +70,18 @@ static struct dst_entry *dst_cache_per_cpu_get(struct
dst_cache *dst_cache,
return NULL;
}
+static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
+ struct dst_cache_pcpu *idst)
+{
+ struct dst_entry *dst;
+
+ dst = __dst_cache_per_cpu_get(dst_cache, idst);
+ if (dst)
+ /* the cache already hold a dst reference; it can't go away */
+ dst_hold(dst);
+ return dst;
+}
+
struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
{
struct dst_entry *dst;
@@ -100,6 +118,28 @@ struct rtable *dst_cache_get_ip4(struct dst_cache
*dst_cache, __be32 *saddr)
}
EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
+struct rtable *dst_cache_get_ip4_rcu(struct dst_cache *dst_cache, __be32
*saddr)
+{
+ struct dst_cache_pcpu *idst;
+ struct dst_entry *dst;
+
+ if (!dst_cache->cache)
+ return NULL;
+
+ local_lock_nested_bh(&dst_cache->cache->bh_lock);
+ idst = this_cpu_ptr(dst_cache->cache);
+ dst = __dst_cache_per_cpu_get(dst_cache, idst);
+ if (!dst) {
+ local_unlock_nested_bh(&dst_cache->cache->bh_lock);
+ return NULL;
+ }
+
+ *saddr = idst->in_saddr.s_addr;
+ local_unlock_nested_bh(&dst_cache->cache->bh_lock);
+ return dst_rtable(dst);
+}
+EXPORT_SYMBOL_GPL(dst_cache_get_ip4_rcu);
+
void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
__be32 saddr)
{
@@ -116,6 +156,24 @@ void dst_cache_set_ip4(struct dst_cache *dst_cache, struct
dst_entry *dst,
}
EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
+void dst_cache_steal_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
+ __be32 saddr)
+{
+ struct dst_cache_pcpu *idst;
+
+ if (!dst_cache->cache) {
+ dst_release(dst);
+ return;
+ }
+
+ local_lock_nested_bh(&dst_cache->cache->bh_lock);
+ idst = this_cpu_ptr(dst_cache->cache);
+ __dst_cache_per_cpu_dst_set(idst, dst, 0);
+ idst->in_saddr.s_addr = saddr;
+ local_unlock_nested_bh(&dst_cache->cache->bh_lock);
+}
+EXPORT_SYMBOL_GPL(dst_cache_steal_ip4);
+
#if IS_ENABLED(CONFIG_IPV6)
void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
const struct in6_addr *saddr)
--
2.51.0
_______________________________________________
Openvpn-devel mailing list
Openvpn-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/openvpn-devel
