From: Willem de Bruijn <will...@google.com>
The gro completion datapath is open coded for all protocols.
Deduplicate with new helper function net_gro_complete.
Signed-off-by: Willem de Bruijn <will...@google.com>
---
drivers/net/geneve.c | 9 +--------
include/linux/netdevice.h | 19 ++++++++++++++++++-
net/8021q/vlan.c | 10 +---------
net/core/dev.c | 24 +-----------------------
net/ethernet/eth.c | 11 +----------
net/ipv4/af_inet.c | 15 ++-------------
net/ipv4/fou.c | 25 +++----------------------
net/ipv4/gre_offload.c | 12 +++---------
net/ipv6/ip6_offload.c | 13 +------------
9 files changed, 31 insertions(+), 107 deletions(-)
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index 6625fabe2c88..a3a4621d9bee 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -488,7 +488,6 @@ static int geneve_gro_complete(struct sock *sk, struct
sk_buff *skb,
int nhoff)
{
struct genevehdr *gh;
- struct packet_offload *ptype;
__be16 type;
int gh_len;
int err = -ENOSYS;
@@ -497,13 +496,7 @@ static int geneve_gro_complete(struct sock *sk, struct
sk_buff *skb,
gh_len = geneve_hlen(gh);
type = gh->proto_type;
- rcu_read_lock();
- ptype = gro_find_complete_by_type(type);
- if (ptype)
- err = ptype->callbacks.gro_complete(skb, nhoff + gh_len);
-
- rcu_read_unlock();
-
+ err = net_gro_complete(dev_offloads, type, skb, nhoff + gh_len);
skb_set_inner_mac_header(skb, nhoff + gh_len);
return err;
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 7425068fa249..0d292ea6716e 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -3557,7 +3557,8 @@ void napi_gro_flush(struct napi_struct *napi, bool
flush_old);
struct sk_buff *napi_get_frags(struct napi_struct *napi);
gro_result_t napi_gro_frags(struct napi_struct *napi);
struct packet_offload *gro_find_receive_by_type(__be16 type);
-struct packet_offload *gro_find_complete_by_type(__be16 type);
+
+extern const struct net_offload __rcu *dev_offloads[256];
static inline u8 net_offload_from_type(u16 type)
{
@@ -3567,6 +3568,22 @@ static inline u8 net_offload_from_type(u16 type)
return type & 0xFF;
}
+static inline int net_gro_complete(const struct net_offload __rcu **offs,
+ u16 type, struct sk_buff *skb, int nhoff)
+{
+ const struct net_offload *off;
+ int ret = -ENOENT;
+
+ rcu_read_lock();
+ off = rcu_dereference(offs[net_offload_from_type(type)]);
+ if (off && off->callbacks.gro_complete &&
+ (!off->type || off->type == type))
+ ret = off->callbacks.gro_complete(skb, nhoff);
+ rcu_read_unlock();
+
+ return ret;
+}
+
static inline void napi_free_frags(struct napi_struct *napi)
{
kfree_skb(napi->skb);
diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c
index 5e9950453955..6ac27aa9f158 100644
--- a/net/8021q/vlan.c
+++ b/net/8021q/vlan.c
@@ -703,16 +703,8 @@ static int vlan_gro_complete(struct sk_buff *skb, int
nhoff)
{
struct vlan_hdr *vhdr = (struct vlan_hdr *)(skb->data + nhoff);
__be16 type = vhdr->h_vlan_encapsulated_proto;
- struct packet_offload *ptype;
- int err = -ENOENT;
- rcu_read_lock();
- ptype = gro_find_complete_by_type(type);
- if (ptype)
- err = ptype->callbacks.gro_complete(skb, nhoff + sizeof(*vhdr));
-
- rcu_read_unlock();
- return err;
+ return net_gro_complete(dev_offloads, type, skb, nhoff + sizeof(*vhdr));
}
static struct packet_offload vlan_packet_offloads[] __read_mostly = {
diff --git a/net/core/dev.c b/net/core/dev.c
index 55f86b6d3182..2c21e507291f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5235,10 +5235,6 @@ static void flush_all_backlogs(void)
static int napi_gro_complete(struct sk_buff *skb)
{
- const struct packet_offload *ptype;
- __be16 type = skb->protocol;
- int err = -ENOENT;
-
BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
if (NAPI_GRO_CB(skb)->count == 1) {
@@ -5246,13 +5242,7 @@ static int napi_gro_complete(struct sk_buff *skb)
goto out;
}
- rcu_read_lock();
- ptype = dev_offloads[net_offload_from_type(type)];
- if (ptype && ptype->callbacks.gro_complete)
- err = ptype->callbacks.gro_complete(skb, 0);
- rcu_read_unlock();
-
- if (err) {
+ if (net_gro_complete(dev_offloads, skb->protocol, skb, 0)) {
kfree_skb(skb);
return NET_RX_SUCCESS;
}
@@ -5505,18 +5495,6 @@ struct packet_offload *gro_find_receive_by_type(__be16
type)
}
EXPORT_SYMBOL(gro_find_receive_by_type);
-struct packet_offload *gro_find_complete_by_type(__be16 type)
-{
- struct net_offload *off;
-
- off = (struct net_offload *) rcu_dereference(dev_offloads[type & 0xFF]);
- if (off && off->type == type && off->callbacks.gro_complete)
- return off;
- else
- return NULL;
-}
-EXPORT_SYMBOL(gro_find_complete_by_type);
-
static void napi_skb_free_stolen_head(struct sk_buff *skb)
{
skb_dst_drop(skb);
diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c
index fd8faa0dfa61..fb17a13722e8 100644
--- a/net/ethernet/eth.c
+++ b/net/ethernet/eth.c
@@ -485,20 +485,11 @@ int eth_gro_complete(struct sk_buff *skb, int nhoff)
{
struct ethhdr *eh = (struct ethhdr *)(skb->data + nhoff);
__be16 type = eh->h_proto;
- struct packet_offload *ptype;
- int err = -ENOSYS;
if (skb->encapsulation)
skb_set_inner_mac_header(skb, nhoff);
- rcu_read_lock();
- ptype = gro_find_complete_by_type(type);
- if (ptype != NULL)
- err = ptype->callbacks.gro_complete(skb, nhoff +
- sizeof(struct ethhdr));
-
- rcu_read_unlock();
- return err;
+ return net_gro_complete(dev_offloads, type, skb, nhoff + sizeof(*eh));
}
EXPORT_SYMBOL(eth_gro_complete);
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 1fbe2f815474..1b72ee4a7811 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1560,9 +1560,7 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff)
{
__be16 newlen = htons(skb->len - nhoff);
struct iphdr *iph = (struct iphdr *)(skb->data + nhoff);
- const struct net_offload *ops;
int proto = iph->protocol;
- int err = -ENOSYS;
if (skb->encapsulation) {
skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IP));
@@ -1572,21 +1570,12 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff)
csum_replace2(&iph->check, iph->tot_len, newlen);
iph->tot_len = newlen;
- rcu_read_lock();
- ops = rcu_dereference(inet_offloads[proto]);
- if (WARN_ON(!ops || !ops->callbacks.gro_complete))
- goto out_unlock;
-
/* Only need to add sizeof(*iph) to get to the next hdr below
* because any hdr with option will have been flushed in
* inet_gro_receive().
*/
- err = ops->callbacks.gro_complete(skb, nhoff + sizeof(*iph));
-
-out_unlock:
- rcu_read_unlock();
-
- return err;
+ return net_gro_complete(inet_offloads, proto, skb,
+ nhoff + sizeof(*iph));
}
EXPORT_SYMBOL(inet_gro_complete);
diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c
index 500a59906b87..c42a3ef17864 100644
--- a/net/ipv4/fou.c
+++ b/net/ipv4/fou.c
@@ -261,24 +261,14 @@ static struct sk_buff *fou_gro_receive(struct sock *sk,
static int fou_gro_complete(struct sock *sk, struct sk_buff *skb,
int nhoff)
{
- const struct net_offload *ops;
u8 proto = fou_from_sock(sk)->protocol;
- int err = -ENOSYS;
const struct net_offload **offloads;
+ int err;
- rcu_read_lock();
offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
- ops = rcu_dereference(offloads[proto]);
- if (WARN_ON(!ops || !ops->callbacks.gro_complete))
- goto out_unlock;
-
- err = ops->callbacks.gro_complete(skb, nhoff);
-
+ err = net_gro_complete(offloads, proto, skb, nhoff);
skb_set_inner_mac_header(skb, nhoff);
-out_unlock:
- rcu_read_unlock();
-
return err;
}
@@ -457,7 +447,6 @@ static int gue_gro_complete(struct sock *sk, struct sk_buff
*skb, int nhoff)
{
const struct net_offload **offloads;
struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
- const struct net_offload *ops;
unsigned int guehlen = 0;
u8 proto;
int err = -ENOENT;
@@ -483,18 +472,10 @@ static int gue_gro_complete(struct sock *sk, struct
sk_buff *skb, int nhoff)
return err;
}
- rcu_read_lock();
offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
- ops = rcu_dereference(offloads[proto]);
- if (WARN_ON(!ops || !ops->callbacks.gro_complete))
- goto out_unlock;
-
- err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
-
+ err = net_gro_complete(offloads, proto, skb, nhoff + guehlen);
skb_set_inner_mac_header(skb, nhoff + guehlen);
-out_unlock:
- rcu_read_unlock();
return err;
}
diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c
index 6c63524f598a..fc8c99e4a058 100644
--- a/net/ipv4/gre_offload.c
+++ b/net/ipv4/gre_offload.c
@@ -231,10 +231,9 @@ static struct sk_buff *gre_gro_receive(struct list_head
*head,
static int gre_gro_complete(struct sk_buff *skb, int nhoff)
{
struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
- struct packet_offload *ptype;
unsigned int grehlen = sizeof(*greh);
- int err = -ENOENT;
__be16 type;
+ int err;
skb->encapsulation = 1;
skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
@@ -246,13 +245,8 @@ static int gre_gro_complete(struct sk_buff *skb, int nhoff)
if (greh->flags & GRE_CSUM)
grehlen += GRE_HEADER_SECTION;
- rcu_read_lock();
- ptype = gro_find_complete_by_type(type);
- if (ptype)
- err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
-
- rcu_read_unlock();
-
+ err = net_gro_complete(dev_offloads, type, skb,
+ nhoff + sizeof(*greh));
skb_set_inner_mac_header(skb, nhoff + grehlen);
return err;
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index c7e495f12011..e8bf554ae611 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -298,7 +298,6 @@ static int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
{
const struct net_offload *ops;
struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + nhoff);
- int err = -ENOSYS;
if (skb->encapsulation) {
skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));
@@ -307,18 +306,8 @@ static int ipv6_gro_complete(struct sk_buff *skb, int
nhoff)
iph->payload_len = htons(skb->len - nhoff - sizeof(*iph));
- rcu_read_lock();
-
nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
- if (WARN_ON(!ops || !ops->callbacks.gro_complete))
- goto out_unlock;
-
- err = ops->callbacks.gro_complete(skb, nhoff);
-
-out_unlock:
- rcu_read_unlock();
-
- return err;
+ return net_gro_complete(inet6_offloads, ops->type, skb, nhoff);
}
static int sit_gro_complete(struct sk_buff *skb, int nhoff)
--
2.19.0.397.gdd90340f6a-goog
