Signed-off-by: Edward Cree <ec...@solarflare.com>
---
include/linux/netdevice.h | 4 +--
include/net/ip.h | 4 +--
include/net/ipv6.h | 4 +--
net/core/dev.c | 63 +++++++++++++++++++++++++++++------------------
net/ipv4/ip_input.c | 39 ++++++++++++++++++-----------
net/ipv6/ip6_input.c | 37 +++++++++++++++++-----------
6 files changed, 92 insertions(+), 59 deletions(-)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 2b53536b1d99..9b3fc5944ba5 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2349,7 +2349,7 @@ struct packet_type {
struct net_device *,
struct packet_type *,
struct net_device *);
- void (*list_func) (struct list_head *,
+ int (*list_func) (struct list_head *,
struct packet_type *,
struct net_device *);
bool (*id_match)(struct packet_type *ptype,
@@ -3546,7 +3546,7 @@ int netif_rx(struct sk_buff *skb);
int netif_rx_ni(struct sk_buff *skb);
int netif_receive_skb(struct sk_buff *skb);
int netif_receive_skb_core(struct sk_buff *skb);
-void netif_receive_skb_list(struct list_head *head);
+int netif_receive_skb_list(struct list_head *head);
gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
int napi_gro_receive_list(struct napi_struct *napi, struct list_head *head);
void napi_gro_flush(struct napi_struct *napi, bool flush_old);
diff --git a/include/net/ip.h b/include/net/ip.h
index e44b1a44f67a..aab1f7eea1e1 100644
--- a/include/net/ip.h
+++ b/include/net/ip.h
@@ -152,8 +152,8 @@ int ip_build_and_send_pkt(struct sk_buff *skb, const struct
sock *sk,
struct ip_options_rcu *opt);
int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt,
struct net_device *orig_dev);
-void ip_list_rcv(struct list_head *head, struct packet_type *pt,
- struct net_device *orig_dev);
+int ip_list_rcv(struct list_head *head, struct packet_type *pt,
+ struct net_device *orig_dev);
int ip_local_deliver(struct sk_buff *skb);
int ip_mr_input(struct sk_buff *skb);
int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb);
diff --git a/include/net/ipv6.h b/include/net/ipv6.h
index ff33f498c137..f15651eabfe0 100644
--- a/include/net/ipv6.h
+++ b/include/net/ipv6.h
@@ -914,8 +914,8 @@ static inline __be32 flowi6_get_flowlabel(const struct
flowi6 *fl6)
int ipv6_rcv(struct sk_buff *skb, struct net_device *dev,
struct packet_type *pt, struct net_device *orig_dev);
-void ipv6_list_rcv(struct list_head *head, struct packet_type *pt,
- struct net_device *orig_dev);
+int ipv6_list_rcv(struct list_head *head, struct packet_type *pt,
+ struct net_device *orig_dev);
int ip6_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb);
diff --git a/net/core/dev.c b/net/core/dev.c
index 8df39ded77bd..69e2819994e4 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4922,24 +4922,27 @@ int netif_receive_skb_core(struct sk_buff *skb)
}
EXPORT_SYMBOL(netif_receive_skb_core);
-static inline void __netif_receive_skb_list_ptype(struct list_head *head,
- struct packet_type *pt_prev,
- struct net_device *orig_dev)
+static inline int __netif_receive_skb_list_ptype(struct list_head *head,
+ struct packet_type *pt_prev,
+ struct net_device *orig_dev)
{
struct sk_buff *skb, *next;
+ int kept = 0;
if (!pt_prev)
- return;
+ return 0;
if (list_empty(head))
- return;
+ return 0;
if (pt_prev->list_func != NULL)
- pt_prev->list_func(head, pt_prev, orig_dev);
+ kept = pt_prev->list_func(head, pt_prev, orig_dev);
else
list_for_each_entry_safe(skb, next, head, list)
- pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
+ if (pt_prev->func(skb, skb->dev, pt_prev, orig_dev) ==
NET_RX_SUCCESS)
+ kept++;
+ return kept;
}
-static void __netif_receive_skb_list_core(struct list_head *head, bool
pfmemalloc)
+static int __netif_receive_skb_list_core(struct list_head *head, bool
pfmemalloc)
{
/* Fast-path assumptions:
* - There is no RX handler.
@@ -4956,6 +4959,7 @@ static void __netif_receive_skb_list_core(struct
list_head *head, bool pfmemallo
struct net_device *od_curr = NULL;
struct list_head sublist;
struct sk_buff *skb, *next;
+ int kept = 0, ret;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
@@ -4963,12 +4967,15 @@ static void __netif_receive_skb_list_core(struct
list_head *head, bool pfmemallo
struct packet_type *pt_prev = NULL;
list_del(&skb->list);
- __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
- if (!pt_prev)
+ ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
+ if (!pt_prev) {
+ if (ret == NET_RX_SUCCESS)
+ kept++;
continue;
+ }
if (pt_curr != pt_prev || od_curr != orig_dev) {
/* dispatch old sublist */
- __netif_receive_skb_list_ptype(&sublist, pt_curr,
od_curr);
+ kept += __netif_receive_skb_list_ptype(&sublist,
pt_curr, od_curr);
/* start new sublist */
INIT_LIST_HEAD(&sublist);
pt_curr = pt_prev;
@@ -4978,7 +4985,8 @@ static void __netif_receive_skb_list_core(struct
list_head *head, bool pfmemallo
}
/* dispatch final sublist */
- __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
+ kept += __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
+ return kept;
}
static int __netif_receive_skb(struct sk_buff *skb)
@@ -5006,11 +5014,12 @@ static int __netif_receive_skb(struct sk_buff *skb)
return ret;
}
-static void __netif_receive_skb_list(struct list_head *head)
+static int __netif_receive_skb_list(struct list_head *head)
{
unsigned long noreclaim_flag = 0;
struct sk_buff *skb, *next;
bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
+ int kept = 0;
list_for_each_entry_safe(skb, next, head, list) {
if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc)
{
@@ -5019,7 +5028,7 @@ static void __netif_receive_skb_list(struct list_head
*head)
/* Handle the previous sublist */
list_cut_before(&sublist, head, &skb->list);
if (!list_empty(&sublist))
- __netif_receive_skb_list_core(&sublist,
pfmemalloc);
+ kept += __netif_receive_skb_list_core(&sublist,
pfmemalloc);
pfmemalloc = !pfmemalloc;
/* See comments in __netif_receive_skb */
if (pfmemalloc)
@@ -5030,10 +5039,11 @@ static void __netif_receive_skb_list(struct list_head
*head)
}
/* Handle the remaining sublist */
if (!list_empty(head))
- __netif_receive_skb_list_core(head, pfmemalloc);
+ kept += __netif_receive_skb_list_core(head, pfmemalloc);
/* Restore pflags */
if (pfmemalloc)
memalloc_noreclaim_restore(noreclaim_flag);
+ return kept;
}
static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
@@ -5109,17 +5119,20 @@ static int netif_receive_skb_internal(struct sk_buff
*skb)
return ret;
}
-static void netif_receive_skb_list_internal(struct list_head *head)
+static int netif_receive_skb_list_internal(struct list_head *head)
{
struct bpf_prog *xdp_prog = NULL;
struct sk_buff *skb, *next;
struct list_head sublist;
+ int kept = 0;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
net_timestamp_check(netdev_tstamp_prequeue, skb);
list_del(&skb->list);
- if (!skb_defer_rx_timestamp(skb))
+ if (skb_defer_rx_timestamp(skb))
+ kept++;
+ else
list_add_tail(&skb->list, &sublist);
}
list_splice_init(&sublist, head);
@@ -5149,13 +5162,15 @@ static void netif_receive_skb_list_internal(struct
list_head *head)
if (cpu >= 0) {
/* Will be handled, remove from list */
list_del(&skb->list);
- enqueue_to_backlog(skb, cpu,
&rflow->last_qtail);
+ if (enqueue_to_backlog(skb, cpu,
&rflow->last_qtail) == NET_RX_SUCCESS)
+ kept++;
}
}
}
#endif
- __netif_receive_skb_list(head);
+ kept += __netif_receive_skb_list(head);
rcu_read_unlock();
+ return kept;
}
/**
@@ -5185,21 +5200,21 @@ EXPORT_SYMBOL(netif_receive_skb);
* netif_receive_skb_list - process many receive buffers from network
* @head: list of skbs to process.
*
- * Since return value of netif_receive_skb() is normally ignored, and
- * wouldn't be meaningful for a list, this function returns void.
+ * Returns the number of skbs for which netif_receive_skb() would have
+ * returned %NET_RX_SUCCESS.
*
* This function may only be called from softirq context and interrupts
* should be enabled.
*/
-void netif_receive_skb_list(struct list_head *head)
+int netif_receive_skb_list(struct list_head *head)
{
struct sk_buff *skb;
if (list_empty(head))
- return;
+ return 0;
list_for_each_entry(skb, head, list)
trace_netif_receive_skb_list_entry(skb);
- netif_receive_skb_list_internal(head);
+ return netif_receive_skb_list_internal(head);
}
EXPORT_SYMBOL(netif_receive_skb_list);
diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c
index 3196cf58f418..75cc5a6ef9b8 100644
--- a/net/ipv4/ip_input.c
+++ b/net/ipv4/ip_input.c
@@ -526,9 +526,10 @@ int ip_rcv(struct sk_buff *skb, struct net_device *dev,
struct packet_type *pt,
ip_rcv_finish);
}
-static void ip_sublist_rcv_finish(struct list_head *head)
+static int ip_sublist_rcv_finish(struct list_head *head)
{
struct sk_buff *skb, *next;
+ int kept = 0;
list_for_each_entry_safe(skb, next, head, list) {
list_del(&skb->list);
@@ -536,16 +537,19 @@ static void ip_sublist_rcv_finish(struct list_head *head)
* another kind of SKB-list usage (see validate_xmit_skb_list)
*/
skb->next = NULL;
- dst_input(skb);
+ if (dst_input(skb) == NET_RX_SUCCESS)
+ kept++;
}
+ return kept;
}
-static void ip_list_rcv_finish(struct net *net, struct sock *sk,
- struct list_head *head)
+static int ip_list_rcv_finish(struct net *net, struct sock *sk,
+ struct list_head *head)
{
struct dst_entry *curr_dst = NULL;
struct sk_buff *skb, *next;
struct list_head sublist;
+ int kept = 0;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
@@ -556,8 +560,10 @@ static void ip_list_rcv_finish(struct net *net, struct
sock *sk,
* skb to its handler for processing
*/
skb = l3mdev_ip_rcv(skb);
- if (!skb)
+ if (!skb) {
+ kept++;
continue;
+ }
if (ip_rcv_finish_core(net, sk, skb) == NET_RX_DROP)
continue;
@@ -565,7 +571,7 @@ static void ip_list_rcv_finish(struct net *net, struct sock
*sk,
if (curr_dst != dst) {
/* dispatch old sublist */
if (!list_empty(&sublist))
- ip_sublist_rcv_finish(&sublist);
+ kept += ip_sublist_rcv_finish(&sublist);
/* start new sublist */
INIT_LIST_HEAD(&sublist);
curr_dst = dst;
@@ -573,25 +579,27 @@ static void ip_list_rcv_finish(struct net *net, struct
sock *sk,
list_add_tail(&skb->list, &sublist);
}
/* dispatch final sublist */
- ip_sublist_rcv_finish(&sublist);
+ kept += ip_sublist_rcv_finish(&sublist);
+ return kept;
}
-static void ip_sublist_rcv(struct list_head *head, struct net_device *dev,
- struct net *net)
+static int ip_sublist_rcv(struct list_head *head, struct net_device *dev,
+ struct net *net)
{
NF_HOOK_LIST(NFPROTO_IPV4, NF_INET_PRE_ROUTING, net, NULL,
head, dev, NULL, ip_rcv_finish);
- ip_list_rcv_finish(net, NULL, head);
+ return ip_list_rcv_finish(net, NULL, head);
}
-/* Receive a list of IP packets */
-void ip_list_rcv(struct list_head *head, struct packet_type *pt,
- struct net_device *orig_dev)
+/* Receive a list of IP packets; return number of successful receives */
+int ip_list_rcv(struct list_head *head, struct packet_type *pt,
+ struct net_device *orig_dev)
{
struct net_device *curr_dev = NULL;
struct net *curr_net = NULL;
struct sk_buff *skb, *next;
struct list_head sublist;
+ int kept = 0;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
@@ -606,7 +614,7 @@ void ip_list_rcv(struct list_head *head, struct packet_type
*pt,
if (curr_dev != dev || curr_net != net) {
/* dispatch old sublist */
if (!list_empty(&sublist))
- ip_sublist_rcv(&sublist, curr_dev, curr_net);
+ kept += ip_sublist_rcv(&sublist, curr_dev,
curr_net);
/* start new sublist */
INIT_LIST_HEAD(&sublist);
curr_dev = dev;
@@ -615,5 +623,6 @@ void ip_list_rcv(struct list_head *head, struct packet_type
*pt,
list_add_tail(&skb->list, &sublist);
}
/* dispatch final sublist */
- ip_sublist_rcv(&sublist, curr_dev, curr_net);
+ kept += ip_sublist_rcv(&sublist, curr_dev, curr_net);
+ return kept;
}
diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c
index 6242682be876..e64b830c9f0f 100644
--- a/net/ipv6/ip6_input.c
+++ b/net/ipv6/ip6_input.c
@@ -76,20 +76,24 @@ int ip6_rcv_finish(struct net *net, struct sock *sk, struct
sk_buff *skb)
return dst_input(skb);
}
-static void ip6_sublist_rcv_finish(struct list_head *head)
+static int ip6_sublist_rcv_finish(struct list_head *head)
{
struct sk_buff *skb, *next;
+ int kept = 0;
list_for_each_entry_safe(skb, next, head, list)
- dst_input(skb);
+ if (dst_input(skb) == NET_RX_SUCCESS)
+ kept++;
+ return kept;
}
-static void ip6_list_rcv_finish(struct net *net, struct sock *sk,
- struct list_head *head)
+static int ip6_list_rcv_finish(struct net *net, struct sock *sk,
+ struct list_head *head)
{
struct dst_entry *curr_dst = NULL;
struct sk_buff *skb, *next;
struct list_head sublist;
+ int kept = 0;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
@@ -100,14 +104,16 @@ static void ip6_list_rcv_finish(struct net *net, struct
sock *sk,
* skb to its handler for processing
*/
skb = l3mdev_ip6_rcv(skb);
- if (!skb)
+ if (!skb) {
+ kept++;
continue;
+ }
ip6_rcv_finish_core(net, sk, skb);
dst = skb_dst(skb);
if (curr_dst != dst) {
/* dispatch old sublist */
if (!list_empty(&sublist))
- ip6_sublist_rcv_finish(&sublist);
+ kept += ip6_sublist_rcv_finish(&sublist);
/* start new sublist */
INIT_LIST_HEAD(&sublist);
curr_dst = dst;
@@ -115,7 +121,8 @@ static void ip6_list_rcv_finish(struct net *net, struct
sock *sk,
list_add_tail(&skb->list, &sublist);
}
/* dispatch final sublist */
- ip6_sublist_rcv_finish(&sublist);
+ kept += ip6_sublist_rcv_finish(&sublist);
+ return kept;
}
static struct sk_buff *ip6_rcv_core(struct sk_buff *skb, struct net_device
*dev,
@@ -273,22 +280,23 @@ int ipv6_rcv(struct sk_buff *skb, struct net_device *dev,
struct packet_type *pt
ip6_rcv_finish);
}
-static void ip6_sublist_rcv(struct list_head *head, struct net_device *dev,
- struct net *net)
+static int ip6_sublist_rcv(struct list_head *head, struct net_device *dev,
+ struct net *net)
{
NF_HOOK_LIST(NFPROTO_IPV6, NF_INET_PRE_ROUTING, net, NULL,
head, dev, NULL, ip6_rcv_finish);
- ip6_list_rcv_finish(net, NULL, head);
+ return ip6_list_rcv_finish(net, NULL, head);
}
/* Receive a list of IPv6 packets */
-void ipv6_list_rcv(struct list_head *head, struct packet_type *pt,
- struct net_device *orig_dev)
+int ipv6_list_rcv(struct list_head *head, struct packet_type *pt,
+ struct net_device *orig_dev)
{
struct net_device *curr_dev = NULL;
struct net *curr_net = NULL;
struct sk_buff *skb, *next;
struct list_head sublist;
+ int kept = 0;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
@@ -303,7 +311,7 @@ void ipv6_list_rcv(struct list_head *head, struct
packet_type *pt,
if (curr_dev != dev || curr_net != net) {
/* dispatch old sublist */
if (!list_empty(&sublist))
- ip6_sublist_rcv(&sublist, curr_dev, curr_net);
+ kept += ip6_sublist_rcv(&sublist, curr_dev,
curr_net);
/* start new sublist */
INIT_LIST_HEAD(&sublist);
curr_dev = dev;
@@ -312,7 +320,8 @@ void ipv6_list_rcv(struct list_head *head, struct
packet_type *pt,
list_add_tail(&skb->list, &sublist);
}
/* dispatch final sublist */
- ip6_sublist_rcv(&sublist, curr_dev, curr_net);
+ kept += ip6_sublist_rcv(&sublist, curr_dev, curr_net);
+ return kept;
}
/*
