Ok, heres the next iteration based on feedback from Thomas and Peter.
It seems this is good form in matching what the original code does.
I have a few doubts in labelled as XXX in there. Thomas, if you have
time, do you mind looking at those?
The usual testing happened i.e "if you can see this email, it must
have worked";->
cheers,
jamal
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index f671cd2..718d6fd 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -83,6 +83,9 @@ struct wireless_dev;
#define NETDEV_TX_OK 0 /* driver took care of packet */
#define NETDEV_TX_BUSY 1 /* driver tx path was busy*/
#define NETDEV_TX_LOCKED -1 /* driver tx lock was already taken */
+#define NETDEV_TX_DROP -2 /* request caller to drop packet */
+#define NETDEV_TX_QUEUE -3 /* request caller to requeue packet */
+
/*
* Compute the worst case header length according to the protocols
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index f28bb2d..91cfcd5 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -59,7 +59,79 @@ void qdisc_unlock_tree(struct net_device *dev)
spin_unlock_bh(&dev->queue_lock);
}
+static inline int qdisc_qlen(struct Qdisc *q)
+{
+ BUG_ON((int) q->q.qlen < 0);
+ return q->q.qlen;
+}
+
+static inline int handle_dev_cpu_collision(struct net_device *dev)
+{
+ if (dev->xmit_lock_owner == smp_processor_id()) {
+ if (net_ratelimit())
+ printk(KERN_DEBUG
+ "Dead loop on netdevice %s, fix it urgently!\n",
+ dev->name);
+ return NETDEV_TX_DROP;
+ }
+ /* XXX: This maintains original code, but i think we should
+ * update cpu_collision always
+ */
+ __get_cpu_var(netdev_rx_stat).cpu_collision++;
+ return NETDEV_TX_QUEUE;
+}
+
+static inline int
+handle_dev_requeue(struct sk_buff *skb, struct net_device *dev, struct Qdisc *q)
+{
+
+ if (unlikely(q == &noop_qdisc))
+ kfree_skb(skb);
+ else if (skb->next)
+ dev->gso_skb = skb;
+ else
+ q->ops->requeue(skb, q);
+ /* XXX: Could netif_schedule fail? Or is that fact we are
+ * requeueing imply the hardware path is closed
+ * and even if we fail, some interupt will wake us
+ */
+ netif_schedule(dev);
+ return 0;
+}
+
+static inline struct sk_buff *
+try_get_tx_pkt(struct net_device *dev, struct Qdisc *q)
+{
+ struct sk_buff *skb = dev->gso_skb;
+
+ if (skb)
+ dev->gso_skb = NULL;
+ else
+ skb = q->dequeue(q);
+
+ return skb;
+}
+
+static inline int
+handle_tx_locked(struct sk_buff *skb, struct net_device *dev, struct Qdisc *q)
+{
+ int ret = handle_dev_cpu_collision(dev);
+
+ if (ret == NETDEV_TX_DROP) {
+ kfree_skb(skb);
+ return qdisc_qlen(q);
+ }
+
+ return handle_dev_requeue(skb, dev, q);
+}
+
+
/*
+ NOTE: Called under dev->queue_lock with locally disabled BH.
+
+ __LINK_STATE_QDISC_RUNNING guarantees only one CPU
+ can enter this region at a time.
+
dev->queue_lock serializes queue accesses for this device
AND dev->qdisc pointer itself.
@@ -67,114 +139,66 @@ void qdisc_unlock_tree(struct net_device *dev)
dev->queue_lock and netif_tx_lock are mutually exclusive,
if one is grabbed, another must be free.
- */
+ Multiple CPUs may contend for the two locks.
-/* Kick device.
+ Note, that this procedure can be called by a watchdog timer
+ Returns to the caller:
Returns: 0 - queue is empty or throttled.
>0 - queue is not empty.
-
- NOTE: Called under dev->queue_lock with locally disabled BH.
*/
static inline int qdisc_restart(struct net_device *dev)
{
struct Qdisc *q = dev->qdisc;
+ unsigned lockless = (dev->features & NETIF_F_LLTX);
struct sk_buff *skb;
+ int ret;
- /* Dequeue packet */
- if (((skb = dev->gso_skb)) || ((skb = q->dequeue(q)))) {
- unsigned nolock = (dev->features & NETIF_F_LLTX);
-
- dev->gso_skb = NULL;
+ skb = try_get_tx_pkt(dev, q);
+ if (skb == NULL)
+ return qdisc_qlen(q);
- /*
- * When the driver has LLTX set it does its own locking
- * in start_xmit. No need to add additional overhead by
- * locking again. These checks are worth it because
- * even uncongested locks can be quite expensive.
- * The driver can do trylock like here too, in case
- * of lock congestion it should return -1 and the packet
- * will be requeued.
- */
- if (!nolock) {
- if (!netif_tx_trylock(dev)) {
- collision:
- /* So, someone grabbed the driver. */
-
- /* It may be transient configuration error,
- when hard_start_xmit() recurses. We detect
- it by checking xmit owner and drop the
- packet when deadloop is detected.
- */
- if (dev->xmit_lock_owner == smp_processor_id()) {
- kfree_skb(skb);
- if (net_ratelimit())
- printk(KERN_DEBUG "Dead loop on netdevice %s, fix it urgently!\n", dev->name);
- goto out;
- }
- __get_cpu_var(netdev_rx_stat).cpu_collision++;
- goto requeue;
- }
- }
-
- {
- /* And release queue */
- spin_unlock(&dev->queue_lock);
-
- if (!netif_queue_stopped(dev)) {
- int ret;
-
- ret = dev_hard_start_xmit(skb, dev);
- if (ret == NETDEV_TX_OK) {
- if (!nolock) {
- netif_tx_unlock(dev);
- }
- spin_lock(&dev->queue_lock);
- q = dev->qdisc;
- goto out;
- }
- if (ret == NETDEV_TX_LOCKED && nolock) {
- spin_lock(&dev->queue_lock);
- q = dev->qdisc;
- goto collision;
- }
- }
-
- /* NETDEV_TX_BUSY - we need to requeue */
- /* Release the driver */
- if (!nolock) {
- netif_tx_unlock(dev);
- }
- spin_lock(&dev->queue_lock);
- q = dev->qdisc;
- }
-
- /* Device kicked us out :(
- This is possible in three cases:
-
- 0. driver is locked
- 1. fastroute is enabled
- 2. device cannot determine busy state
- before start of transmission (f.e. dialout)
- 3. device is buggy (ppp)
- */
-
-requeue:
- if (unlikely(q == &noop_qdisc))
- kfree_skb(skb);
- else if (skb->next)
- dev->gso_skb = skb;
- else
- q->ops->requeue(skb, q);
- netif_schedule(dev);
- return 0;
+ /* we have a packet to send */
+ if (!lockless) {
+ if (!netif_tx_trylock(dev))
+ return handle_tx_locked(skb, dev, q);
+ }
+
+ /* all clear .. */
+ spin_unlock(&dev->queue_lock);
+
+ ret = NETDEV_TX_BUSY;
+ if (!netif_queue_stopped(dev))
+ /* churn baby churn .. */
+ ret = dev_hard_start_xmit(skb, dev);
+
+ if (!lockless)
+ netif_tx_unlock(dev);
+
+ spin_lock(&dev->queue_lock);
+
+ /* we need to refresh q because it may be invalid since we
+ * dropped dev->queue_lock earlier ...
+ * So dont try to be clever grasshopper
+ */
+ q = dev->qdisc;
+ /* most likely result, packet went ok */
+ if (ret == NETDEV_TX_OK)
+ return qdisc_qlen(q);
+ /* only for lockless drivers .. */
+ if (ret == NETDEV_TX_LOCKED && lockless)
+ return handle_tx_locked(skb, dev, q);
+
+ if (unlikely (ret != NETDEV_TX_BUSY)) {
+ /* XXX: Do we need a ratelimit? or put a
+ * BUG_ON((int) ret != NETDEV_TX_BUSY) ?
+ **/
+ printk("BUG %s code %d qlen %d\n",dev->name, ret, q->q.qlen);
}
-out:
- BUG_ON((int) q->q.qlen < 0);
- return q->q.qlen;
+ return handle_dev_requeue(skb, dev, q);
}
void __qdisc_run(struct net_device *dev)
