let etherip(4) output directly to the ip stack

[David Gwynne]

with the previous if_ethersubr.c diff, this allows etherip(4) to output
directly to the network stack.

direct output relies on the interface using priq, since hfsc uses the
ifq machinery to work. priq implies you dont want to delay packets, so
it lets etherip push the packet straight through.

i dont think the ifq stuff is slow, but it does allow the stack to
concurrently send packets with etherip without having to create an ifq
per cpu. it does rely on per cpu counters, but theyre relatively small.

the other advantage is it stops ether_output recursing in the etherip
path. this helps profiling, but that is a fairly niche benefit so maybe
just focus on the concurrency.

ok?

Index: if_etherip.c
===================================================================
RCS file: /cvs/src/sys/net/if_etherip.c,v
retrieving revision 1.40
diff -u -p -r1.40 if_etherip.c
--- if_etherip.c        12 Nov 2018 23:57:06 -0000      1.40
+++ if_etherip.c        12 Dec 2018 01:58:57 -0000
@@ -26,6 +26,7 @@
 #include <sys/device.h>
 #include <sys/sysctl.h>
 #include <sys/tree.h>
+#include <sys/percpu.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
@@ -102,7 +103,10 @@ void etheripattach(int);
 int etherip_clone_create(struct if_clone *, int);
 int etherip_clone_destroy(struct ifnet *);
 int etherip_ioctl(struct ifnet *, u_long, caddr_t);
-void etherip_start(struct ifnet *);
+int etherip_output(struct ifnet *, struct mbuf *, struct sockaddr *,
+    struct rtentry *rt);
+void etherip_start(struct ifqueue *);
+void etherip_send(struct ifnet *, struct mbuf *);
 int etherip_media_change(struct ifnet *);
 void etherip_media_status(struct ifnet *, struct ifmediareq *);
 int etherip_set_tunnel(struct etherip_softc *, struct if_laddrreq *);
@@ -144,9 +148,10 @@ etherip_clone_create(struct if_clone *if
        ifp->if_softc = sc;
        ifp->if_hardmtu = ETHER_MAX_HARDMTU_LEN;
        ifp->if_ioctl = etherip_ioctl;
-       ifp->if_start = etherip_start;
+       ifp->if_output = etherip_output;
+       ifp->if_qstart = etherip_start;
        ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
-       ifp->if_xflags = IFXF_CLONED;
+       ifp->if_xflags = IFXF_MPSAFE | IFXF_CLONED;
        IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
        ifp->if_capabilities = IFCAP_VLAN_MTU;
        ether_fakeaddr(ifp);
@@ -159,6 +164,8 @@ etherip_clone_create(struct if_clone *if
        if_attach(ifp);
        ether_ifattach(ifp);
 
+       if_counters_alloc(ifp);
+
        NET_LOCK();
        TAILQ_INSERT_TAIL(&etherip_list, &sc->sc_tunnel, t_entry);
        NET_UNLOCK();
@@ -201,40 +208,65 @@ etherip_media_status(struct ifnet *ifp, 
 }
 
 void
-etherip_start(struct ifnet *ifp)
+etherip_send(struct ifnet *ifp, struct mbuf *m)
 {
        struct etherip_softc *sc = ifp->if_softc;
-       struct mbuf *m;
        int error;
-#if NBPFILTER > 0
-       caddr_t if_bpf;
-#endif
 
-       while ((m = ifq_dequeue(&ifp->if_snd)) != NULL) {
 #if NBPFILTER > 0
-               if_bpf = ifp->if_bpf;
-               if (if_bpf)
-                       bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_OUT);
+       caddr_t if_bpf = ifp->if_bpf;
+       if (if_bpf)
+               bpf_mtap_ether(if_bpf, m, BPF_DIRECTION_OUT);
 #endif
 
-               switch (sc->sc_tunnel.t_af) {
-               case AF_INET:
-                       error = ip_etherip_output(ifp, m);
-                       break;
+       switch (sc->sc_tunnel.t_af) {
+       case AF_INET:
+               error = ip_etherip_output(ifp, m);
+               break;
 #ifdef INET6
-               case AF_INET6:
-                       error = ip6_etherip_output(ifp, m);
-                       break;
-#endif
-               default:
-                       /* unhandled_af(sc->sc_tunnel.t_af); */
-                       m_freem(m);
-                       continue;
-               }
+       case AF_INET6:
+               error = ip6_etherip_output(ifp, m);
+               break;
+#endif
+       default:
+               /* unhandled_af(sc->sc_tunnel.t_af); */
+               m_freem(m);
+               error = ENETDOWN;
+               break;
+       }
+
+       if (error)
+               counters_inc(ifp->if_counters, ifc_oerrors);
+}
 
-               if (error)
-                       ifp->if_oerrors++;
+void
+etherip_start(struct ifqueue *ifq)
+{
+       struct ifnet *ifp = ifq->ifq_if;
+       struct mbuf *m;
+
+       while ((m = ifq_dequeue(ifq)) != NULL)
+               etherip_send(ifp, m);
+}
+
+int
+etherip_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
+    struct rtentry *rt)
+{
+       int error;
+
+       m = ether_encap(ifp, m, dst, rt, &error);
+       if (m == NULL)
+               return (error);
+
+       if (ifq_is_priq(&ifp->if_snd)) {
+               counters_pkt(ifp->if_counters,
+                   ifc_opackets, ifc_obytes, m->m_pkthdr.len); 
+               etherip_send(ifp, m);
+               return (0);
        }
+
+       return (ifq_enqueue(&ifp->if_snd, m));
 }
 
 int