Re: 5.4-RELEASE lockups on amd64 SMP

Matthew Grooms Wed, 08 Jun 2005 14:18:54 -0700

BTW : Had you tested pfsync between two SMP systems with decent trafficflow? It usually took about 3 days to hit the LOR but the panic shows upin about 10-20 minutes.


Matthew Grooms
Network Engineer
Seton Healthcare Network
http://www.seton.net/
[EMAIL PROTECTED]
(512) 324 9913


Max Laier wrote:

Matthew,

can you try the attached diff. Available for 5 and CURRENT. I recall thatthis problem was seen before, strange that I didn't see the problem. Soundsfamiliar to you? Please try the patch and let me know if that helps. Thanksa lot.


On Wednesday 08 June 2005 01:35, Matthew Grooms wrote:

Once again, here are the backtraces for the panic and lor ...

Tracing id 110 tid 100089 td 0xffffff012f3f0c80
kdb_enter() at kdb_enter+0x2f
panic() at panic+0x249
uma_dbg_free() at uma_dbg_free+0x188
uma_zfree_arg() at uma_zfree_arg+0x1b0
pf_purge_expired_states() at pf_purge_expired_states+0x41
pfsync_input at pfsync_input+xb35
pf_input() at ip_input+0x10f
netisr_processqueue() at netisr_processqueue+0x17
swi_net() at swi_net+0xa8
ithread_loop() at ithread_loop+0xd9
fork_exit() at fork_exit+0xc3
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffffffb44f9d00, rbp = 0 ---
db> continue
boot() called on cpu#0
Uptime: 13h42m43s
Dumping 4864 MB
 16 32 ...

lock order reversal

...

alltraps_with_regs_pushed() at alltraps_with_regs_pushed+0x5
pf_state_tree_lan_ext_RB_REMOVE() at pf_state_tree_lan_ext_RB_REMOVE+0x10c



This LOR is a consequence of the fault, so it can be disregarded.



------------------------------------------------------------------------

Index: if_pfsync.c
===================================================================
RCS file: /usr/store/mlaier/fcvs/src/sys/contrib/pf/net/if_pfsync.c,v
retrieving revision 1.15
diff -u -r1.15 if_pfsync.c
--- if_pfsync.c 3 May 2005 16:43:32 -0000       1.15
+++ if_pfsync.c 8 Jun 2005 14:04:44 -0000
@@ -132,6 +132,7 @@

static void pfsync_clone_destroy(struct ifnet *);

 static int     pfsync_clone_create(struct if_clone *, int);
+static void    pfsync_senddef(void *);
 #else
 void   pfsyncattach(int);
 #endif
@@ -174,6 +175,8 @@
        callout_stop(&sc->sc_bulk_tmo);
        callout_stop(&sc->sc_bulkfail_tmo);

+ callout_stop(&sc->sc_send_tmo);

+
 #if NBPFILTER > 0
         bpfdetach(ifp);
 #endif
@@ -220,6 +223,7 @@
        callout_init(&sc->sc_tmo, 0);
        callout_init(&sc->sc_bulk_tmo, 0);
        callout_init(&sc->sc_bulkfail_tmo, 0);
+       callout_init(&sc->sc_send_tmo, 0);
        if_attach(ifp);

LIST_INSERT_HEAD(&pfsync_list, sc, sc_next);

@@ -1033,6 +1037,7 @@
                if (pfsyncr.pfsyncr_maxupdates > 255)
                        return (EINVAL);
 #ifdef __FreeBSD__
+               callout_drain(&sc->sc_send_tmo);
                PF_LOCK();
 #endif
                sc->sc_maxupdates = pfsyncr.pfsyncr_maxupdates;
@@ -1789,15 +1794,14 @@
 #endif

pfsyncstats.pfsyncs_opackets++;

-
 #ifdef __FreeBSD__
-               PF_UNLOCK();
-#endif
+               if (IF_HANDOFF(&sc->sc_ifq, m, NULL))
+                       pfsyncstats.pfsyncs_oerrors++;
+               else
+                       callout_reset(&sc->sc_send_tmo, 1, pfsync_senddef, sc);
+#else
                if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL))
                        pfsyncstats.pfsyncs_oerrors++;
-
-#ifdef __FreeBSD__
-               PF_LOCK();
 #endif
        } else
                m_freem(m);
@@ -1807,6 +1811,22 @@

#ifdef __FreeBSD__

+static void
+pfsync_senddef(void *arg)
+{
+       struct pfsync_softc *sc = (struct pfsync_softc *)arg;
+       struct mbuf *m;
+
+       for(;;) {
+               IF_DEQUEUE(&sc->sc_ifq, m);
+               if (m == NULL)
+                       break;
+               if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL))
+                       pfsyncstats.pfsyncs_oerrors++;
+       }
+}
+
+
 static int
 pfsync_modevent(module_t mod, int type, void *data)
 {
Index: if_pfsync.h
===================================================================
RCS file: /usr/store/mlaier/fcvs/src/sys/contrib/pf/net/if_pfsync.h,v
retrieving revision 1.5
diff -u -r1.5 if_pfsync.h
--- if_pfsync.h 3 May 2005 16:43:32 -0000       1.5
+++ if_pfsync.h 8 Jun 2005 14:06:03 -0000
@@ -164,6 +164,10 @@
        struct in_addr           sc_sendaddr;
        struct mbuf             *sc_mbuf;       /* current cumulative mbuf */
        struct mbuf             *sc_mbuf_net;   /* current cumulative mbuf */
+#ifdef __FreeBSD__
+       struct ifqueue           sc_ifq;
+       struct callout           sc_send_tmo;
+#endif
        union sc_statep          sc_statep;
        union sc_statep          sc_statep_net;
        u_int32_t                sc_ureq_received;


------------------------------------------------------------------------

Index: if_pfsync.c
===================================================================
RCS file: /usr/store/mlaier/fcvs/src/sys/contrib/pf/net/if_pfsync.c,v
retrieving revision 1.11.2.2
diff -u -r1.11.2.2 if_pfsync.c
--- if_pfsync.c 19 May 2005 10:59:22 -0000      1.11.2.2
+++ if_pfsync.c 8 Jun 2005 14:07:17 -0000
@@ -130,6 +130,7 @@