https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234026
Bug ID: 234026 Summary: [panic] [dummynet] Repeatable panic in dummynet due to locking issues and use-after-free Product: Base System Version: 11.2-STABLE Hardware: Any OS: Any Status: New Keywords: crash Severity: Affects Some People Priority: --- Component: kern Assignee: n...@freebsd.org Reporter: eu...@freebsd.org Hi! I run multiple routers using FreeBSD 11.2-STABLE/amd64 r336962, ipfw+dummynet and net/mpd5 daemon that dynamically creates/destroys ngXXX interfaces for multiple PPPoE clients. If an interface ngXXX is destroyed while dummynet pipe/queue keeps mbuf with m_pkthdr.rcvif pointing to freed struct ifnet, kernel panices when taskqueue runs dummynet_task/dummynet_send/netisr_dispatch_src/ip_input sequence and I have crashdump. kgdb session follows: Script started on Sat Dec 15 06:47:49 2018 Command: kgdb kernel.debug /home/nanobsd/pppoe/crash/vmcore.0 GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd"... Unread portion of the kernel message buffer: stack pointer = 0x28:0xfffffe01244bb920 frame pointer = 0x28:0xfffffe01244bb9a0 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 0 (dummynet) trap number = 12 panic: page fault cpuid = 0 KDB: stack backtrace: db_trace_self_wrapper() at 0xffffffff802fc89b = db_trace_self_wrapper+0x2b/frame 0xfffffe01244bb5d0 vpanic() at 0xffffffff804f0ac7 = vpanic+0x177/frame 0xfffffe01244bb630 panic() at 0xffffffff804f0943 = panic+0x43/frame 0xfffffe01244bb690 trap_fatal() at 0xffffffff8076f2af = trap_fatal+0x35f/frame 0xfffffe01244bb6e0 trap_pfault() at 0xffffffff8076f309 = trap_pfault+0x49/frame 0xfffffe01244bb740 trap() at 0xffffffff8076eae4 = trap+0x2d4/frame 0xfffffe01244bb850 calltrap() at 0xffffffff8074ff3c = calltrap+0x8/frame 0xfffffe01244bb850 --- trap 0xc, rip = 0xffffffff804ec893, rsp = 0xfffffe01244bb920, rbp = 0xfffffe01244bb9a0 --- __rw_rlock_hard() at 0xffffffff804ec893 = __rw_rlock_hard+0xf3/frame 0xfffffe01244bb9a0 ip_input() at 0xffffffff806444ca = ip_input+0x53a/frame 0xfffffe01244bba30 netisr_dispatch_src() at 0xffffffff8060ebe8 = netisr_dispatch_src+0xa8/frame 0xfffffe01244bba80 dummynet_send() at 0xffffffff806723dd = dummynet_send+0x10d/frame 0xfffffe01244bbab0 dummynet_task() at 0xffffffff80671e1c = dummynet_task+0x2ec/frame 0xfffffe01244bbb20 taskqueue_run_locked() at 0xffffffff80548a54 = taskqueue_run_locked+0x154/frame 0xfffffe01244bbb80 taskqueue_thread_loop() at 0xffffffff80549bb8 = taskqueue_thread_loop+0x98/frame 0xfffffe01244bbbb0 fork_exit() at 0xffffffff804ba803 = fork_exit+0x83/frame 0xfffffe01244bbbf0 fork_trampoline() at 0xffffffff80750eee = fork_trampoline+0xe/frame 0xfffffe01244bbbf0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Uptime: 57d17h28m40s Dumping 467 out of 4073 MB:..4%..11%..21%..31%..42%..52%..62%..72%..83%..93% Reading symbols from /boot/modules/tmpfs.ko...done. Loaded symbols for /boot/modules/tmpfs.ko #0 doadump (textdump=1) at pcpu.h:230 230 __asm("movq %%gs:%1,%0" : "=r" (td) (kgdb) bt #0 doadump (textdump=1) at pcpu.h:230 #1 0xffffffff804f06c0 in kern_reboot (howto=260) at /home/src/sys/kern/kern_shutdown.c:383 #2 0xffffffff804f0b01 in vpanic (fmt=<value optimized out>, ap=<value optimized out>) at /home/src/sys/kern/kern_shutdown.c:776 #3 0xffffffff804f0943 in panic (fmt=<value optimized out>) at /home/src/sys/kern/kern_shutdown.c:707 #4 0xffffffff8076f2af in trap_fatal (frame=0xfffffe01244bb860, eva=274877908504) at /home/src/sys/amd64/amd64/trap.c:877 #5 0xffffffff8076f309 in trap_pfault (frame=0xfffffe01244bb860, usermode=0) at pcpu.h:230 #6 0xffffffff8076eae4 in trap (frame=0xfffffe01244bb860) at /home/src/sys/amd64/amd64/trap.c:415 #7 0xffffffff8074ff3c in calltrap () at /home/src/sys/amd64/amd64/exception.S:231 #8 0xffffffff804ec893 in __rw_rlock_hard (rw=0xfffff80092e78190, td=0xfffff80001d02620, v=<value optimized out>) at /home/src/sys/kern/kern_rwlock.c:493 #9 0xffffffff806444ca in ip_input (m=<value optimized out>) at /home/src/sys/netinet/ip_input.c:795 #10 0xffffffff8060ebe8 in netisr_dispatch_src (proto=1, source=<value optimized out>, m=<value optimized out>) at /home/src/sys/net/netisr.c:1120 #11 0xffffffff806723dd in dummynet_send (m=0x0) at /home/src/sys/netpfil/ipfw/ip_dn_io.c:774 #12 0xffffffff80671e1c in dummynet_task (context=<value optimized out>, pending=<value optimized out>) at /home/src/sys/netpfil/ipfw/ip_dn_io.c:729 #13 0xffffffff80548a54 in taskqueue_run_locked (queue=0xfffff80006085e00) at /home/src/sys/kern/subr_taskqueue.c:463 #14 0xffffffff80549bb8 in taskqueue_thread_loop (arg=<value optimized out>) at /home/src/sys/kern/subr_taskqueue.c:755 #15 0xffffffff804ba803 in fork_exit (callout=0xffffffff80549b20 <taskqueue_thread_loop>, arg=0xffffffff80c82c38, frame=0xfffffe01244bbc00) at /home/src/sys/kern/kern_fork.c:1072 #16 0xffffffff80750eee in fork_trampoline () at /home/src/sys/amd64/amd64/exception.S:972 ---Type <return> to continue, or q <return> to quit--- #17 0x0000000000000000 in ?? () Current language: auto; currently minimal (kgdb) frame 9 #9 0xffffffff806444ca in ip_input (m=<value optimized out>) at /home/src/sys/netinet/ip_input.c:795 795 IF_ADDR_RLOCK(ifp); (kgdb) l 790 * interface. Reception of forwarded directed broadcasts would 791 * be handled via ip_forward() and ether_output() with the loopback 792 * into the stack for SIMPLEX interfaces handled by ether_output(). 793 */ 794 if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { 795 IF_ADDR_RLOCK(ifp); 796 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 797 if (ifa->ifa_addr->sa_family != AF_INET) 798 continue; 799 ia = ifatoia(ifa); (kgdb) p *ifp $1 = {if_link = {tqe_next = 0x4000000004, tqe_prev = 0x4000000006}, if_clones = { le_next = 0x4000000007, le_prev = 0x4000000009}, if_groups = {tqh_first = 0x400000000a, tqh_last = 0x4000000011}, if_alloctype = 250 'З', if_softc = 0x4000000104, if_llsoftc = 0x40000004d0, if_l2com = 0x40000004d4, if_dname = 0x4000000184 <Address 0x4000000184 out of bounds>, if_dunit = 218, if_index = 64, if_index_reserved = 0, if_xname = 0xfffff80092e78060 "\220\001", if_description = 0x400000035e <Address 0x400000035e out of bounds>, if_flags = 1050, if_drv_flags = 64, if_capabilities = 454, if_capenable = 64, if_linkmib = 0x4000000386, if_linkmiblen = 274877907462, if_refcount = 682, if_type = 64 '@', if_addrlen = 0 '\0', if_hdrlen = 0 '\0', if_link_state = 0 '\0', if_mtu = 522, if_metric = 64, if_baudrate = 274877907476, if_hwassist = 274877907488, if_epoch = 274877907500, if_lastchange = {tv_sec = 274877908294, tv_usec = 274877907730}, if_snd = { ifq_head = 0x40000002e0, ifq_tail = 0x4000000334, ifq_len = 824, ifq_maxlen = 64, ifq_mtx = { lock_object = {lo_name = 0x40000003c6 <Address 0x40000003c6 out of bounds>, lo_flags = 1298, lo_data = 64, lo_witness = 0x4000000332}, mtx_lock = 274877907950}, ifq_drv_head = 0x40000002ae, ifq_drv_tail = 0x40000000fc, ifq_drv_len = 858, ifq_drv_maxlen = 64, altq_type = 870, altq_flags = 64, altq_disc = 0x400000036a, altq_ifp = 0x4000000124, altq_enqueue = 0x4000000318, altq_dequeue = 0x400000030a, altq_request = 0x400000036c, altq_clfier = 0x4000000188, altq_classify = 0x400000058d, altq_tbr = 0x400000058f, altq_cdnr = 0x4000000376}, if_linktask = {ta_link = { stqe_next = 0x4000000262}, ta_pending = 460, ta_priority = 0, ta_func = 0x4000000264, ta_context = 0x40000001b6}, if_addr_lock = {lock_object = { lo_name = 0x40000001b8 <Address 0x40000001b8 out of bounds>, lo_flags = 1072, lo_data = 64, lo_witness = 0x400000026a}, rw_lock = 274877907356}, if_addrhead = { tqh_first = 0x4000000382, tqh_last = 0x4000000196}, if_multiaddrs = { tqh_first = 0x4000000120, tqh_last = 0x4000000218}, if_amcount = 294, if_addr = 0x40000001be, if_broadcastaddr = 0x4000000064 <Address 0x4000000064 out of bounds>, if_afdata_lock = { ---Type <return> to continue, or q <return> to quit--- lock_object = {lo_name = 0x4000000192 <Address 0x4000000192 out of bounds>, lo_flags = 810, lo_data = 64, lo_witness = 0x40000002de}, rw_lock = 274877907684}, if_afdata = 0xfffff80092e78208, if_afdata_initialized = 441, if_fib = 64, if_vnet = 0x40000000db, if_home_vnet = 0x4000000411, if_vlantrunk = 0x40000001bf, if_bpf = 0x40000001c1, if_pcount = 1051, if_bridge = 0x40000001c7, if_lagg = 0x40000003ef, if_pf_kif = 0x4000000207, if_carp = 0x400000020b, if_label = 0x40000002ab, if_netmap = 0x4000000215, if_output = 0x4000000219, if_input = 0x40000002af, if_start = 0x4000000221, if_ioctl = 0x400000022d, if_init = 0x40000002e1, if_resolvemulti = 0x40000002e5, if_qflush = 0x4000000305, if_transmit = 0x4000000263, if_reassign = 0x4000000265, if_get_counter = 0x400000030b, if_requestencap = 0x400000026b, if_counters = 0xfffff80092e78410, if_hw_tsomax = 999, if_hw_tsomaxsegcount = 64, if_hw_tsomaxsegsize = 735, if_pspare = 0xfffff80092e78480, if_hw_addr = 0x4000000039, if_pcp = 101 'e', if_bspare = 0xfffff80092e784a1 "", if_ispare = 0xfffff80092e784a4} (kgdb) frame 11 #11 0xffffffff806723dd in dummynet_send (m=0x0) at /home/src/sys/netpfil/ipfw/ip_dn_io.c:774 774 netisr_dispatch(NETISR_IP, m); (kgdb) p m $2 = (struct mbuf *) 0x0 (kgdb) l 769 case DIR_OUT: 770 ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL); 771 break ; 772 773 case DIR_IN : 774 netisr_dispatch(NETISR_IP, m); 775 break; 776 777 #ifdef INET6 778 case DIR_IN | PROTO_IPV6: (kgdb) quit -- You are receiving this mail because: You are the assignee for the bug. _______________________________________________ freebsd-net@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/freebsd-net To unsubscribe, send any mail to "freebsd-net-unsubscr...@freebsd.org"