Hello Alexander,
I've applied your patch but crashed again. Here it is:
ddb{1}> show panic
*cpu1: kernel diagnostic assertion "refcnt_read(&rt->rt_refcnt) >= 2" failed: f
ile "/usr/src/sys/net/rtable.c", line 828
ddb{1}> trace
db_enter() at db_enter+0x10
panic(ffffffff82063c23) at panic+0xbf
__assert(ffffffff820e125c,ffffffff820f6e3e,33c,ffffffff8208222f) at __assert+0x
25
rtable_mpath_reprio(0,ffff800001b97340,18,30,fffffd81db326718) at rtable_mpath_
reprio+0x251
rt_if_linkstate_change(fffffd81db326718,ffff800001a39000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800007fb7800,ffff800022564cc0,ffff800022564c18,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8064e9d100,fffffd81e04ee3c8) at route_output+0x3bc
route_send(fffffd81e04ee3c8,fffffd8064e9d100,0,0) at route_send+0x57
sosend(fffffd81e04ee3c8,0,ffff800022564f50,0,0,80) at sosend+0x37f
dofilewritev(ffff8000225465d0,6,ffff800022564f50,0,ffff800022565050) at dofilew
ritev+0x14d
sys_writev(ffff8000225465d0,ffff800022564ff0,ffff800022565050) at sys_writev+0x
d2
syscall(ffff8000225650c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x79ef723b9510, count: -13
ddb{1}> machine ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff82467ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
msleep_nsec(fffffd821836c010,fffffd821836c010,318,ffffffff820d31b6,22888020) at
msleep_nsec+0xcf
kqueue_sleep(fffffd821836c010,ffff800022893968) at kqueue_sleep+0xbe
kqueue_scan(ffff800022893868,8,ffff800022893760,ffff800022893968,ffff8000226b1b
d8,ffff8000228939bc) at kqueue_scan+0x108
sys_kevent(ffff8000226b1bd8,ffff800022893a20,ffff800022893a80) at sys_kevent+0x
371
syscall(ffff800022893af0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x23718fe20, count: 3
ddb{0}> machine ddbcpu 1
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82063c23) at panic+0xbf
__assert(ffffffff820e125c,ffffffff820f6e3e,33c,ffffffff8208222f) at __assert+0x
25
rtable_mpath_reprio(0,ffff800001b97340,18,30,fffffd81db326718) at rtable_mpath_
reprio+0x251
rt_if_linkstate_change(fffffd81db326718,ffff800001a39000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800007fb7800,ffff800022564cc0,ffff800022564c18,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8064e9d100,fffffd81e04ee3c8) at route_output+0x3bc
route_send(fffffd81e04ee3c8,fffffd8064e9d100,0,0) at route_send+0x57
sosend(fffffd81e04ee3c8,0,ffff800022564f50,0,0,80) at sosend+0x37f
dofilewritev(ffff8000225465d0,6,ffff800022564f50,0,ffff800022565050) at dofilew
ritev+0x14d
sys_writev(ffff8000225465d0,ffff800022564ff0,ffff800022565050) at sys_writev+0x
d2
syscall(ffff8000225650c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x79ef723b9510, count: 2
ddb{1}> machine ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
vn_write(fffffd8211484450,ffff8000226d9310,0) at vn_write+0x3b
dofilewritev(ffff800022484b38,12,ffff8000226d9310,0,ffff8000226d9410) at dofile
writev+0x14d
sys_writev(ffff800022484b38,ffff8000226d93b0,ffff8000226d9410) at sys_writev+0x
d2
syscall(ffff8000226d9480) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x78a408e10cd0, count: 6
ddb{2}> machine ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(fffffd81c0443cf8) at mtx_enter+0x2c
kqueue_scan(ffff8000224efbf8,8,ffff8000224efaf0,ffff8000224efcf8,ffff800022524b
58,ffff8000224efd4c) at kqueue_scan+0xb8
sys_kevent(ffff800022524b58,ffff8000224efdb0,ffff8000224efe10) at sys_kevent+0x
371
syscall(ffff8000224efe80) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x70b4dc0afb20, count: 7
ddb{3}> machine ddbcpu 4
Invalid cpu 4
ddb{3}> machine ddbcpu 0t0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff82467ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
softintr_dispatch(0) at softintr_dispatch+0x49
Xsoftclock() at Xsoftclock+0x1f
msleep_nsec(fffffd821836c010,fffffd821836c010,318,ffffffff820d31b6,22888020) at
msleep_nsec+0xcf
kqueue_sleep(fffffd821836c010,ffff800022893968) at kqueue_sleep+0xbe
kqueue_scan(ffff800022893868,8,ffff800022893760,ffff800022893968,ffff8000226b1b
d8,ffff8000228939bc) at kqueue_scan+0x108
sys_kevent(ffff8000226b1bd8,ffff800022893a20,ffff800022893a80) at sys_kevent+0x
371
syscall(ffff800022893af0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x23718fe20, count: 3
ddb{0}> machine ddbcpu 0t1
Stopped at db_enter+0x10: popq %rbp
db_enter() at db_enter+0x10
panic(ffffffff82063c23) at panic+0xbf
__assert(ffffffff820e125c,ffffffff820f6e3e,33c,ffffffff8208222f) at __assert+0x
25
rtable_mpath_reprio(0,ffff800001b97340,18,30,fffffd81db326718) at rtable_mpath_
reprio+0x251
rt_if_linkstate_change(fffffd81db326718,ffff800001a39000,0) at rt_if_linkstate_
change+0xcd
rtm_output(ffff800007fb7800,ffff800022564cc0,ffff800022564c18,30,0) at rtm_outp
ut+0x71e
route_output(fffffd8064e9d100,fffffd81e04ee3c8) at route_output+0x3bc
route_send(fffffd81e04ee3c8,fffffd8064e9d100,0,0) at route_send+0x57
sosend(fffffd81e04ee3c8,0,ffff800022564f50,0,0,80) at sosend+0x37f
dofilewritev(ffff8000225465d0,6,ffff800022564f50,0,ffff800022565050) at dofilew
ritev+0x14d
sys_writev(ffff8000225465d0,ffff800022564ff0,ffff800022565050) at sys_writev+0x
d2
syscall(ffff8000225650c0) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x79ef723b9510, count: 2
ddb{1}> machine ddbcpu 0t2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218d1ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
_kernel_lock() at _kernel_lock+0xb2
vn_write(fffffd8211484450,ffff8000226d9310,0) at vn_write+0x3b
dofilewritev(ffff800022484b38,12,ffff8000226d9310,0,ffff8000226d9410) at dofile
writev+0x14d
sys_writev(ffff800022484b38,ffff8000226d93b0,ffff8000226d9410) at sys_writev+0x
d2
syscall(ffff8000226d9480) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x78a408e10cd0, count: 6
ddb{2}> machine ddbcpu 0t3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff8000218daff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
mtx_enter(fffffd81c0443cf8) at mtx_enter+0x2c
kqueue_scan(ffff8000224efbf8,8,ffff8000224efaf0,ffff8000224efcf8,ffff800022524b
58,ffff8000224efd4c) at kqueue_scan+0xb8
sys_kevent(ffff800022524b58,ffff8000224efdb0,ffff8000224efe10) at sys_kevent+0x
371
syscall(ffff8000224efe80) at syscall+0x3d4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x70b4dc0afb20, count: 7
ddb{3}>
825 rt->rt_priority = prio;
826 } else {
827 // rtref(rt); /* keep rt alive in between remove and insert */
828 KASSERT(refcnt_read(&rt->rt_refcnt) >= 2);
829 SRPL_REMOVE_LOCKED(&rt_rc, &an->an_rtlist,
830 rt, rtentry, rt_next);
831 rt->rt_priority = prio;
832 rtable_mpath_insert(an, rt);
833 // rtfree(rt);
834 error = EAGAIN;
835 }
________________________________
From: Alexander Bluhm <alexander.bl...@gmx.net>
Sent: Thursday, July 6, 2023 13:54
To: Valdrin MUJA <valdrin_m...@outlook.com>
Cc: bugs@openbsd.org <bugs@openbsd.org>
Subject: Re: kernel diagnostic assertion "!_kernel_lock_held()" failed
On Wed, Jul 05, 2023 at 12:17:15PM +0000, Valdrin MUJA wrote:
> ddb{3}> show panic
> *cpu3: kernel diagnostic assertion "!ISSET(rt->rt_flags, RTF_UP)" failed:
> file "
> /usr/src/sys/net/route.c", line 496
>
> ddb{3}> trace
> db_enter() at db_enter+0x10
> panic(ffffffff82067518) at panic+0xbf
> __assert(ffffffff820de23b,ffffffff8206be5d,1f0,ffffffff820e901b) at
> __assert+0x
> 25
> rtfree(fffffd8275365a90) at rtfree+0x1af
> route_output(fffffd8065dd1f00,fffffd821540a920) at route_output+0x413
> route_send(fffffd821540a920,fffffd8065dd1f00,0,0) at route_send+0x57
> sosend(fffffd821540a920,0,ffff80002254d3e0,0,0,80) at sosend+0x37f
> dofilewritev(ffff80002251f390,6,ffff80002254d3e0,0,ffff80002254d4e0) at
> dofilew
> ritev+0x14d
> sys_writev(ffff80002251f390,ffff80002254d480,ffff80002254d4e0) at
> sys_writev+0x
> d2
> syscall(ffff80002254d550) at syscall+0x3d4
> Xsyscall() at Xsyscall+0x128
Looks like your routing table is busted. I just found a bug in
-current. Maybe this also causes your problem.
Could you apply the diff below an recompile the kernel. It should
be the same fix for 7.3.
bluhm
Index: net/rtable.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/net/rtable.c,v
retrieving revision 1.82
diff -u -p -r1.82 rtable.c
--- net/rtable.c 19 Apr 2023 17:42:47 -0000 1.82
+++ net/rtable.c 5 Jul 2023 20:05:26 -0000
@@ -604,6 +604,11 @@ rtable_insert(unsigned int rtableid, str
SRPL_INSERT_HEAD_LOCKED(&rt_rc, &an->an_rtlist, rt, rt_next);
prev = art_insert(ar, an, addr, plen);
+ if (prev == an) {
+ rw_exit_write(&ar->ar_lock);
+ /* keep the refcount for rt while it is in an_rtlist */
+ return (0);
+ }
if (prev != an) {
SRPL_REMOVE_LOCKED(&rt_rc, &an->an_rtlist, rt, rtentry,
rt_next);
@@ -689,9 +694,10 @@ rtable_delete(unsigned int rtableid, str
npaths++;
if (npaths > 1) {
- KASSERT(refcnt_read(&rt->rt_refcnt) >= 1);
+ KASSERT(refcnt_read(&rt->rt_refcnt) >= 2);
SRPL_REMOVE_LOCKED(&rt_rc, &an->an_rtlist, rt, rtentry,
rt_next);
+ rtfree(rt);
mrt = SRPL_FIRST_LOCKED(&an->an_rtlist);
if (npaths == 2)
@@ -703,8 +709,9 @@ rtable_delete(unsigned int rtableid, str
if (art_delete(ar, an, addr, plen) == NULL)
panic("art_delete failed to find node %p", an);
- KASSERT(refcnt_read(&rt->rt_refcnt) >= 1);
+ KASSERT(refcnt_read(&rt->rt_refcnt) >= 2);
SRPL_REMOVE_LOCKED(&rt_rc, &an->an_rtlist, rt, rtentry, rt_next);
+ rtfree(rt);
art_put(an);
leave:
@@ -821,12 +828,11 @@ rtable_mpath_reprio(unsigned int rtablei
*/
rt->rt_priority = prio;
} else {
- rtref(rt); /* keep rt alive in between remove and insert */
+ KASSERT(refcnt_read(&rt->rt_refcnt) >= 2);
SRPL_REMOVE_LOCKED(&rt_rc, &an->an_rtlist,
rt, rtentry, rt_next);
rt->rt_priority = prio;
rtable_mpath_insert(an, rt);
- rtfree(rt);
error = EAGAIN;
}
rw_exit_write(&ar->ar_lock);
@@ -839,6 +845,9 @@ rtable_mpath_insert(struct art_node *an,
{
struct rtentry *mrt, *prt = NULL;
uint8_t prio = rt->rt_priority;
+
+ /* increment the refcount for rt while it is in an_rtlist */
+ rtref(rt);
if ((mrt = SRPL_FIRST_LOCKED(&an->an_rtlist)) == NULL) {
SRPL_INSERT_HEAD_LOCKED(&rt_rc, &an->an_rtlist, rt, rt_next);
