On 7.7.2021. 12:46, Hrvoje Popovski wrote:
> Panic can be triggered when i have parallel diff and sending traffic
> over ipsec tunnel and on other side while traffic is flowing i'm
> restarting isakmpd daemon and while negotiating ipsec doing ifconfig ix1
> down && ifconfig ix1 up ... sometimes it panic all by itself.. it's not
> that i know exactly what's going on ..
Different panic on same setup ...
uvm_fault(0xffffffff8221de20, 0x8, 0, 1) -> e
kernel: page fault trap, code=0
Stopped at pool_get+0x6e: movq 0x8(%rax),%r11
TID PID UID PRFLAGS PFLAGS CPU COMMAND
288284 51828 0 0x3 0 0 ifconfig
329904 31282 0 0x14000 0x200 4 softnet
157417 71744 0 0x14000 0x200 2 softnet
*152664 84791 0 0x14000 0x200 3K softnet
58812 4615 0 0x14000 0x200 1 softnet
pool_get(ffffffff82153918,a) at pool_get+0x6e
ipsp_acquire_sa(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,ffff80002386b068,ffffff84)
at ipsp_acquire_sa+0x124
ipsp_spd_lookup(fffffd80b842db00,2,14,ffff80002386b24c,2,0) at
ipsp_spd_lookup+0xb4e
ip_output_ipsec_lookup(fffffd80b842db00,14,ffff80002386b24c,0,0) at
ip_output_ipsec_lookup+0x4d
ip_output(fffffd80b842db00,0,ffff80002386b3d8,1,0,0) at ip_output+0x42a
ip_forward(fffffd80b842db00,ffff800000087048,fffffd83b3fecb68,0) at
ip_forward+0x26a
ip_input_if(ffff80002386b518,ffff80002386b524,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd80b842db00) at ipv4_input+0x39
if_input_process(ffff800000087048,ffff80002386b598) at if_input_process+0x6f
ifiq_process(ffff800000086d00) at ifiq_process+0x69
taskq_thread(ffff800000030100) at taskq_thread+0x9f
end trace frame: 0x0, count: 4
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{3}> show reg
rdi 0xffffffff82153918 ipsec_acquire_pool
rsi 0xa
rbp 0xffff80002386af70
rbx 0
rdx 0xfffffd842ae97650
rcx 0x1
rax 0
r8 0xffffff84
r9 0xffff80000016a604
r10 0xffff8000012c7d40
r11 0x4f581b7438ceb9e9
r12 0xfffffd842ae9766c
r13 0xfffffd842ae97580
r14 0xffffffff82153918 ipsec_acquire_pool
r15 0xa
rip 0xffffffff8133e97e pool_get+0x6e
cs 0x8
rflags 0x10246 __ALIGN_SIZE+0xf246
rsp 0xffff80002386aed0
ss 0
pool_get+0x6e: movq 0x8(%rax),%r11
ddb{3}>
ddb{3}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
51828 288284 1009 0 7 0x3 ifconfig
1009 461325 1 0 3 0x10008b sigsusp ksh
56847 290195 1 0 3 0x100098 poll cron
83452 510449 26068 95 3 0x100092 kqread smtpd
41700 201644 26068 103 3 0x100092 kqread smtpd
26028 130559 26068 95 3 0x100092 kqread smtpd
11949 518693 26068 95 3 0x100092 kqread smtpd
4520 211282 26068 95 3 0x100092 kqread smtpd
82047 205282 26068 95 3 0x100092 kqread smtpd
26068 82309 1 0 3 0x100080 kqread smtpd
46745 316858 1 0 3 0x88 select sshd
5198 371736 71443 68 3 0x90 select isakmpd
71443 13600 1 0 3 0x80 netio isakmpd
71184 265525 1 0 3 0x100080 poll ntpd
26181 322162 3615 83 3 0x100092 poll ntpd
3615 143908 1 83 3 0x100092 poll ntpd
82091 209375 43348 73 3 0x100090 kqread syslogd
43348 363890 1 0 3 0x100082 netio syslogd
16351 58282 0 0 3 0x14200 bored smr
27233 372715 0 0 3 0x14200 pgzero zerothread
68321 430498 0 0 3 0x14200 aiodoned aiodoned
10629 99914 0 0 3 0x14200 syncer update
74747 455079 0 0 3 0x14200 cleaner cleaner
67178 305272 0 0 3 0x14200 reaper reaper
54113 244787 0 0 3 0x14200 pgdaemon pagedaemon
88516 52910 0 0 3 0x14200 bored crynlk
84865 523006 0 0 3 0x14200 bored crypto
87886 398410 0 0 3 0x14200 usbtsk usbtask
79482 516951 0 0 3 0x14200 usbatsk usbatsk
7554 475208 0 0 3 0x40014200 acpi0 acpi0
55853 38718 0 0 7 0x40014200 idle5
13965 462123 0 0 3 0x40014200 idle4
14774 279411 0 0 3 0x40014200 idle3
25764 508246 0 0 3 0x40014200 idle2
14959 77683 0 0 3 0x40014200 idle1
95869 101965 0 0 3 0x14200 bored sensors
31282 329904 0 0 7 0x14200 softnet
71744 157417 0 0 7 0x14200 softnet
*84791 152664 0 0 7 0x14200 softnet
4615 58812 0 0 7 0x14200 softnet
45524 379795 0 0 3 0x14200 bored systqmp
11914 16163 0 0 3 0x14200 bored systq
48612 387270 0 0 3 0x40014200 netlock softclock
64200 21926 0 0 3 0x40014200 idle0
1 236011 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{3}>
ddb{3}> trace /t 0t288284
__kernel_virt_to_phys() at __kernel_virt_to_phys+0x1ddc045
end of kernel
end trace frame: 0x1d717518000, count: -1
ddb{3}>
ddb{3}> trace /t 0t329904
fffffd80b842d800(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638
) at 0xfffffd80b842d800
Bad frame pointer: 0xffff800000087000
end trace frame: 0xffff800000087000, count: -1
ddb{3}> trace /t 0t157417
fffffd80bab65800(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638
) at 0xfffffd80bab65800
Bad frame pointer: 0xffff800000087000
end trace frame: 0xffff800000087000, count: -1
ddb{3}> trace /t 0t152664
fffffd80b842db00(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638
) at 0xfffffd80b842db00
Bad frame pointer: 0xffff800000087000
end trace frame: 0xffff800000087000, count: -1
ddb{3}>
ddb{3}> trace /t 0t58812
fffffd80bb6bd000(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638
) at 0xfffffd80bb6bd000
Bad frame pointer: 0xffff800000087000
end trace frame: 0xffff800000087000, count: -1
ddb{3}>
ddb{3}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff82110ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82230fb0) at __mp_lock+0xb3
softintr_dispatch(0) at softintr_dispatch+0x4a
Xsoftclock() at Xsoftclock+0x1f
__mp_lock(ffffffff82230fb0) at __mp_lock+0xb3
syscall(ffff800023915380) at syscall+0x2a4
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffe98e0, count: 6
ddb{0}>
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82230fb0) at __mp_lock+0xa7
esp_output(fffffd80bb6bd000,ffff8000012ef360,0,14,9) at esp_output+0x8c3
ipsp_process_packet(fffffd80bb6bd000,ffff8000012ef360,2,0) at
ipsp_process_packet+0x466
ip_output_ipsec_send(ffff8000012ef360,fffffd80bb6bd000,ffff800023865548,1) at
ip_output_ipsec_send+0x171
ip_output(fffffd80bb6bd000,0,ffff800023865548,1,0,0) at ip_output+0x8cd
ip_forward(fffffd80bb6bd000,ffff800000087048,fffffd83b3fecb68,0) at
ip_forward+0x26a
ip_input_if(ffff800023865688,ffff800023865694,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd80bb6bd000) at ipv4_input+0x39
if_input_process(ffff800000087048,ffff800023865708) at if_input_process+0x6f
ifiq_process(ffff800000086c00) at ifiq_process+0x69
taskq_thread(ffff800000030000) at taskq_thread+0x9f
end trace frame: 0x0, count: 1
ddb{1}>
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022412ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82230fb0) at __mp_lock+0xac
selwakeup(fffffd83b0d0d4e0) at selwakeup+0x11
sorwakeup(fffffd83b0d0d3c8) at sorwakeup+0x77
pfkeyv2_sendmessage(ffff800023870bb0,2,0,2,0,0) at pfkeyv2_sendmessage+0x2e1
pfkeyv2_acquire(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,fffffd8386f2
fe8c,ffff800023870db8) at pfkeyv2_acquire+0x54f
ipsp_spd_lookup(fffffd80bab65800,2,14,ffff800023870f9c,2,0) at
ipsp_spd_lookup+0xb4e
ip_output_ipsec_lookup(fffffd80bab65800,14,ffff800023870f9c,0,0) at
ip_output_ipsec_lookup+0x4d
ip_output(fffffd80bab65800,0,ffff800023871128,1,0,0) at ip_output+0x42a
ip_forward(fffffd80bab65800,ffff800000087048,fffffd83b3fecb68,0) at
ip_forward+0x26a
ip_input_if(ffff800023871268,ffff800023871274,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd80bab65800) at ipv4_input+0x39
end trace frame: 0xffff8000238712d0, count: 0
ddb{2}>
ddb{2}> mach ddbcpu 3
Stopped at pool_get+0x6e: movq 0x8(%rax),%r11
pool_get(ffffffff82153918,a) at pool_get+0x6e
ipsp_acquire_sa(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,ffff80002386b068,ffffff84)
at ipsp_acquire_sa+0x124
ipsp_spd_lookup(fffffd80b842db00,2,14,ffff80002386b24c,2,0) at
ipsp_spd_lookup+0xb4e
ip_output_ipsec_lookup(fffffd80b842db00,14,ffff80002386b24c,0,0) at
ip_output_ipsec_lookup+0x4d
ip_output(fffffd80b842db00,0,ffff80002386b3d8,1,0,0) at ip_output+0x42a
ip_forward(fffffd80b842db00,ffff800000087048,fffffd83b3fecb68,0) at
ip_forward+0x26a
ip_input_if(ffff80002386b518,ffff80002386b524,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd80b842db00) at ipv4_input+0x39
if_input_process(ffff800000087048,ffff80002386b598) at if_input_process+0x6f
ifiq_process(ffff800000086d00) at ifiq_process+0x69
taskq_thread(ffff800000030100) at taskq_thread+0x9f
end trace frame: 0x0, count: 4
ddb{3}>
ddb{3}> mach ddbcpu 4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82230fb0) at __mp_lock+0xb3
selwakeup(fffffd83b0d0d4e0) at selwakeup+0x11
sorwakeup(fffffd83b0d0d3c8) at sorwakeup+0x77
pfkeyv2_sendmessage(ffff800023876d20,2,0,2,0,0) at pfkeyv2_sendmessage+0x2e1
pfkeyv2_acquire(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,fffffd8386f2
fd84,ffff800023876f28) at pfkeyv2_acquire+0x54f
ipsp_spd_lookup(fffffd80b842d800,2,14,ffff80002387710c,2,0) at
ipsp_spd_lookup+0xb4e
ip_output_ipsec_lookup(fffffd80b842d800,14,ffff80002387710c,0,0) at
ip_output_ipsec_lookup+0x4d
ip_output(fffffd80b842d800,0,ffff800023877298,1,0,0) at ip_output+0x42a
ip_forward(fffffd80b842d800,ffff800000087048,fffffd83b3fecb68,0) at
ip_forward+0x26a
ip_input_if(ffff8000238773d8,ffff8000238773e4,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd80b842d800) at ipv4_input+0x39
end trace frame: 0xffff800023877440, count: 0
ddb{4}>
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff80002242dff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{5}>
ddb{3}> show all pool
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
ipsec acquire 264 2 0 0 2 0 2 2 0 8 1
tdb 1096 9 0 7 1 0 1 1 0 8 0
plcache 128 66 0 0 3 0 3 3 0 8 0
rtpcb 120 16 0 16 1 1 0 1 0 8 0
rtentry 112 54 0 12 2 0 2 2 0 8 0
unpcb 120 65 0 17 2 0 2 2 0 8 0
tcpcb 736 9 0 4 1 0 1 1 0 8 0
arp 120 19 0 12 1 0 1 1 0 8 0
inpcb 304 160 0 140 2 0 2 2 0 8 0
nd6 48 3 0 0 1 0 1 1 0 8 0
ipsec policy 352 4 0 2 1 0 1 1 0 8 0
pkpcb 40 2 0 1 1 0 1 1 0 8 0
cryptop 352 1146235 0 1146234 79970 48548 31422 79970 0 8 31421
art_heap8 4096 1 0 0 1 0 1 1 0 8 0
art_heap4 256 175 0 1 11 0 11 11 0 8 0
art_table 32 176 0 1 2 0 2 2 0 8 0
art_node 16 54 0 12 1 0 1 1 0 8 0
dirhash 1024 84 0 40 6 0 6 6 0 8 0
newdirblk 32 16 0 16 1 1 0 1 0 8 0
dirrem 64 1643 0 1643 26 26 0 26 0 8 0
mkdir 56 16 0 16 1 1 0 1 0 8 0
diradd 56 1652 0 1652 23 23 0 23 0 8 0
freefile 48 1624 0 1624 22 22 0 20 0 8 0
freeblks 192 1645 0 1645 82 82 0 81 0 8 0
freefrag 64 16 0 16 2 2 0 1 0 8 0
allocindir 104 10863 0 10863 239 239 0 208 0 8 0
indirdep 56 19 0 19 1 1 0 1 0 8 0
allocdir 128 2741 0 2741 76 76 0 76 0 8 0
bmsafemap 64 34 0 34 1 1 0 1 0 8 0
newblk 64 13604 0 13604 4 4 0 1 0 8 0
inodedep 160 1688 0 1688 70 70 0 70 0 8 0
pagedep 128 27 0 27 1 1 0 1 0 8 0
dino2pl 256 5101 0 1650 224 7 217 217 0 8 0
ffsino 272 5101 0 1650 238 7 231 231 0 8 0
nchpl 144 5411 0 2209 124 5 119 119 0 8 0
rtmask 32 2 0 1 1 0 1 1 0 8 0
uvmvnodes 72 5118 0 0 94 0 94 94 0 8 0
vnodes 224 5118 0 0 302 0 302 302 0 8 0
namei 1024 17535 0 17535 12 11 1 1 0 8 1
percpumem 48 32 0 0 1 0 1 1 0 8 0
aesni 576 6 0 4 1 0 1 1 1 8 0
ehcixfer 296 184 0 179 1 0 1 1 0 8 0
scxspl 216 89562 0 89562 16 15 1 2 0 8 1
plimitpl 152 25 0 12 1 0 1 1 0 8 0
sigapl 424 407 0 362 7 1 6 7 0 8 0
futexpl 56 4705 0 4705 2 2 0 1 0 8 0
knotepl 112 48 0 0 2 0 2 2 0 8 0
kqueuepl 216 8 0 0 1 0 1 1 0 8 0
pipepl 336 94 0 94 4 4 0 1 0 8 0
fdescpl 496 382 0 362 4 0 4 4 0 8 0
filepl 152 7094 0 6994 5 0 5 5 0 8 0
lockfpl 104 4 0 4 1 1 0 1 0 8 0
lockfspl 48 2 0 2 1 1 0 1 0 8 0
sessionpl 144 11 0 1 1 0 1 1 0 8 0
pgrppl 48 31 0 20 1 0 1 1 0 8 0
ucredpl 96 75 0 56 1 0 1 1 0 8 0
zombiepl 144 363 0 363 9 8 1 1 0 8 1
processpl 1080 407 0 362 5 1 4 4 0 8 0
procpl 672 438 0 393 6 1 5 6 0 8 0
sockpl 480 243 0 174 10 1 9 10 0 8 0
mcl4k 4096 1 0 0 1 0 1 1 0 8 0
mcl2k2 2112 879900 39 0 58661 1 58660 58660 0 8 0
mcl2k 2048 5 0 0 1 0 1 1 0 8 0
mtagpl 96 1736 0 0 43 0 43 43 0 8 0
mbufpl 256 879932 0 0 54997 2 54995 54995 0 8 0
bufpl 280 103327 0 64461 2778 1 2777 2777 0 8 0
anonpl 24 152833 0 146253 272 225 47 241 0 1513 0
amapchunkpl 152 10595 0 10161 112 92 20 104 0 158 0
amappl16 200 909 0 902 27 26 1 15 0 8 0
amappl15 192 152 0 140 2 1 1 1 0 8 0
amappl14 184 26 0 26 5 5 0 1 0 8 0
amappl13 176 27 0 26 1 0 1 1 0 8 0
amappl12 168 77 0 77 6 6 0 3 0 8 0
amappl11 160 120 0 86 3 1 2 2 0 8 0
amappl10 152 22 0 22 2 2 0 1 0 8 0
amappl9 144 20 0 20 5 5 0 1 0 8 0
amappl8 136 1114 0 1100 21 20 1 13 0 8 0
amappl7 128 119 0 116 2 1 1 2 0 8 0
amappl6 120 340 0 310 8 7 1 4 0 8 0
amappl5 112 217 0 199 3 2 1 3 0 8 0
amappl4 104 2223 0 2170 29 26 3 23 0 8 0
amappl3 96 689 0 663 9 8 1 7 0 8 0
amappl2 88 2994 0 2843 41 36 5 29 0 8 0
amappl1 80 10164 0 9465 21 4 17 19 0 8 0
amappl 88 4095 0 3925 22 16 6 21 0 92 0
dma8192 8192 1 0 1 1 1 0 1 0 8 0
dma4096 4096 7 0 1 1 0 1 1 0 8 0
dma2048 2048 76 0 76 37 36 1 1 0 8 1
dma1024 1024 74 0 74 37 36 1 1 0 8 1
dma512 512 47 0 46 1 0 1 1 0 8 0
dma256 256 7 0 7 1 1 0 1 0 8 0
dma128 128 17 0 17 1 1 0 1 0 8 0
dma64 64 2 0 2 1 1 0 1 0 8 0
dma32 32 9 0 9 1 1 0 1 0 8 0
dma16 16 10 0 10 3 3 0 1 0 8 0
aobjpl 64 2 0 0 1 0 1 1 0 8 0
uaddrrnd 24 382 0 362 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 382 0 362 1 0 1 1 0 8 0
vmmpekpl 168 98569 0 98546 2 0 2 2 0 8 0
vmmpepl 168 49045 0 47276 350 259 91 328 0 357 0
vmsppl 368 381 0 362 3 0 3 3 0 8 0
rwobjpl 56 17758 0 16585 84 63 21 70 0 8 0
pdppl 4096 771 0 724 92 45 47 69 0 8 0
pvpl 32 737117 0 723805 1309 1161 148 1267 0 265 23
pmappl 224 381 0 362 2 0 2 2 0 8 0
extentpl 40 179 0 114 1 0 1 1 0 8 0
phpool 112 55669 0 150 1589 2 1587 1587 0 8 0
ddb{3}>
