Hi,
i don't want to pollute bluhm@ parallel forwarding mail on tech@ so i'm
sending this report as a separate thread. this panic it's dependent on
bluhm@ parallel diff ... and i found it yesterday
I'm having ipsec tunnel between two hosts without pf and i'm sending
traffic over that tunnel .. i'm sending 200Kpps
With plain kernel every 20 sec or so traffic stops and i need to run
ifconfig ix1 down && ifconfig ix1 up to flow again ..
When traffic stops with "vmstat -m" i'm seeing Fail mcl2k2,mcl2k
counters ...
And same behavior is with parallel diff ...
It seems that this is not big problem but i just wanted to describe what
i'm seeing ..
Panic can be triggered when i have parallel diff and sending traffic
over ipsec tunnel and on other side while traffic is flowing i'm
restarting isakmpd daemon and while negotiating ipsec doing ifconfig ix1
down && ifconfig ix1 up ... sometimes it panic all by itself.. it's not
that i know exactly what's going on ..
i couldn't trigger panic without parallel diff ..
r620-1 is box that panic and ddb output is in attachment
ipsec configuration:
r620-1# cat /etc/ipsec.conf
ike esp from 192.168.232.0/24 to 192.168.123.0/24 \
local 192.168.42.1 peer 192.168.42.111 \
main auth hmac-sha1 enc aes group modp1024 \
quick enc aes-128-gcm group modp1024 \
psk "123"
r620-1# ipsecctl -sa
FLOWS:
flow esp in from 192.168.123.0/24 to 192.168.232.0/24 peer
192.168.42.111 srcid 192.168.4
2.1/32 dstid 192.168.42.111/32 type require
flow esp out from 192.168.232.0/24 to 192.168.123.0/24 peer
192.168.42.111 srcid 192.168.
42.1/32 dstid 192.168.42.111/32 type require
SAD:
esp tunnel from 192.168.42.111 to 192.168.42.1 spi 0x9710a75f enc
aes-128-gcm
esp tunnel from 192.168.42.1 to 192.168.42.111 spi 0xa5484a26 enc
aes-128-gcm
r620-1# ifconfig ix
ix0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr ec:f4:bb:da:f7:f8
index 1 priority 0 llprio 3
media: Ethernet autoselect (10GSFP+Cu full-duplex,rxpause,txpause)
status: active
inet 192.168.42.1 netmask 0xffffff00 broadcast 192.168.42.255
inet6 fe80::eef4:bbff:feda:f7f8%ix0 prefixlen 64 scopeid 0x1
inet6 2001:db8:42::1 prefixlen 64
ix1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr ec:f4:bb:da:f7:fa
index 2 priority 0 llprio 3
media: Ethernet autoselect (10GSFP+Cu full-duplex,rxpause,txpause)
status: active
inet 192.168.232.1 netmask 0xffffff00 broadcast 192.168.232.255
x3550m4# cat /etc/ipsec.conf
ike esp from 192.168.123.0/24 to 192.168.232.0/24 \
local 192.168.42.111 peer 192.168.42.1 \
main auth hmac-sha1 enc aes group modp1024 \
quick enc aes-128-gcm group modp1024 \
psk "123"
x3550m4# ipsecctl -sa
FLOWS:
flow esp in from 192.168.232.0/24 to 192.168.123.0/24 peer 192.168.42.1
srcid 192.168.42.111/32 dstid 192.168.42.1/32 type require
flow esp out from 192.168.123.0/24 to 192.168.232.0/24 peer 192.168.42.1
srcid 192.168.42.111/32 dstid 192.168.42.1/32 type require
x3550m4# ifconfig ix
ix0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr a0:36:9f:2e:96:a0
index 5 priority 0 llprio 3
media: Ethernet autoselect (10GSFP+Cu full-duplex,rxpause,txpause)
status: active
inet 192.168.42.111 netmask 0xffffff00 broadcast 192.168.42.255
ix1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr a0:36:9f:2e:96:a1
index 6 priority 0 llprio 3
media: Ethernet autoselect (10GSFP+Cu full-duplex,rxpause,txpause)
status: active
inet 192.168.123.1 netmask 0xffffff00 broadcast 192.168.123.255
r620-1# uvm_fault(0xfffffd842f83e450, 0x147, 0, 2) -> e
kernel: page fault trap, code=0
Stopped at tdb_free+0x9c: movq %rsi,0(%rdi)
TID PID UID PRFLAGS PFLAGS CPU COMMAND
*238511 80709 68 0x10 0 4 isakmpd
391606 55892 0 0x14000 0x200 5 softnet
135192 35596 0 0x14000 0x200 2 softnet
211135 99090 0 0x14000 0x200 0 softnet
tdb_free(ffff8000012e6ad8) at tdb_free+0x9c
pfkeyv2_send(fffffd83b2ab4970,ffff8000012bc600,50) at pfkeyv2_send+0x1191
pfkeyv2_output(fffffd80a19cf200,fffffd83b2ab4970,0,0) at pfkeyv2_output+0x8a
pfkeyv2_usrreq(fffffd83b2ab4970,9,fffffd80a19cf200,0,0,ffff800023908d20) at
pfkeyv2_usrreq+0x1b0
sosend(fffffd83b2ab4970,0,ffff800023959890,0,0,0) at sosend+0x3a9
dofilewritev(ffff800023908d20,7,ffff800023959890,0,ffff800023959990) at
dofilewritev+0x14d
sys_writev(ffff800023908d20,ffff800023959930,ffff800023959990) at
sys_writev+0xe2
syscall(ffff800023959a00) at syscall+0x3b9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffaeea0, count: 6
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports. Insufficient info makes it difficult to find and fix bugs.
ddb{4}>
ddb{4}> show reg
rdi 0x147
rsi 0xffffffffffffffff
rbp 0xffff800023959360
rbx 0xffff8000012bc610
rdx 0xffffffffffffffff
rcx 0xffff8000012e6f08
rax 0xfffffd839f1fcdc8
r8 0xffffffff822894e8 w_locklistdata+0x41f98
r9 0x2
r10 0xffffffff81e23003 cmd0646_9_tim_udma+0x3cb96
r11 0x18821b02c518c33e
r12 0xfffffd83b0f0bf50
r13 0x50
r14 0xffff8000012e6ad8
r15 0xffff8000012bc640
rip 0xffffffff81c4cf4c tdb_free+0x9c
cs 0x8
rflags 0x10286 __ALIGN_SIZE+0xf286
rsp 0xffff800023959340
ss 0x10
tdb_free+0x9c: movq %rsi,0(%rdi)
ddb{4}> show locks
exclusive kernel_lock &kernel_lock r = 0 (0xffffffff82187a50)
#0 witness_lock+0x339
#1 kpageflttrap+0x178
#2 kerntrap+0x91
#3 alltraps_kern_meltdown+0x7b
#4 tdb_free+0x9c
#5 pfkeyv2_send+0x1191
#6 pfkeyv2_output+0x8a
#7 pfkeyv2_usrreq+0x1b0
#8 sosend+0x3a9
#9 dofilewritev+0x14d
#10 sys_writev+0xe2
#11 syscall+0x3b9
#12 Xsyscall+0x128
exclusive rwlock netlock r = 0 (0xffffffff8212b2d8)
#0 witness_lock+0x339
#1 rw_enter+0x281
#2 pfkeyv2_send+0xcc1
#3 pfkeyv2_output+0x8a
#4 pfkeyv2_usrreq+0x1b0
#5 sosend+0x3a9
#6 dofilewritev+0x14d
#7 sys_writev+0xe2
#8 syscall+0x3b9
#9 Xsyscall+0x128
ddb{4}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
15357 175314 1 0 3 0x100083 ttyin ksh
52237 277225 1 0 3 0x100098 poll cron
68272 345681 52177 95 3 0x100092 kqread smtpd
7357 187627 52177 103 3 0x100092 kqread smtpd
5215 435731 52177 95 3 0x100092 kqread smtpd
64452 401214 52177 95 3 0x100092 kqread smtpd
15157 418175 52177 95 3 0x100092 kqread smtpd
45 449996 52177 95 3 0x100092 kqread smtpd
52177 25822 1 0 3 0x100080 kqread smtpd
26161 516463 1 0 3 0x88 select sshd
*80709 238511 77513 68 7 0x10 isakmpd
77513 232247 1 0 3 0x80 netio isakmpd
26598 322150 1 0 3 0x100080 poll ntpd
64531 95405 43784 83 3 0x100092 poll ntpd
43784 487089 1 83 3 0x100092 poll ntpd
7138 372732 87769 73 3 0x100090 kqread syslogd
87769 385727 1 0 3 0x100082 netio syslogd
32527 266499 0 0 3 0x14200 bored smr
11127 144074 0 0 3 0x14200 pgzero zerothread
87883 40578 0 0 3 0x14200 aiodoned aiodoned
94792 208459 0 0 3 0x14200 syncer update
36451 67930 0 0 3 0x14200 cleaner cleaner
68346 96973 0 0 3 0x14200 reaper reaper
22105 204183 0 0 3 0x14200 pgdaemon pagedaemon
4199 282083 0 0 3 0x14200 bored crynlk
45296 347210 0 0 3 0x14200 netlock crypto
94244 432751 0 0 3 0x14200 usbtsk usbtask
39030 67520 0 0 3 0x14200 usbatsk usbatsk
39590 440340 0 0 3 0x40014200 acpi0 acpi0
11635 507665 0 0 3 0x40014200 idle5
86869 66399 0 0 3 0x40014200 idle4
60333 49123 0 0 7 0x40014200 idle3
35768 450103 0 0 3 0x40014200 idle2
1875 170980 0 0 7 0x40014200 idle1
15909 510253 0 0 3 0x14200 bored sensors
55892 391606 0 0 7 0x14200 softnet
5582 192858 0 0 3 0x14200 bored softnet
35596 135192 0 0 7 0x14200 softnet
99090 211135 0 0 7 0x14200 softnet
30368 141130 0 0 3 0x14200 bored systqmp
99159 484403 0 0 3 0x14200 bored systq
70666 397342 0 0 3 0x40014200 bored softclock
70067 78444 0 0 3 0x40014200 idle0
1 133505 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{4}> trace /t 0t238511
ffff800023959290(ffffffff81dd91ce,ffff800023959310,ffff800023959290,ffff8000239
59290,7dd6cddfcdb456e4,ffff8000012bc640) at 0xffff800023959290
end of kernel
end trace frame: 0x147, count: -1
ddb{4}>
ddb{4}> trace /t 0t391606
ffff800000087048(ffff800023876fe8,ffffffff81aa0769,ffff800023876f30,ffff8000238
76f08,ffff800023876f14,4) at 0xffff800000087048
acpi_pdirpa() at acpi_pdirpa+0x72eb0
end trace frame: 0x0, count: -2
ddb{4}>
ddb{4}> trace /t 0t135192
ffff800000087048(ffff80002386b778,ffffffff81aa0769,ffff80002386b6c0,ffff8000238
6b698,ffff80002386b6a4,4) at 0xffff800000087048
acpi_pdirpa() at acpi_pdirpa+0x72eb0
end trace frame: 0x0, count: -2
ddb{4}>
ddb{4}> trace /t 0t211135
ffff800000087048(ffff800023865308,ffffffff81aa0769,ffff800023865250,ffff8000238
65228,ffff800023865234,4) at 0xffff800000087048
acpi_pdirpa() at acpi_pdirpa+0x72eb0
end trace frame: 0x0, count: -2
ddb{4}>
ddb{4}> show all pool
Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
ipsec acquire 264 1 0 1 1 0 1 1 0 8 1
tdb 1096 18 0 16 2 1 1 1 0 8 0
plcache 128 66 0 0 3 0 3 3 0 8 0
rtpcb 120 16 0 16 1 1 0 1 0 8 0
rtentry 112 69 0 12 2 0 2 2 0 8 0
unpcb 120 87 0 39 2 0 2 2 0 8 0
tcpcb 736 21 0 16 1 0 1 1 0 8 0
arp 120 34 0 12 1 0 1 1 0 8 0
inpcb 304 166 0 147 2 0 2 2 0 8 0
nd6 48 3 0 0 1 0 1 1 0 8 0
ipsec policy 352 4 0 3 2 1 1 1 0 8 0
pkpcb 40 2 0 1 1 0 1 1 0 8 0
cryptop 352 3252681 0 2762355 121524 76947 44577 79965 0 8 1
art_heap8 4096 1 0 0 1 0 1 1 0 8 0
art_heap4 256 176 0 1 11 0 11 11 0 8 0
art_table 32 177 0 1 2 0 2 2 0 8 0
art_node 16 69 0 12 1 0 1 1 0 8 0
dirhash 1024 84 0 40 6 0 6 6 0 8 0
newdirblk 32 16 0 16 1 1 0 1 0 8 0
dirrem 64 1665 0 1665 28 28 0 27 0 8 0
mkdir 56 16 0 16 1 1 0 1 0 8 0
diradd 56 1668 0 1668 24 24 0 23 0 8 0
freefile 48 1633 0 1633 22 22 0 20 0 8 0
freeblks 192 1653 0 1653 83 83 0 81 0 8 0
freefrag 64 14 0 14 2 2 0 1 0 8 0
allocindir 104 10649 0 10649 274 274 0 243 0 8 0
indirdep 56 19 0 19 2 2 0 1 0 8 0
allocdir 128 2733 0 2733 78 78 0 76 0 8 0
bmsafemap 64 40 0 40 4 4 0 1 0 8 0
newblk 64 13382 0 13382 5 5 0 1 0 8 0
inodedep 160 1713 0 1711 70 68 2 70 0 8 0
pagedep 128 28 0 28 2 2 0 1 0 8 0
dino2pl 256 5128 0 1659 223 6 217 217 0 8 0
ffsino 272 5128 0 1659 240 8 232 232 0 8 0
nchpl 144 5459 0 2250 123 4 119 119 0 8 0
rtmask 32 2 0 1 2 1 1 1 0 8 0
uvmvnodes 72 5144 0 0 94 0 94 94 0 8 0
vnodes 224 5144 0 0 303 0 303 303 0 8 0
namei 1024 18060 0 18060 88 87 1 1 0 8 1
percpumem 48 32 0 0 1 0 1 1 0 8 0
aesni 576 12 0 10 1 0 1 1 1 8 0
ehcixfer 296 185 0 180 1 0 1 1 0 8 0
scxspl 216 89649 0 89649 318 317 1 2 0 8 1
plimitpl 152 26 0 13 1 0 1 1 0 8 0
sigapl 424 416 0 372 7 1 6 7 0 8 0
futexpl 56 4943 0 4943 2 2 0 1 0 8 0
knotepl 112 47 0 0 2 0 2 2 0 8 0
kqueuepl 216 8 0 0 1 0 1 1 0 8 0
pipepl 336 96 0 96 6 6 0 1 0 8 0
fdescpl 496 391 0 372 4 0 4 4 0 8 0
filepl 152 7376 0 7277 5 0 5 5 0 8 0
lockfpl 104 2 0 2 1 1 0 1 0 8 0
lockfspl 48 1 0 1 1 1 0 1 0 8 0
sessionpl 144 12 0 2 1 0 1 1 0 8 0
pgrppl 48 22 0 12 1 0 1 1 0 8 0
ucredpl 96 103 0 84 1 0 1 1 0 8 0
zombiepl 144 372 0 372 12 11 1 1 0 8 1
processpl 1080 416 0 372 5 1 4 4 0 8 0
procpl 672 447 0 403 6 1 5 6 0 8 0
sockpl 480 271 0 203 14 5 9 10 0 8 0
mcl4k 4096 1 0 0 1 0 1 1 0 8 0
mcl2k2 2112 879900 29 0 58662 2 58660 58660 0 8 0
mcl2k 2048 7 0 0 1 0 1 1 0 8 0
mtagpl 96 1189 0 0 26 1 25 26 0 8 0
mbufpl 256 879942 0 0 54998 3 54995 54995 0 8 0
bufpl 280 103015 0 64403 2759 0 2759 2759 0 8 0
anonpl 24 157802 0 151179 290 247 43 242 0 1513 0
amapchunkpl 152 11025 0 10591 114 94 20 104 0 158 0
amappl16 200 909 0 903 27 26 1 15 0 8 0
amappl15 192 160 0 147 2 1 1 1 0 8 0
amappl14 184 24 0 24 4 4 0 1 0 8 0
amappl13 176 33 0 32 1 0 1 1 0 8 0
amappl12 168 109 0 101 6 5 1 4 0 8 0
amappl11 160 128 0 100 3 1 2 2 0 8 0
amappl10 152 21 0 21 2 2 0 1 0 8 0
amappl9 144 20 0 20 3 3 0 1 0 8 0
amappl8 136 1126 0 1111 21 20 1 13 0 8 0
amappl7 128 121 0 120 1 0 1 1 0 8 0
amappl6 120 345 0 315 11 10 1 4 0 8 0
amappl5 112 234 0 217 3 2 1 3 0 8 0
amappl4 104 2246 0 2192 29 27 2 23 0 8 0
amappl3 96 678 0 660 10 9 1 7 0 8 0
amappl2 88 3072 0 2925 40 35 5 29 0 8 0
amappl1 80 10687 0 9994 22 5 17 19 0 8 0
amappl 88 4239 0 4069 22 17 5 21 0 92 0
dma8192 8192 1 0 1 1 1 0 1 0 8 0
dma4096 4096 7 0 1 1 0 1 1 0 8 0
dma2048 2048 1022 0 1022 510 509 1 1 0 8 1
dma1024 1024 1020 0 1020 510 509 1 1 0 8 1
dma512 512 520 0 519 1 0 1 1 0 8 0
dma256 256 7 0 7 1 1 0 1 0 8 0
dma128 128 17 0 17 1 1 0 1 0 8 0
dma64 64 2 0 2 1 1 0 1 0 8 0
dma32 32 9 0 9 1 1 0 1 0 8 0
dma16 16 10 0 10 3 3 0 1 0 8 0
aobjpl 64 2 0 0 1 0 1 1 0 8 0
uaddrrnd 24 391 0 372 1 0 1 1 0 8 0
uaddrbest 32 2 0 0 1 0 1 1 0 8 0
uaddr 24 391 0 372 1 0 1 1 0 8 0
vmmpekpl 168 102922 0 102899 2 0 2 2 0 8 0
vmmpepl 168 51408 0 49648 365 279 86 329 0 357 0
vmsppl 368 390 0 372 3 0 3 3 0 8 0
rwobjpl 56 18431 0 17273 92 74 18 70 0 8 0
pdppl 4096 789 0 744 119 72 47 69 0 8 2
pvpl 32 753660 0 740188 1378 1261 117 1267 0 265 0
pmappl 224 390 0 372 2 0 2 2 0 8 0
extentpl 40 179 0 114 1 0 1 1 0 8 0
phpool 112 56722 0 1202 1593 6 1587 1587 0 8 0
ddb{4}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff82109ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82187848) at __mp_lock+0xb3
ether_resolve(ffff8000000ae048,fffffd80a0908400,ffff80000013a7a0,fffffd83b428ba
f8,ffff800023864e68) at ether_resolve+0x1ad
ether_output(ffff8000000ae048,fffffd80a0908400,ffff80000013a7a0,fffffd83b428baf8)
at ether_output+0x2c
ip_output(fffffd80a0908400,0,ffff8000238650e8,1,0,0) at ip_output+0xa2e
ip_forward(fffffd80a0908400,ffff800000087048,fffffd83b428baf8,0) at
ip_forward+0x26a
ip_input_if(ffff800023865228,ffff800023865234,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd80a0908400) at ipv4_input+0x39
if_input_process(ffff800000087048,ffff8000238652a8) at if_input_process+0x6f
ifiq_process(ffff800000086c00) at ifiq_process+0x69
taskq_thread(ffff800000030000) at taskq_thread+0x9f
end trace frame: 0x0, count: 2
ddb{0}>
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff800022409ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{1}>
ddb{1}> mach ddbcpu 2
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022412ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82187848) at __mp_lock+0xaf
ether_resolve(ffff8000000ae048,fffffd809bc64900,ffff80000013a7a0,fffffd83b428baf8,ffff80002386b2d8)
at ether_resolve+0x1ad
ether_output(ffff8000000ae048,fffffd809bc64900,ffff80000013a7a0,fffffd83b428baf8)
at ether_output+0x2c
ip_output(fffffd809bc64900,0,ffff80002386b558,1,0,0) at ip_output+0xa2e
ip_forward(fffffd809bc64900,ffff800000087048,fffffd83b428baf8,0) at
ip_forward+0x26a
ip_input_if(ffff80002386b698,ffff80002386b6a4,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd809bc64900) at ipv4_input+0x39
if_input_process(ffff800000087048,ffff80002386b718) at if_input_process+0x6f
ifiq_process(ffff800000086d00) at ifiq_process+0x69
taskq_thread(ffff800000030100) at taskq_thread+0x9f
end trace frame: 0x0, count: 2
ddb{2}>
ddb{2}> mach ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002241bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff80002241bff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{3}>
ddb{3}> mach ddbcpu 4
Stopped at tdb_free+0x9c: movq %rsi,0(%rdi)
tdb_free(ffff8000012e6ad8) at tdb_free+0x9c
pfkeyv2_send(fffffd83b2ab4970,ffff8000012bc600,50) at pfkeyv2_send+0x1191
pfkeyv2_output(fffffd80a19cf200,fffffd83b2ab4970,0,0) at pfkeyv2_output+0x8a
pfkeyv2_usrreq(fffffd83b2ab4970,9,fffffd80a19cf200,0,0,ffff800023908d20) at
pfkeyv2_usrreq+0x1b0
sosend(fffffd83b2ab4970,0,ffff800023959890,0,0,0) at sosend+0x3a9
dofilewritev(ffff800023908d20,7,ffff800023959890,0,ffff800023959990) at
dofilewritev+0x14d
sys_writev(ffff800023908d20,ffff800023959930,ffff800023959990) at
sys_writev+0xe2
syscall(ffff800023959a00) at syscall+0x3b9
Xsyscall() at Xsyscall+0x128
end of kernel
end trace frame: 0x7f7ffffaeea0, count: 6
ddb{4}>
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
__mp_lock(ffffffff82187848) at __mp_lock+0xaf
ether_resolve(ffff8000000ae048,fffffd809bc63d00,ffff80000013a7a0,fffffd83b428ba
f8,ffff800023876b48) at ether_resolve+0x1ad
ether_output(ffff8000000ae048,fffffd809bc63d00,ffff80000013a7a0,fffffd83b428baf8)
at ether_output+0x2c
ip_output(fffffd809bc63d00,0,ffff800023876dc8,1,0,0) at ip_output+0xa2e
ip_forward(fffffd809bc63d00,ffff800000087048,fffffd83b428baf8,0) at
ip_forward+0x26a
ip_input_if(ffff800023876f08,ffff800023876f14,4,0,ffff800000087048) at
ip_input_if+0x365
ipv4_input(ffff800000087048,fffffd809bc63d00) at ipv4_input+0x39
if_input_process(ffff800000087048,ffff800023876f88) at if_input_process+0x6f
ifiq_process(ffff800000086b00) at ifiq_process+0x69
taskq_thread(ffff800000030300) at taskq_thread+0x9f
end trace frame: 0x0, count: 2
ddb{5}>
