On 3.12.2021. 20:35, Alexander Bluhm wrote:
> Hi,
>
> After implementing MP safe refcounting for IPsec TDB, I wonder how
> stable my diff for forwarding on multiple CPU is.
>
> Note that IPsec still has the workaround to disable multiple queues.
> We do not have a mutex that protects the TDB fields yet. We have
> only fixed the memory management.
>
> My goal is to get real MP pressure on the lower part of the IP
> network stack. Only this will show remaining bugs.
Hi,
with only plain forwarding it seems that everything works just fine, but
with ipsec i'm getting this panic. i will leave ddb console active if
something else will be needed
r620-2# uvm_fault(0xffffffff8229d4e0, 0x137, 0, 2) -> e
kernel: page fault trap, code=0
Stopped at ipsp_spd_lookup+0xa2f: movq %rax,0(%rcx)
TID PID UID PRFLAGS PFLAGS CPU COMMAND
419237 67407 0 0x14000 0x200 0 softnet
*157694 94649 0 0x14000 0x200 2K softnet
ipsp_spd_lookup(fffffd80a4139800,2,14,2,0,0,5b815d966b14b44b,fffffd80a4139800)
at ipsp_spd_lookup+0xa2f
ip_output_ipsec_lookup(fffffd80a4139800,14,0,ffff800022c60228,0) at
ip_output_ipsec_lookup+0x4c
ip_output(fffffd80a4139800,0,ffff800022c603e8,1,0,0,3ada3367ffb43fe1) at
ip_output+0x39d
ip_forward(fffffd80a4139800,ffff800000087048,fffffd8394511078,0) at
ip_forward+0x26a
ip_input_if(ffff800022c60528,ffff800022c60534,4,0,ffff800000087048) at
ip_input_if+0x353
ipv4_input(ffff800000087048,fffffd80a4139800) at ipv4_input+0x39
ether_input(ffff800000087048,fffffd80a4139800) at ether_input+0x3aa
if_input_process(ffff800000087048,ffff800022c60618) at if_input_process+0x92
ifiq_process(ffff800000087458) at ifiq_process+0x69
taskq_thread(ffff80000002f080) at taskq_thread+0x81
end trace frame: 0x0, count: 5
https://www.openbsd.org/ddb.html describes the minimum info required in
bug reports. Insufficient info makes it difficult to find and fix bugs.
ddb{2}>
ddb{2}> ps
PID TID PPID UID S FLAGS WAIT COMMAND
84832 15199 94097 68 3 0x10 netlock isakmpd
94097 209788 1 0 3 0x80 netio isakmpd
15214 351483 1 0 3 0x100083 ttyin ksh
12905 119865 1 0 3 0x100098 poll cron
84388 106290 61660 95 3 0x100092 kqread smtpd
49054 464016 61660 103 3 0x100092 kqread smtpd
24623 388600 61660 95 3 0x100092 kqread smtpd
25783 71877 61660 95 3 0x100092 kqread smtpd
44044 451515 61660 95 3 0x100092 kqread smtpd
74629 163500 61660 95 3 0x100092 kqread smtpd
61660 91566 1 0 3 0x100080 kqread smtpd
44941 436617 1 0 3 0x88 poll sshd
63116 236112 1 0 3 0x100080 poll ntpd
67685 455461 13478 83 3 0x100092 poll ntpd
13478 271069 1 83 3 0x100092 poll ntpd
4556 497201 93102 73 3 0x100090 kqread syslogd
93102 715 1 0 3 0x100082 netio syslogd
75807 17005 0 0 3 0x14200 bored smr
19891 296875 0 0 3 0x14200 pgzero zerothread
85805 493633 0 0 3 0x14200 aiodoned aiodoned
30502 291595 0 0 3 0x14200 syncer update
70346 429811 0 0 3 0x14200 cleaner cleaner
2149 206582 0 0 3 0x14200 reaper reaper
34585 457270 0 0 3 0x14200 pgdaemon pagedaemon
68729 397903 0 0 3 0x14200 usbtsk usbtask
65328 269730 0 0 3 0x14200 usbatsk usbatsk
69367 498053 0 0 3 0x40014200 acpi0 acpi0
5156 405582 0 0 7 0x40014200 idle5
53211 338561 0 0 7 0x40014200 idle4
19386 131626 0 0 7 0x40014200 idle3
22779 421169 0 0 3 0x40014200 idle2
27190 363359 0 0 7 0x40014200 idle1
1080 80711 0 0 3 0x14200 bored sensors
67407 419237 0 0 7 0x14200 softnet
34617 10042 0 0 3 0x14200 netlock softnet
50133 404397 0 0 3 0x14200 netlock softnet
*94649 157694 0 0 7 0x14200 softnet
51030 188067 0 0 3 0x14200 bored systqmp
46228 294212 0 0 3 0x14200 bored systq
62154 451682 0 0 2 0x40014200 softclock
34808 227380 0 0 3 0x40014200 idle0
1 24458 0 0 3 0x82 wait init
0 0 -1 0 3 0x10200 scheduler swapper
ddb{2}> trace /t 0t157694
ffff800000087048(1,ffffffff8122bc39,ffff800022c60550,ffff800022c60528,ffff80002
2c60534,4) at 0xffff800000087048
__kernel_phys_end() at 0x22c605c0
end trace frame: 0x0, count: -2
ddb{2}>
ddb{2}> mach ddbcpu 0
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffffffff8218eff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
pool_cache_get(ffffffff8229ee58) at pool_cache_get+0x19
pool_get(ffffffff8229ee58,2) at pool_get+0x5e
m_clget(0,2,802) at m_clget+0x1b4
ixgbe_get_buf(ffff80000008a000,6e) at ixgbe_get_buf+0xa3
ixgbe_rxfill(ffff80000008a000) at ixgbe_rxfill+0xae
ixgbe_queue_intr(ffff800000085400) at ixgbe_queue_intr+0x4f
intr_handler(ffff800022c71920,ffff800000078480) at intr_handler+0x6e
Xintr_ioapic_edge18_untramp() at Xintr_ioapic_edge18_untramp+0x18f
_kernel_lock() at _kernel_lock+0xa6
ether_resolve(ffff8000000a0048,fffffd80a3bd3100,ffff800022c71b08,fffffd83945114
68,ffff800022c71a88) at ether_resolve+0x1a8
ether_output(ffff8000000a0048,fffffd80a3bd3100,ffff800022c71b08,fffffd839451146
8) at ether_output+0x2c
end trace frame: 0xffff800022c71be0, count: 0
ddb{0}>
ddb{0}> mach ddbcpu 1
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff800022409ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{1}>
ddb{1}> mach ddbcpu 3
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002241bff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff80002241bff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{3}>
ddb{3}> mach ddbcpu 4
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff800022424ff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{4}>
ddb{4}> mach ddbcpu 5
Stopped at x86_ipi_db+0x12: leave
x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12
x86_ipi_handler() at x86_ipi_handler+0x80
Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23
acpicpu_idle() at acpicpu_idle+0x281
sched_idle(ffff80002242dff0) at sched_idle+0x27e
end trace frame: 0x0, count: 10
ddb{5}>
ddb{2}> show tdb
0xffffffff81cfe23f: 000041bd (unknown address family)->(unknown address
family)
:15 #-1958084609 74f68445
ddb{2}> show tdb /f 0xffffffff81cfe23f
tdb at 0xffffffff81cfe23f
hnext: 0x130808b49018948
dnext: 0x138888b490000
snext: 0x3880c74901894800
inext: 0x49ffffffff000001
onext: 0xffff0000013080c7
xform: 0x8b4cffa5bd2be800
refcnt: -1958084609
encalgxform: 0x11080c7498845
authalgxform: 0x80c7490000000000
compalgxform: 0xf8
flags:
74f68445<UNIQUE,BYTES,DELETED,SOFT_FIRSTUSE,UDPENCAP,PFSYNC
,ESN>
seq: -34635
exp_allocations: -2133708289
soft_allocations: 1207998333
cur_allocations: 949306639
exp_bytes: -53887557573804033
soft_bytes: -7572652700634382337
cur_bytes: -8986685076615987194
exp_timeout: 1274901163972733
soft_timeout: -1044682726943537152
established: -23376416473088
first_use: 1207830954720511
soft_first_use: -2045619644365824
exp_first_use: -956873529326089985
last_used: -528463706791608320
last_marked: -5151950549254864897
cryptoid: 434571561984018687
tdb_spi: 000041bd
amxkeylen: 13
emxkeylen: 0
ivlen: 49285
sproto: 15
wnd: 133
satype: 76
updates: 248
dst: (unknown address family)
src: (unknown address family)
amxkey: 0x110b88b49ffffff
emxkey: 0x4cffa5bae2e80000
rpl: 76702787085813131
ids: 0xf2b8809845
ids_swapped: -578482432
mtu: 1224736758
mtutimeout: 264675222111627
udpencap_port: 0
tag: 4073
tap: 1006632963
rdomain: 1012364293
rdomain_post: -1048244478
