Is the large (greater than 2^32) value of pr_nget below
exceptional? My crashes only happen for long-running
heavy workloads so a big value seems plausible but
maybe there is some limit I'm supposed to reconfigure
for such workloads?
panic: pmap_enter: failed to allocate pted
Stopped at panic+0x134: ori r0,r0,0x0
TID PID UID PRFLAGS PFLAGS CPU COMMAND
234577 13111 8889 0x18000001 0 3 go
491287 13050 8889 0xa001007 0 2 asm
331460 14455 8889 0x1a000003 0 5 asm
66317 80047 8889 0x1a000003 0 1 compile
331134 57997 8889 0x1a000003 0x4000000 0 go
240615 91938 8889 0x1a000003 0x4000000 6 go
*247303 89969 8889 0x1a000003 0x4000000 4K go
28756 33586 0 0x14000 0x200 7 pagedaemon
panic+0x134
pmap_enter+0x218
uvm_km_kmemalloc_pla+0x1f4
uvm_uarea_alloc+0x70
thread_fork+0xd8
sys___tfork+0xc4
syscall+0x530
trap+0x5dc
trapagain+0x4
--- syscall (number 8) ---
End of kernel: 0x457c99db0 lr 0x4bddd2604
ddb{4}>show struct pool pmap_pted_pool
struct pmap_pted_pool at 0x1089a98 (424 bytes) {pr_lock = {prl_mtx = {mtx_owner
= (void *)0x0, mtx_wantipl = 7, mtx_oldipl = 0}, prl_rwlock = {rwl_owner = 0, r
wl_name = (const unsigned char *)0x700000000}}, pr_lock_ops = (const pool_lock_
ops *)0xfc4950, pr_poollist = {sqe_next = (struct pool *)0x10898f0}, pr_emptypa
ges = {tqh_first = (struct pool_page_header *)0x0, tqh_last = 0x1089ab8}, pr_fu
llpages = {tqh_first = (struct pool_page_header *)0xc00000003e3b2f90, tqh_last =
0xc00000014b344f90}, pr_partpages = {tqh_first = (struct pool_page_header *)0x
c000000140250f90, tqh_last = 0xc000000146821f90}, pr_curpage = (struct pool_pag
e_header *)0x0, pr_size = 56, pr_minitems = 20, pr_minpages = 1, pr_maxpages = 8
, pr_npages = 4632, pr_itemsperpage = 71, pr_slack = 0, pr_nitems = 328872, pr_
nout = 328870, pr_hardlimit = 4294967295, pr_serial = 14, pr_pgsize = 4096, pr_
pgmask = 18446744073709547520, pr_alloc = (struct pool_allocator *)0xfe9b40, pr
_wchan = (const unsigned char *)0xeb08e4, pr_flags = 0, pr_ipl = 7, pr_phtree =
{rbh_root = {rbt_root = (struct rb_entry *)0x0}}, pr_cache = (struct cpumem *)
0x0, pr_cache_magic = [0,0], pr_cache_lock = {prl_mtx = {mtx_owner = (void *)0x
0, mtx_wantipl = 0, mtx_oldipl = 0}, prl_rwlock = {rwl_owner = 0, rwl_name = (c
onst unsigned char *)0x0}}, pr_cache_lists = {tqh_first = (struct pool_cache_it
em *)0x0, tqh_last = 0x0}, pr_cache_nitems = 0, pr_cache_items = 0, pr_cache_co
ntention = 0, pr_cache_contention_prev = 0, pr_cache_timestamp = 0, pr_cache_ng
c = 0, pr_cache_nout = 0, pr_align = 8, pr_maxcolors = 2, pr_phoffset = 3984, p
r_hardlimit_warning = (const unsigned char *)0x0, pr_hardlimit_ratecap = {tv_se
c = 0, tv_usec = 0}, pr_hardlimit_warning_last = {tv_sec = 0, tv_usec = 0}, pr_
requests_lock = {prl_mtx = {mtx_owner = (void *)0x0, mtx_wantipl = 7, mtx_oldip
l = 0}, prl_rwlock = {rwl_owner = 0, rwl_name = (const unsigned char *)0x700000
000}}, pr_requests = {tqh_first = (struct pool_request *)0x0, tqh_last = 0x1089
be8}, pr_requesting = 0, pr_nget = 5523479116, pr_nfail = 13, pr_nput = 5523150
246, pr_npagealloc = 1387529, pr_npagefree = 1382897, pr_hiwat = 10072, pr_nidl
e = 0, pr_crange = (const kmem_pa_mode *)0xfc60b0}
ddb{4}>show struct pool pmap_vp_pool
struct pmap_vp_pool at 0x10898f0 (424 bytes) {pr_lock = {prl_mtx = {mtx_owner =
(void *)0x0, mtx_wantipl = 7, mtx_oldipl = 7}, prl_rwlock = {rwl_owner = 0, rw
l_name = (const unsigned char *)0x700000007}}, pr_lock_ops = (const pool_lock_o
ps *)0xfc4950, pr_poollist = {sqe_next = (struct pool *)0x1089c40}, pr_emptypag
es = {tqh_first = (struct pool_page_header *)0xc0000001449f7af0, tqh_last = 0xc
000000142a861d8}, pr_fullpages = {tqh_first = (struct pool_page_header *)0xc000
00003e3bbd20, tqh_last = 0xc00000014d3f9248}, pr_partpages = {tqh_first = (stru
ct pool_page_header *)0xc00000003c3e9250, tqh_last = 0xc00000014a4e4d28}, pr_cu
rpage = (struct pool_page_header *)0xc00000014a4e4d28, pr_size = 2048, pr_minit
ems = 10, pr_minpages = 5, pr_maxpages = 8, pr_npages = 17150, pr_itemsperpage =
2, pr_slack = 0, pr_nitems = 34300, pr_nout = 34255, pr_hardlimit = 4294967295
, pr_serial = 13, pr_pgsize = 4096, pr_pgmask = 18446744073709547520, pr_alloc =
(struct pool_allocator *)0xfe9b40, pr_wchan = (const unsigned char *)0xee10ef,
pr_flags = 0, pr_ipl = 7, pr_phtree = {rbh_root = {rbt_root = (struct rb_entry
*)0xc00000013f618a08}}, pr_cache = (struct cpumem *)0x0, pr_cache_magic = [0,0
], pr_cache_lock = {prl_mtx = {mtx_owner = (void *)0x0, mtx_wantipl = 0, mtx_ol
dipl = 0}, prl_rwlock = {rwl_owner = 0, rwl_name = (const unsigned char *)0x0}}
, pr_cache_lists = {tqh_first = (struct pool_cache_item *)0x0, tqh_last = 0x0},
pr_cache_nitems = 0, pr_cache_items = 0, pr_cache_contention = 0, pr_cache_con
tention_prev = 0, pr_cache_timestamp = 0, pr_cache_ngc = 0, pr_cache_nout = 0, p
r_align = 8, pr_maxcolors = 1, pr_phoffset = 0, pr_hardlimit_warning = (const u
nsigned char *)0x0, pr_hardlimit_ratecap = {tv_sec = 0, tv_usec = 0}, pr_hardli
mit_warning_last = {tv_sec = 0, tv_usec = 0}, pr_requests_lock = {prl_mtx = {mt
x_owner = (void *)0x0, mtx_wantipl = 7, mtx_oldipl = 0}, prl_rwlock = {rwl_owne
r = 0, rwl_name = (const unsigned char *)0x700000000}}, pr_requests = {tqh_firs
t = (struct pool_request *)0x0, tqh_last = 0x1089a40}, pr_requesting = 0, pr_ng
et = 147054103, pr_nfail = 0, pr_nput = 147019848, pr_npagealloc = 6817420, pr_
npagefree = 6800270, pr_hiwat = 20556, pr_nidle = 17, pr_crange = (const kmem_p
a_mode *)0xfc60b0}
ddb{4}> show struct pool pmap_pmap_pool
struct pmap_pmap_pool at 0x1089c40 (424 bytes) {pr_lock = {prl_mtx = {mtx_owner
= (void *)0x0, mtx_wantipl = 7, mtx_oldipl = 0}, prl_rwlock = {rwl_owner = 0, r
wl_name = (const unsigned char *)0x700000000}}, pr_lock_ops = (const pool_lock_
ops *)0xfc4950, pr_poollist = {sqe_next = (struct pool *)0x104b5c0}, pr_emptypa
ges = {tqh_first = (struct pool_page_header *)0xc00000014ef78f90, tqh_last = 0x
c00000014ef78f90}, pr_fullpages = {tqh_first = (struct pool_page_header *)0x0, t
qh_last = 0x1089c70}, pr_partpages = {tqh_first = (struct pool_page_header *)0x
c00000003e3aef90, tqh_last = 0xc000000149c6ef90}, pr_curpage = (struct pool_pag
e_header *)0xc00000003e3aef90, pr_size = 48, pr_minitems = 2, pr_minpages = 1, p
r_maxpages = 8, pr_npages = 3, pr_itemsperpage = 83, pr_slack = 0, pr_nitems = 2
49, pr_nout = 138, pr_hardlimit = 4294967295, pr_serial = 12, pr_pgsize = 4096,
pr_pgmask = 18446744073709547520, pr_alloc = (struct pool_allocator *)0xfe9b40
, pr_wchan = (const unsigned char *)0xe91044, pr_flags = 0, pr_ipl = 7, pr_phtr
ee = {rbh_root = {rbt_root = (struct rb_entry *)0x0}}, pr_cache = (struct cpume
m *)0x0, pr_cache_magic = [0,0], pr_cache_lock = {prl_mtx = {mtx_owner = (void *
)0x0, mtx_wantipl = 0, mtx_oldipl = 0}, prl_rwlock = {rwl_owner = 0, rwl_name =
(const unsigned char *)0x0}}, pr_cache_lists = {tqh_first = (struct pool_cache
_item *)0x0, tqh_last = 0x0}, pr_cache_nitems = 0, pr_cache_items = 0, pr_cache
_contention = 0, pr_cache_contention_prev = 0, pr_cache_timestamp = 0, pr_cache
_ngc = 0, pr_cache_nout = 0, pr_align = 8, pr_maxcolors = 1, pr_phoffset = 3984
, pr_hardlimit_warning = (const unsigned char *)0x0, pr_hardlimit_ratecap = {tv
_sec = 0, tv_usec = 0}, pr_hardlimit_warning_last = {tv_sec = 0, tv_usec = 0}, p
r_requests_lock = {prl_mtx = {mtx_owner = (void *)0x0, mtx_wantipl = 7, mtx_old
ipl = 0}, prl_rwlock = {rwl_owner = 0, rwl_name = (const unsigned char *)0x7000
00000}}, pr_requests = {tqh_first = (struct pool_request *)0x0, tqh_last = 0x10
89d90}, pr_requesting = 0, pr_nget = 1113457, pr_nfail = 0, pr_nput = 1113319, p
r_npagealloc = 3, pr_npagefree = 0, pr_hiwat = 3, pr_nidle = 1, pr_crange = (co
nst kmem_pa_mode *)0xfc60b0}
ddb{4}>
On Wed, Jun 5, 2024 at 12:16 PM Miod Vallat <m...@online.fr> wrote:
>
> > There's a corruption...
> >
> > > ddb{7}> show panic
> > > cpu6: kernel diagnostic assertion "((flags & PGO_LOCKED) != 0 &&
> > > rw_lock_held(
> > > uobj->vmobjlock)) || (flags & PGO_LOCKED) == 0" failed: file
> > > "/sys/uvm/uvm_vnod
> > > e.c", line 953
> > >
> > > *cpu7: assertwaitok: non-zero mutex count: 1
> > > ddb{7}> trace
> > > panic+0x134
> > > assertwaitok+0xf8
> > > mi_switch+0x5c
> > > sleep_finish+0x160
> > > rw_enter+0x1cc
> > > vm_map_lock_read_ln+0x38
> > > uvmfault_lookup+0x114
> > > uvm_fault_check+0x68
> > > uvm_fault+0x12c
> > > trap+0x7a4
> > > trapagain+0x4
> > > --- trap (type 0x300) ---
> > > phtree_RBT_COMPARE+0x28
> > > pool_do_put+0x94
> > > pool_put+0x94
> > ^^^^
> > ...inside this pool. Which of the 3 is it? Can someone with a ppc64
> > figure out?
>
> It's pmap_vp_pool.
