Module Name: src
Committed By: ad
Date: Sat Sep 23 18:20:20 UTC 2023
Modified Files:
src/sys/uvm: uvm_page.c uvm_physseg.c
Log Message:
uvm_phys_to_vm_page() turns out to be a fairly central routine due to the
way that some of the pmaps work, so try to optimise it a little.
To generate a diff of this commit:
cvs rdiff -u -r1.253 -r1.254 src/sys/uvm/uvm_page.c
cvs rdiff -u -r1.18 -r1.19 src/sys/uvm/uvm_physseg.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/uvm/uvm_page.c
diff -u src/sys/uvm/uvm_page.c:1.253 src/sys/uvm/uvm_page.c:1.254
--- src/sys/uvm/uvm_page.c:1.253 Mon Jul 17 12:55:37 2023
+++ src/sys/uvm/uvm_page.c Sat Sep 23 18:20:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_page.c,v 1.253 2023/07/17 12:55:37 riastradh Exp $ */
+/* $NetBSD: uvm_page.c,v 1.254 2023/09/23 18:20:20 ad Exp $ */
/*-
* Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
@@ -95,7 +95,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.253 2023/07/17 12:55:37 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.254 2023/09/23 18:20:20 ad Exp $");
#include "opt_ddb.h"
#include "opt_uvm.h"
@@ -671,23 +671,6 @@ uvm_page_physget(paddr_t *paddrp)
}
#endif /* PMAP_STEAL_MEMORY */
-/*
- * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages
- * back from an I/O mapping (ugh!). used in some MD code as well.
- */
-struct vm_page *
-uvm_phys_to_vm_page(paddr_t pa)
-{
- paddr_t pf = atop(pa);
- paddr_t off;
- uvm_physseg_t upm;
-
- upm = uvm_physseg_find(pf, &off);
- if (upm != UVM_PHYSSEG_TYPE_INVALID)
- return uvm_physseg_get_pg(upm, off);
- return(NULL);
-}
-
paddr_t
uvm_vm_page_to_phys(const struct vm_page *pg)
{
Index: src/sys/uvm/uvm_physseg.c
diff -u src/sys/uvm/uvm_physseg.c:1.18 src/sys/uvm/uvm_physseg.c:1.19
--- src/sys/uvm/uvm_physseg.c:1.18 Sun Apr 9 09:00:56 2023
+++ src/sys/uvm/uvm_physseg.c Sat Sep 23 18:20:20 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_physseg.c,v 1.18 2023/04/09 09:00:56 riastradh Exp $ */
+/* $NetBSD: uvm_physseg.c,v 1.19 2023/09/23 18:20:20 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -88,7 +88,9 @@
*/
struct uvm_physseg {
/* used during RB tree lookup for PHYS_TO_VM_PAGE(). */
+#if defined(UVM_HOTPLUG)
struct rb_node rb_node; /* tree information */
+#endif
paddr_t start; /* PF# of first page in segment */
paddr_t end; /* (PF# of last page in segment) + 1 */
struct vm_page *pgs; /* vm_page structures (from start) */
@@ -561,8 +563,10 @@ uvm_physseg_find(paddr_t pframe, psize_t
#define HANDLE_TO_PHYSSEG_NODE(h) (VM_PHYSMEM_PTR((int)h))
#define PHYSSEG_NODE_TO_HANDLE(u) ((int)((vsize_t) (u - vm_physmem) / sizeof(struct uvm_physseg)))
-static struct uvm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */
-static int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
+/* XXXCDC: uvm.physmem */
+static struct uvm_physseg vm_physmem[VM_PHYSSEG_MAX] __read_mostly;
+/* XXXCDC: uvm.nphysseg */
+static int vm_nphysseg __read_mostly = 0;
#define vm_nphysmem vm_nphysseg
void
@@ -851,7 +855,7 @@ static inline int vm_physseg_find_linear
/*
* vm_physseg_find: find vm_physseg structure that belongs to a PA
*/
-int
+inline int
uvm_physseg_find(paddr_t pframe, psize_t *offp)
{
@@ -943,6 +947,40 @@ vm_physseg_find_linear(struct uvm_physse
#endif
#endif /* UVM_HOTPLUG */
+/*
+ * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages
+ * back from an I/O mapping (ugh!). used in some MD code as well. it can
+ * be prominent in flamegraphs, so optimise it and try to make it easy for
+ * the compiler by including next to the inline lookup routines.
+ */
+struct vm_page *
+uvm_phys_to_vm_page(paddr_t pa)
+{
+#if VM_PHYSSEG_STRAT != VM_PSTRAT_BSEARCH
+ /* 'contig' and linear cases */
+ KASSERT(vm_nphysseg > 0);
+ struct uvm_physseg *ps = &vm_physmem[0];
+ struct uvm_physseg *end = &vm_physmem[vm_nphysseg];
+ paddr_t pframe = atop(pa);
+ do {
+ if (pframe >= ps->start && pframe < ps->end) {
+ return &ps->pgs[pframe - ps->start];
+ }
+ } while (VM_PHYSSEG_MAX > 1 && __predict_false(++ps < end));
+ return NULL;
+#else
+ /* binary search for it */
+ paddr_t pf = atop(pa);
+ paddr_t off;
+ uvm_physseg_t upm;
+
+ upm = uvm_physseg_find(pf, &off);
+ if (upm != UVM_PHYSSEG_TYPE_INVALID)
+ return uvm_physseg_get_pg(upm, off);
+ return(NULL);
+#endif
+}
+
bool
uvm_physseg_valid_p(uvm_physseg_t upm)
{
@@ -1047,7 +1085,7 @@ uvm_physseg_get_avail_end(uvm_physseg_t
return HANDLE_TO_PHYSSEG_NODE(upm)->avail_end;
}
-struct vm_page *
+inline struct vm_page *
uvm_physseg_get_pg(uvm_physseg_t upm, paddr_t idx)
{
KASSERT(uvm_physseg_valid_p(upm));
