Hi Hakan, I am sending the physical-memory discovery and Sv39 bootstrap patches as a small series. The first implements `pmap_discover_physical_memory()` using the DTB interface already integrated in your branch; the second implements `pmap_bootstrap()` and enables the initial Sv39 mappings.
Best regards, Diego El sáb, 5 sept 2026 a la(s) 6:45 p.m., Hakan Candar ([email protected]) escribió: > Hi Diego, > > I imported and integrated Sergey's DTB parser from his > `wip-aarch64` branch into ours. You can check `early_dtb_walk()` > under `model_dep.c` to see the initial scaffolding. > > Currently, we just iterate over the nodes and do nothing. > Next up, we'd want to discover and register the physical memory. > > Since you mentioned having Sv39 paging ready, I think you > have already implemented `pmap_discover_physical_memory()` > and `pmap_bootstrap()`? If that's the case, could you kindly > send your implementation as a patch? You can send it as a patch > series alongside the Sv39 implementation, if possible :) > > I was tinkering with the DTB parser and created a branch named > `demo/dtb-early-walk` in which I recursively print the DTB > in a human readable format similar to DTS. You can check > it to get a feeling of the DTB parser interface, if you'd like. > > Also, Sergey's branch is a very valuable reference for us, so > definitely check that out if you haven't already. I am currently > tracking it at this link: > > https://github.com/bugaevc/gnumach/tree/wip-aarch64 > > For Sergey: Is the link I mentioned above current? If you moved > the branch to another git instance, let me know so we can track > that instead. Thanks! > > Hakan >
From a8c265c303c2da6939dca975da60dec28d729e0f Mon Sep 17 00:00:00 2001 From: Diego Meretta <[email protected]> Date: Sat, 5 Sep 2026 20:23:19 -0300 Subject: [PATCH] riscv64: Discover physical memory from the device tree * riscv64/riscv64/model_dep.c: Register memory nodes found during the early DTB walk. * riscv64/riscv64/pmap.c: Read memory ranges from the DTB and load the largest region into the VM page database. * riscv64/riscv64/pmap.h: Declare the discovery interface. diff --git a/riscv64/riscv64/model_dep.c b/riscv64/riscv64/model_dep.c index d06d1af6..c22caef6 100644 --- a/riscv64/riscv64/model_dep.c +++ b/riscv64/riscv64/model_dep.c @@ -55,6 +55,7 @@ #include <kern/smp.h> #include <sys/types.h> #include <vm/vm_page.h> +#include <vm/pmap.h> #include <riscv64/model_dep.h> #include <riscv64/db_interface.h> @@ -204,8 +205,9 @@ early_dtb_walk(void) } dtb_for_each_prop(node, prop) { if (!strcmp(prop.name, "device_type") - && !strcmp(prop.data, "memory")) - {} /* TODO: discover physical memory */ + && !strcmp(prop.data, "memory")) { + pmap_discover_physical_memory(&node); + } } early_dtb_walk_visit_node(&node, NULL); } diff --git a/riscv64/riscv64/pmap.c b/riscv64/riscv64/pmap.c index 516554b2..b6be7bab 100644 --- a/riscv64/riscv64/pmap.c +++ b/riscv64/riscv64/pmap.c @@ -165,6 +165,36 @@ boolean_t pmap_initialized = FALSE; vm_offset_t kernel_virtual_start; vm_offset_t kernel_virtual_end; +/* Largest usable RAM region reported by the device tree. */ +static phys_addr_t phys_mem_start; +static vm_size_t phys_mem_size; + +void +pmap_discover_physical_memory(struct dtb_node *node) +{ + struct dtb_prop prop; + vm_size_t off = 0; + + prop = dtb_node_find_prop(node, "reg"); + assert(!DTB_IS_SENTINEL(prop)); + + while (off < prop.length) { + phys_addr_t start; + vm_size_t size; + + start = dtb_prop_read_cells(&prop, node->address_cells, &off); + size = dtb_prop_read_cells(&prop, node->size_cells, &off); + if (size > phys_mem_size) { + phys_mem_start = start; + phys_mem_size = size; + } + } + + assert(phys_mem_size != 0); + vm_page_load(VM_PAGE_SEG_DMA, phys_mem_start, + phys_mem_start + phys_mem_size); +} + /* * Index into pv_head table, its lock bits, and the modify/reference * bits. diff --git a/riscv64/riscv64/pmap.h b/riscv64/riscv64/pmap.h index 41aca6e9..49753e5a 100644 --- a/riscv64/riscv64/pmap.h +++ b/riscv64/riscv64/pmap.h @@ -473,6 +473,9 @@ extern pt_entry_t *kernel_page_dir; extern vm_offset_t kernel_virtual_start; extern vm_offset_t kernel_virtual_end; +struct dtb_node; +extern void pmap_discover_physical_memory(struct dtb_node *node); + /* * Bootstrap the system enough to run with virtual memory. * Allocate the kernel page directory and page tables,
From 0b53645c04a27e7bf096badc243d204c37ae9bde Mon Sep 17 00:00:00 2001 From: Diego Meretta <[email protected]> Date: Sat, 5 Sep 2026 20:23:19 -0300 Subject: [PATCH] riscv64: Bootstrap Sv39 page tables * riscv64/riscv64/pmap.c: Build the initial Sv39 tables and direct mappings. * riscv64/riscv64/pmap.h: Define the RISC-V page-table layout and PTE flags. * riscv64/riscv64/proc_reg.h: Add CSR and TLB helpers used by the bootstrap. * riscv64/riscv64/model_dep.c: Leave early page allocation to the pmap bootstrap. diff --git a/riscv64/riscv64/model_dep.c b/riscv64/riscv64/model_dep.c index d06d1af6..4110d093 100644 --- a/riscv64/riscv64/model_dep.c +++ b/riscv64/riscv64/model_dep.c @@ -55,6 +55,7 @@ #include <kern/smp.h> #include <sys/types.h> #include <vm/vm_page.h> +#include <vm/pmap.h> #include <riscv64/model_dep.h> #include <riscv64/db_interface.h> @@ -204,8 +205,9 @@ early_dtb_walk(void) } dtb_for_each_prop(node, prop) { if (!strcmp(prop.name, "device_type") - && !strcmp(prop.data, "memory")) - {} /* TODO: discover physical memory */ + && !strcmp(prop.data, "memory")) { + pmap_discover_physical_memory(&node); + } } early_dtb_walk_visit_node(&node, NULL); } @@ -271,15 +273,3 @@ init_alloc_aligned(vm_size_t size, vm_offset_t *addrp) /* TODO: implement */ return FALSE; } - -/* Grab a physical page: - the standard memory allocation mechanism - during system initialization. */ -vm_offset_t -pmap_grab_page(void) -{ - vm_offset_t addr; - if (!init_alloc_aligned(PAGE_SIZE, &addr)) - panic("Not enough memory to initialize Mach"); - return addr; -} diff --git a/riscv64/riscv64/pmap.c b/riscv64/riscv64/pmap.c index 516554b2..f006257b 100644 --- a/riscv64/riscv64/pmap.c +++ b/riscv64/riscv64/pmap.c @@ -74,6 +74,7 @@ #include <mach/vm_prot.h> #include <vm/vm_object.h> #include <vm/vm_page.h> +#include <device/cons.h> #include <vm/vm_user.h> #include <mach/machine/vm_param.h> @@ -165,6 +166,49 @@ boolean_t pmap_initialized = FALSE; vm_offset_t kernel_virtual_start; vm_offset_t kernel_virtual_end; +/* Largest usable RAM region reported by the device tree. */ +static phys_addr_t phys_mem_start; +static vm_size_t phys_mem_size; + +/* Early physical-memory allocator used while page tables are built. */ +static vm_offset_t bootstrap_heap; +extern char _end[]; + +vm_offset_t +pmap_grab_page(void) +{ + vm_offset_t page = bootstrap_heap; + + bootstrap_heap += PAGE_SIZE; + return page; +} + +void +pmap_discover_physical_memory(struct dtb_node *node) +{ + struct dtb_prop prop; + vm_size_t off = 0; + + prop = dtb_node_find_prop(node, "reg"); + assert(!DTB_IS_SENTINEL(prop)); + + while (off < prop.length) { + phys_addr_t start; + vm_size_t size; + + start = dtb_prop_read_cells(&prop, node->address_cells, &off); + size = dtb_prop_read_cells(&prop, node->size_cells, &off); + if (size > phys_mem_size) { + phys_mem_start = start; + phys_mem_size = size; + } + } + + assert(phys_mem_size != 0); + vm_page_load(VM_PAGE_SEG_DMA, phys_mem_start, + phys_mem_start + phys_mem_size); +} + /* * Index into pv_head table, its lock bits, and the modify/reference * bits. @@ -353,7 +397,7 @@ MACRO_END * invlpgs. But it surely is more expensive than just one invlpg. */ #define INVALIDATE_TLB(pmap, s, e) \ MACRO_BEGIN \ - panic("TODO: Not implemented"); \ + sfence_vma(); \ MACRO_END #endif /* MACH_PV_PAGETABLES */ @@ -439,26 +483,48 @@ static pmap_mapwindow_t mapwindows[PMAP_NMAPWINDOWS * NCPUS]; static inline pt_entry_t * pmap_pde(const pmap_t pmap, vm_offset_t addr) { - pt_entry_t *page_dir; if (pmap == kernel_pmap) addr = kvtolin(addr); - panic("TODO: not implemented"); + /* Sv39: root table entry (L2) for this address */ + return &pmap->root_table[lin2vpn2(addr)]; } /* * Given an offset and a map, compute the address of the * pte. If the address is invalid with respect to the map * then PT_ENTRY_NULL is returned (and the map may need to grow). * - * This is only used internally. + * Sv39: walk root → L1 → L0 to find the PTE for addr. */ pt_entry_t * pmap_pte(const pmap_t pmap, vm_offset_t addr) { - pt_entry_t *ptp; - pt_entry_t pte; + pt_entry_t *l2, *l1, *l0; + vm_offset_t lin = (pmap == kernel_pmap) ? kvtolin(addr) : addr; + + l2 = pmap->root_table; + if (l2 == NULL) + return PT_ENTRY_NULL; + + /* L2 entry */ + pt_entry_t l2e = l2[lin2vpn2(lin)]; + if (!(l2e & RISCV_PTE_V)) + return PT_ENTRY_NULL; + if (RISCV_PTE_IS_LEAF(l2e)) + return &l2[lin2vpn2(lin)]; /* 1GB megapage */ + + /* L1 entry */ + l1 = (pt_entry_t *)phystokv(pte_to_pa(l2e)); + pt_entry_t l1e = l1[lin2vpn1(lin)]; + if (!(l1e & RISCV_PTE_V)) + return PT_ENTRY_NULL; + if (RISCV_PTE_IS_LEAF(l1e)) { + return &l1[lin2vpn1(lin)]; /* 2MB megapage */ + } - panic("TODO: not implemented"); + /* L0 entry */ + l0 = (pt_entry_t *)phystokv(pte_to_pa(l1e)); + return &l0[lin2vpn0(lin)]; } #define DEBUG_PTE_PAGE 0 @@ -522,46 +588,184 @@ static void pmap_bootstrap_xen(pt_entry_t *l1_map[NSUP_L1]) * Bootstrap the system enough to run with virtual memory. * Allocate the kernel page directory and page tables, * and direct-map all physical memory. - * Called with mapping off. + * Called with mapping off (satp.MODE = Bare). */ void pmap_bootstrap(void) { + if (phys_mem_size == 0) + panic("riscv64: physical memory was not discovered"); + + bootstrap_heap = round_page((vm_offset_t) _end); + kernel_virtual_start = phystokv(round_page(phys_mem_start + phys_mem_size)); + kernel_virtual_end = VM_MAX_KERNEL_ADDRESS - PAGE_SIZE; + + pt_entry_t *root_table; /* L2 root — 512 entries, each covers 1GB */ + pt_entry_t *l1_kernel; /* L1 for kernel phys mem (root[2] and root[510]) */ + pt_entry_t *l1_devices; /* L1 for device MMIO (root[0]) */ +#define MAX_L0_TABLES 16 + pt_entry_t *l0_tables[MAX_L0_TABLES]; /* L0 tables for kernel VA */ + phys_addr_t l0_pas[MAX_L0_TABLES]; + phys_addr_t root_pa, l1k_pa, l1d_pa; + vm_offset_t pa; + unsigned int idx; + vm_offset_t directmap_end; + unsigned int num_l0_tables; + unsigned long nr_phys_pages; + /* - * Mapping is turned off; we must reference only physical addresses. - * The load image of the system is to be mapped 1-1 physical = virtual. + * The kernel's pmap is statically allocated. */ + kernel_pmap = &kernel_pmap_store; + +#if NCPUS > 1 + lock_init(&pmap_system_lock, FALSE); +#endif /* NCPUS > 1 */ + + simple_lock_init(&kernel_pmap->lock); + kernel_pmap->ref_count = 1; /* - * Set ptes_per_vm_page for general use. + * kernel_virtual_start was already set by c_boot_entry() + * to phystokv(end_of_physical_memory). Derive the physical + * end from it. */ -#if 0 - ptes_per_vm_page = PAGE_SIZE / INTEL_PGBYTES; -#endif + directmap_end = _kvtophys(kernel_virtual_start); + + printf("pmap: direct map ends at phys %lx, kernel virt %lx-%lx\n", + (unsigned long) directmap_end, + (unsigned long) kernel_virtual_start, + (unsigned long) kernel_virtual_end); /* - * The kernel's pmap is statically allocated so we don't - * have to use pmap_create, which is unlikely to work - * correctly at this part of the boot sequence. + * Allocate page table pages. At this point satp.MODE = Bare, + * so physical == virtual. We use physical addresses directly + * (no phystokv) until after we enable Sv39. */ + root_pa = pmap_grab_page(); + l1k_pa = pmap_grab_page(); + l1d_pa = pmap_grab_page(); - kernel_pmap = &kernel_pmap_store; + root_table = (pt_entry_t *)(vm_offset_t)root_pa; + l1_kernel = (pt_entry_t *)(vm_offset_t)l1k_pa; + l1_devices = (pt_entry_t *)(vm_offset_t)l1d_pa; -#if NCPUS > 1 - lock_init(&pmap_system_lock, FALSE); /* NOT a sleep lock */ -#endif /* NCPUS > 1 */ + memset(root_table, 0, PAGE_SIZE); + memset(l1_kernel, 0, PAGE_SIZE); + memset(l1_devices, 0, PAGE_SIZE); - simple_lock_init(&kernel_pmap->lock); + /* + * Allocate L0 page tables for the kernel virtual address space. + * + * pmap_steal_memory() allocates virtual pages starting at + * kernel_virtual_start and maps them with pmap_enter(). Each + * L0 table covers 512 × 4KB = 2MB of virtual space. We need + * enough L0 tables to hold: + * - hash buckets: ~nr_pages × 8 bytes + * - page table: ~nr_pages × sizeof(struct vm_page) bytes + * - slab/kmem/etc: additional bootstrap allocations + * + * Use ~512 bytes per physical page as a generous estimate + * to cover all bootstrap allocations (hash buckets, page table, + * slab_bootstrap, kmem_init, etc.). Each L0 table costs only + * 4KB of physical memory, so over-allocating is cheap. + */ + nr_phys_pages = (directmap_end - 0x80000000UL) / PAGE_SIZE; + num_l0_tables = (nr_phys_pages * 512 + (2 * 1024 * 1024) - 1) + / (2 * 1024 * 1024) + 4; + if (num_l0_tables < 8) + num_l0_tables = 8; + if (num_l0_tables > MAX_L0_TABLES) + num_l0_tables = MAX_L0_TABLES; + + printf("pmap: allocating %u L0 tables (%lu KB) for kernel VA\n", + num_l0_tables, + (unsigned long)(num_l0_tables * PAGE_SIZE / 1024)); + + for (idx = 0; idx < num_l0_tables; idx++) { + l0_pas[idx] = pmap_grab_page(); + l0_tables[idx] = (pt_entry_t *)(vm_offset_t)l0_pas[idx]; + memset(l0_tables[idx], 0, PAGE_SIZE); + } + printf("pmap: L0[0] at pa=%lx va=%lx\n", + (unsigned long)l0_pas[0], + (unsigned long)phystokv(l0_pas[0])); - kernel_pmap->ref_count = 1; + /* + * Populate L1 tables with 2MB megapages. + */ + { + pt_entry_t pte_flags = RISCV_PTE_V | RISCV_PTE_R | RISCV_PTE_W + | RISCV_PTE_X | RISCV_PTE_G + | RISCV_PTE_A | RISCV_PTE_D; + + unsigned int num_kernel = (directmap_end - 0x80000000) + / (2*1024*1024); + if (num_kernel > RISCV_PT_ENTRIES) + num_kernel = RISCV_PT_ENTRIES; + + for (idx = 0; idx < num_kernel; idx++) { + pa = 0x80000000UL + (vm_offset_t)idx * (2 * 1024 * 1024); + l1_kernel[idx] = pa_to_pte(pa) | pte_flags; + } + + unsigned int num_devices = 136; /* 136 × 2MB = 272MB, covers up to 0x11000000 */ + for (idx = 0; idx < num_devices; idx++) { + pa = (vm_offset_t)idx * (2 * 1024 * 1024); + l1_devices[idx] = pa_to_pte(pa) | pte_flags; + } + + } /* - * Determine the kernel virtual address range. - * It starts at the end of the physical memory - * mapped into the kernel address space, - * and extends to a stupid arbitrary limit beyond that. + * Wire L0 tables into l1_kernel for kernel_virtual_start. + * kernel_virtual_start = 0xFFFFFFFF88000000 has VPN[1]=64. + * Each L0 table covers 512 × 4KB = 2MB of virtual space. + * l1_kernel[64] covers VA 0xFFFFFFFF88000000-0xFFFFFFFF881FFFFF + * l1_kernel[65] covers VA 0xFFFFFFFF88200000-0xFFFFFFFF883FFFFF + * etc. */ + { + pt_entry_t ptr_flags = RISCV_PTE_V; + for (idx = 0; idx < num_l0_tables; idx++) { + l1_kernel[64 + idx] = pa_to_pte(l0_pas[idx]) | ptr_flags; + } + } - panic("TODO: not implemented"); + /* + * Wire up root table (pointer PTEs, not leaf). + */ + { + pt_entry_t ptr_flags = RISCV_PTE_V; + + root_table[0] = pa_to_pte(l1d_pa) | ptr_flags; + root_table[2] = pa_to_pte(l1k_pa) | ptr_flags; + root_table[510] = pa_to_pte(l1k_pa) | ptr_flags; + } + + /* + * Store in kernel_pmap (use phystokv now that paging is about to be on). + */ + kernel_pmap->root_table = (pt_entry_t *)phystokv(root_pa); + kernel_page_dir = kernel_pmap->root_table; + + /* + * Enable Sv39 paging. + */ + { + unsigned long satp_val = satp_sv39(root_pa); + satp_write(satp_val); + sfence_vma(); + } + + /* + * Now paging is ON. The identity map at root[2] covers + * VA 0x80000000 → PA 0x80000000, so the code at ~0x802xxxxx + * keeps working. phystokv() addresses now also work via root[510]. + */ + kernel_pmap->cpus_using = 1; + + printf("pmap: Sv39 enabled, satp=%lx, root=%p\n", + (unsigned long)satp_read(), kernel_pmap->root_table); } #ifdef MACH_PV_PAGETABLES @@ -691,7 +895,22 @@ void pmap_init(void) pmap_phys_attributes = (char *) addr; - panic("TODO: not implemented"); + /* + * Create the cache of physical maps, + * and of the physical-to-virtual entries. + */ + s = (vm_size_t) sizeof(struct pmap); + kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, 0); + kmem_cache_init(&pt_cache, "pmap_L0", + PAGE_SIZE, PAGE_SIZE, NULL, + KMEM_CACHE_PHYSMEM); + kmem_cache_init(&pd_cache, "pmap_L1", + PAGE_SIZE, PAGE_SIZE, NULL, + KMEM_CACHE_PHYSMEM); + s = (vm_size_t) sizeof(struct pv_entry); + kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, 0); + + pmap_initialized = TRUE; } static inline boolean_t @@ -1244,21 +1463,77 @@ void pmap_enter( vm_prot_t prot, boolean_t wired) { - boolean_t is_physmem; pt_entry_t *pte; - pv_entry_t pv_h; - unsigned long i, pai; - pv_entry_t pv_e; pt_entry_t template; int spl; - phys_addr_t old_pa; assert(pa != vm_page_fictitious_addr); if (pmap_debug) printf("pmap(%zx, %llx)\n", v, (unsigned long long) pa); if (pmap == PMAP_NULL) return; - panic("TODO: not implemented"); + PMAP_READ_LOCK(pmap, spl); + + pte = pmap_pte(pmap, v); + + if (pte == PT_ENTRY_NULL) { + /* + * No PTE exists — need to expand the page table. + * For now, only handle the kernel pmap during bootstrap. + */ + if (pmap == kernel_pmap) { + /* + * The kernel pmap has 2MB megapages at L1. + * pmap_pte returns NULL only if the L1 entry + * is not valid. This shouldn't happen after + * pmap_bootstrap(). Panic for debugging. + */ + panic("pmap_enter: no PTE for kernel va %lx", (unsigned long)v); + } + /* + * User pmap: would need pmap_expand. For now panic. + */ + panic("pmap_enter: pmap_expand not implemented for user pmap"); + } + + /* + * Build the new PTE. + */ + template = pa_to_pte(pa) | RISCV_PTE_V | RISCV_PTE_A | RISCV_PTE_D; + if (pmap != kernel_pmap) + template |= RISCV_PTE_U; + if (prot & VM_PROT_WRITE) + template |= RISCV_PTE_W; + if (prot & VM_PROT_READ) + template |= RISCV_PTE_R; + if (prot & VM_PROT_EXECUTE) + template |= RISCV_PTE_X; + + /* + * Check if we're pointing at a leaf PTE (2MB or 1GB megapage). + * If so, the mapping already covers this VA range. For the + * kernel bootstrap case (pmap_steal_memory), the physical + * address is within the direct-mapped region, so the existing + * megapage already maps it. Just update the PTE if needed. + */ + if (RISCV_PTE_IS_LEAF(*pte)) { + /* + * Already mapped by a megapage. For bootstrap, this is fine — + * the direct map already covers this PA. Nothing to do. + */ + PMAP_READ_UNLOCK(pmap, spl); + return; + } + + /* + * Regular 4KB page PTE — write it. + */ + WRITE_PTE(pte, template); + /* Force TLB flush during bootstrap */ + sfence_vma(); + + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); + PMAP_READ_UNLOCK(pmap, spl); } /* diff --git a/riscv64/riscv64/pmap.h b/riscv64/riscv64/pmap.h index 41aca6e9..0e6cfeac 100644 --- a/riscv64/riscv64/pmap.h +++ b/riscv64/riscv64/pmap.h @@ -49,138 +49,123 @@ */ /* - * i386/i486 Page Table Entry + * RISC-V Sv39 Page Table Entry */ -typedef phys_addr_t pt_entry_t; +typedef uint64_t pt_entry_t; #define PT_ENTRY_NULL ((pt_entry_t *) 0) -/* TODO IMPLEMENT COMPLETELY */ #endif /* __ASSEMBLER__ */ -#define INTEL_OFFMASK 0xfff /* offset within page */ -#if PAE -#define L4SHIFT 39 /* L4 shift */ -#define L4MASK 0x1ff /* mask for L4 index */ -#define PDPNUM_KERNEL (((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) >> PDPSHIFT) + 1) -#define PDPMASK 0x1ff /* mask for page directory pointer index */ -#define PDPSHIFT 30 /* page directory pointer */ -#define PDESHIFT 21 /* page descriptor shift */ -#define PDEMASK 0x1ff /* mask for page descriptor index */ -#define PTESHIFT 12 /* page table shift */ -#define PTEMASK 0x1ff /* mask for page table index */ -#else /* PAE */ -#define PDPNUM 1 /* number of page directory pointers */ -#define PDESHIFT 22 /* page descriptor shift */ -#define PDEMASK 0x3ff /* mask for page descriptor index */ -#define PTESHIFT 12 /* page table shift */ -#define PTEMASK 0x3ff /* mask for page table index */ -#endif /* PAE */ - -/* - * Convert linear offset to L4 pointer index +/* ---- Sv39 virtual address layout ---- + * Bits 63:39 = bits[38] sign-extended (canonical) + * Bits 38:30 = VPN[2] (9 bits) → L2 index (root) + * Bits 29:21 = VPN[1] (9 bits) → L1 index + * Bits 20:12 = VPN[0] (9 bits) → L0 index + * Bits 11:0 = page offset (12 bits) */ -#define lin2l4num(a) (((a) >> L4SHIFT) & L4MASK) - -/* - * Convert linear offset to page descriptor index - */ -#define lin2pdenum(a) (((a) >> PDESHIFT) & PDEMASK) - -#if PAE -/* Special version assuming contiguous page directories. Making it - include the page directory pointer table index too. */ -#define lin2pdenum_cont(a) (((a) >> PDESHIFT) & 0x3ff) -#else -#define lin2pdenum_cont(a) lin2pdenum(a) -#endif - -/* - * Convert linear offset to page directory pointer index - */ -#if PAE -#define lin2pdpnum(a) (((a) >> PDPSHIFT) & PDPMASK) -#endif - -/* - * Convert page descriptor index to linear address - */ -#define pdenum2lin(a) ((vm_offset_t)(a) << PDESHIFT) - -#if PAE -#define pagenum2lin(l4num, l3num, l2num, l1num) \ - (((vm_offset_t)(l4num) << L4SHIFT) + \ - ((vm_offset_t)(l3num) << PDPSHIFT) + \ - ((vm_offset_t)(l2num) << PDESHIFT) + \ - ((vm_offset_t)(l1num) << PTESHIFT)) -#else /* PAE */ -#define pagenum2lin(l4num, l3num, l2num, l1num) \ - (((vm_offset_t)(l2num) << PDESHIFT) + \ - ((vm_offset_t)(l1num) << PTESHIFT)) -#endif - -/* - * Convert linear offset to page table index - */ -#define ptenum(a) (((a) >> PTESHIFT) & PTEMASK) - -#define NPTES (intel_ptob(1)/sizeof(pt_entry_t)) -#define NPDES (PDPNUM * (intel_ptob(1)/sizeof(pt_entry_t))) - -/* - * Hardware pte bit definitions (to be used directly on the ptes - * without using the bit fields). +#define INTEL_OFFMASK 0xfff /* offset within page — kept for compat */ + +/* Sv39 page table geometry */ +#define RISCV_PT_LEVELS 3 +#define RISCV_PT_SHIFT 12 /* page table covers 2^12 = 4096 entries? No: 2^9 = 512 */ +#define RISCV_VPN_BITS 9 +#define RISCV_PTE_SIZE 8 /* sizeof(pt_entry_t) */ +#define RISCV_PT_ENTRIES 512 /* 2^9 entries per page table */ + +/* VPN extraction from virtual address */ +#define RISCV_VPN2_SHIFT 30 +#define RISCV_VPN1_SHIFT 21 +#define RISCV_VPN0_SHIFT 12 + +#define RISCV_VPN_MASK 0x1ff /* 9-bit mask */ + +#define lin2vpn2(a) (((a) >> RISCV_VPN2_SHIFT) & RISCV_VPN_MASK) +#define lin2vpn1(a) (((a) >> RISCV_VPN1_SHIFT) & RISCV_VPN_MASK) +#define lin2vpn0(a) (((a) >> RISCV_VPN0_SHIFT) & RISCV_VPN_MASK) + +/* Compat aliases for i386-originated code */ +#define L4SHIFT RISCV_VPN2_SHIFT +#define L4MASK RISCV_VPN_MASK +#define PDPSHIFT RISCV_VPN2_SHIFT +#define PDPMASK RISCV_VPN_MASK +#define PDESHIFT RISCV_VPN1_SHIFT +#define PDEMASK RISCV_VPN_MASK +#define PTESHIFT RISCV_VPN0_SHIFT +#define PTEMASK RISCV_VPN_MASK + +#define lin2l4num(a) lin2vpn2(a) +#define lin2pdenum(a) lin2vpn1(a) +#define lin2pdenum_cont(a) lin2vpn1(a) +#define lin2pdpnum(a) lin2vpn2(a) +#define ptenum(a) lin2vpn0(a) + +#define pdenum2lin(a) ((vm_offset_t)(a) << PDESHIFT) + +#define pagenum2lin(l4, l3, l2, l1) \ + (((vm_offset_t)(l4) << RISCV_VPN2_SHIFT) + \ + ((vm_offset_t)(l3) << RISCV_VPN1_SHIFT) + \ + ((vm_offset_t)(l2) << RISCV_VPN0_SHIFT) + \ + ((vm_offset_t)(l1) << 12)) + +#define PDPNUM RISCV_PT_ENTRIES +#define PDPNUM_KERNEL RISCV_PT_ENTRIES + +#define NPTES RISCV_PT_ENTRIES +#define NPDES (PDPNUM * RISCV_PT_ENTRIES) + +/* ---- RISC-V Sv39 PTE flags (bits 0-7) ---- */ + +#define RISCV_PTE_V (1UL << 0) /* Valid */ +#define RISCV_PTE_R (1UL << 1) /* Read */ +#define RISCV_PTE_W (1UL << 2) /* Write */ +#define RISCV_PTE_X (1UL << 3) /* Execute */ +#define RISCV_PTE_U (1UL << 4) /* User */ +#define RISCV_PTE_G (1UL << 5) /* Global */ +#define RISCV_PTE_A (1UL << 6) /* Accessed */ +#define RISCV_PTE_D (1UL << 7) /* Dirty */ + +/* PPN field: bits 10:53 (Sv39) */ +#define RISCV_PTE_PPN_SHIFT 10 +#define RISCV_PTE_PPN_MASK 0x3FFFFFFFFFFFFC00UL + +/* Leaf PTE test: R, W, or X set */ +#define RISCV_PTE_IS_LEAF(pte) ((pte) & (RISCV_PTE_R | RISCV_PTE_W | RISCV_PTE_X)) + +/* ---- Compat aliases mapping Intel PTE names to RISC-V ---- */ + +#define INTEL_PTE_VALID RISCV_PTE_V +#define INTEL_PTE_WRITE RISCV_PTE_W +#define INTEL_PTE_USER RISCV_PTE_U +#define INTEL_PTE_WTHRU 0 +#define INTEL_PTE_NCACHE 0 +#define INTEL_PTE_REF RISCV_PTE_A +#define INTEL_PTE_MOD RISCV_PTE_D +#define INTEL_PTE_PS 0 +#define INTEL_PTE_GLOBAL RISCV_PTE_G +#define INTEL_PTE_WIRED 0 +#define INTEL_PTE_PFN RISCV_PTE_PPN_MASK + +/* PTE ↔ physical address conversion + * + * In Sv39, PPN occupies bits [53:10] of the PTE. + * PPN = PA >> 12, so PTE_PPN field = (PA >> 12) << 10 = PA >> 2. + * Conversely, PA = PPN << 12 = (PTE_PPN >> 10) << 12 = PTE_PPN << 2. */ +#define pa_to_pte(a) (((pt_entry_t)(a) >> 2) & RISCV_PTE_PPN_MASK) +#define pte_to_pa(p) (((p) & RISCV_PTE_PPN_MASK) << 2) +#define pte_increment_pa(p) ((p) += (1UL << RISCV_PTE_PPN_SHIFT)) /* +4KB */ -#define INTEL_PTE_VALID 0x00000001 -#define INTEL_PTE_WRITE 0x00000002 -#define INTEL_PTE_USER 0x00000004 -#define INTEL_PTE_WTHRU 0x00000008 -#define INTEL_PTE_NCACHE 0x00000010 -#define INTEL_PTE_REF 0x00000020 -#define INTEL_PTE_MOD 0x00000040 -#define INTEL_PTE_PS 0x00000080 -#ifdef MACH_PV_PAGETABLES -/* Not supported */ -#define INTEL_PTE_GLOBAL 0x00000000 -#else /* MACH_PV_PAGETABLES */ -#define INTEL_PTE_GLOBAL 0x00000100 -#endif /* MACH_PV_PAGETABLES */ -#define INTEL_PTE_WIRED 0x00000200 -#ifdef PAE -#define INTEL_PTE_PFN 0xfffffffffffff000ULL -#else -#define INTEL_PTE_PFN 0xfffff000 -#endif - -#define pa_to_pte(a) ((a) & INTEL_PTE_PFN) -#ifdef MACH_PSEUDO_PHYS -#define pte_to_pa(p) ma_to_pa((p) & INTEL_PTE_PFN) -#else /* MACH_PSEUDO_PHYS */ -#define pte_to_pa(p) ((p) & INTEL_PTE_PFN) -#endif /* MACH_PSEUDO_PHYS */ -#define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1) - -/* - * Convert page table entry to kernel virtual address - */ -#define ptetokv(a) (phystokv(pte_to_pa(a))) +/* Convert PTE to kernel virtual address */ +#define ptetokv(a) (phystokv(pte_to_pa(a))) #ifndef __ASSEMBLER__ typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */ /* changed by other processors */ struct pmap { -#if ! PAE - pt_entry_t *dirbase; /* page directory table */ -#else /* PAE */ - pt_entry_t *l4base; /* l4 table */ -#ifdef MACH_HYP - pt_entry_t *user_l4base; /* Userland l4 table */ - pt_entry_t *user_pdpbase; /* Userland l4 table */ -#endif /* MACH_HYP */ -#endif /* PAE */ + pt_entry_t *root_table; /* Sv39 root page table (satp.PPN) */ int ref_count; /* reference count */ decl_simple_lock_data(,lock) /* lock on map */ @@ -200,19 +185,12 @@ extern void pmap_map_mfn(void *addr, unsigned long mfn); extern void pmap_clear_bootstrap_pagetable(pt_entry_t *addr); #endif /* MACH_PV_PAGETABLES */ -#if PAE -/* TODO: support PCID */ -#ifdef MACH_HYP +/* Switch the hardware page table to the given pmap's root table */ #define set_pmap(pmap) \ MACRO_BEGIN \ - panic("not implemented"); \ + satp_write(satp_sv39(kvtophys((vm_offset_t)(pmap)->root_table))); \ + sfence_vma(); \ MACRO_END -#else /* MACH_HYP */ - #define set_pmap(pmap) panic("not implemented"); /* set_cr3(kvtophys((vm_offset_t)(pmap)->l4base)) */ -#endif /* MACH_HYP */ -#else /* PAE */ - #define set_pmap(pmap) panic("not implemented"); /* set_cr3(kvtophys((vm_offset_t)(pmap)->dirbase)) */ -#endif /* PAE */ typedef struct { pt_entry_t *entry; @@ -462,8 +440,8 @@ MACRO_END #define pmap_kernel() (kernel_pmap) #define pmap_resident_count(pmap) ((pmap)->stats.resident_count) -#define pmap_phys_address(frame) (0) /* TODO: implement */ -#define pmap_phys_to_frame(phys) (0) /* TODO: implement */ +#define pmap_phys_address(frame) ((frame) << PAGE_SHIFT) +#define pmap_phys_to_frame(phys) ((phys) >> PAGE_SHIFT) #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr) #define pmap_attribute(pmap,addr,size,attr,value) \ (KERN_INVALID_ADDRESS) diff --git a/riscv64/riscv64/proc_reg.h b/riscv64/riscv64/proc_reg.h index 13a90517..0b4c2e39 100644 --- a/riscv64/riscv64/proc_reg.h +++ b/riscv64/riscv64/proc_reg.h @@ -29,6 +29,77 @@ #ifndef _RISCV64_PROC_REG_H_ #define _RISCV64_PROC_REG_H_ -#define SSTATUS_SIE (1 << 1) +/* ---- CSR access macros ---- */ + +#define csr_read(csr) \ +({ \ + unsigned long __val; \ + asm volatile ("csrr %0, " #csr : "=r" (__val)); \ + __val; \ +}) + +#define csr_write(csr, val) \ +({ \ + unsigned long __val = (unsigned long)(val); \ + asm volatile ("csrw " #csr ", %0" :: "r" (__val)); \ +}) + +/* ---- Supervisor Address Translation and Protection (satp) ---- */ + +#define SATP_MODE_BARE 0UL +#define SATP_MODE_SV39 8UL +#define SATP_MODE_SV48 9UL + +#define SATP_MODE_SHIFT 60 +#define SATP_ASID_SHIFT 44 +#define SATP_PPN_SHIFT 0 +#define SATP_PPN_MASK 0xFFFFFFFFFFFUL + +#define SATP_MODE(mode) ((unsigned long)(mode) << SATP_MODE_SHIFT) + +#define satp_read() csr_read(satp) +#define satp_write(val) csr_write(satp, val) + +/* Build satp value for Sv39 */ +#define satp_sv39(pa) (SATP_MODE(SATP_MODE_SV39) | \ + (((unsigned long)(pa)) >> 12)) + +/* ---- TLB management ---- */ + +#define sfence_vma() \ +({ \ + asm volatile ("sfence.vma" ::: "memory"); \ +}) + +#define sfence_vma_addr(va) \ +({ \ + asm volatile ("sfence.vma %0" :: "r" (va) : "memory"); \ +}) + +/* ---- Supervisor status register (sstatus) ---- */ + +#define SSTATUS_SPP (1UL << 8) +#define SSTATUS_SPIE (1UL << 5) +#define SSTATUS_SIE (1UL << 1) +#define SSTATUS_SUM (1UL << 18) + +#define sstatus_read() csr_read(sstatus) +#define sstatus_write(val) csr_write(sstatus, val) + +/* ---- Supervisor trap vector (stvec) ---- */ + +#define stvec_read() csr_read(stvec) +#define stvec_write(val) csr_write(stvec, val) + +/* ---- Other supervisor CSRs ---- */ + +#define sscratch_read() csr_read(sscratch) +#define sscratch_write(val) csr_write(sscratch, val) + +#define sepc_read() csr_read(sepc) +#define sepc_write(val) csr_write(sepc, val) + +#define scause_read() csr_read(scause) +#define stval_read() csr_read(stval) #endif /* _RISCV64_PROC_REG_H_ */
