Le 15/02/2023 à 03:01, Rohan McLure a écrit :
> On creation and clearing of a page table mapping, instrument such calls
> by invoking page_table_check_pte_set and page_table_check_pte_clear
> respectively. These calls serve as a sanity check against illegal
> mappings.
>
> Enable ARCH_SUPPORTS_PAGE_TABLE_CHECK for all platforms.
>
> Use set_pte internally, and cause this function to reassign a page table
> entry without instrumentation. Generic code will be instrumented, as it
> references set_pte_at.
>
> See also:
>
> riscv support in commit 3fee229a8eb9 ("riscv/mm: enable
> ARCH_SUPPORTS_PAGE_TABLE_CHECK")
> arm64 in commit 42b2547137f5 ("arm64/mm: enable
> ARCH_SUPPORTS_PAGE_TABLE_CHECK")
> x86_64 in commit d283d422c6c4 ("x86: mm: add x86_64 support for page table
> check")
>
> Signed-off-by: Rohan McLure <rmcl...@linux.ibm.com>
Reviewed-by: Christophe Leroy <christophe.le...@csgroup.eu>
> ---
> V2: Update spacing and types assigned to pte_update calls.
> V3: Update one last pte_update call to remove __pte invocation.
> V5: Fix 32-bit nohash double set
> V6: Omit __set_pte_at instrumentation - should be instrumented by
> set_pte_at, with set_pte in between, performing all prior checks.
> Instrument pmds. Use set_pte where needed.
> V7: Make set_pte_at an inline function. Fix commit message.
> Detail changes of internal references to set_pte_at, and its semantics.
> ---
> arch/powerpc/Kconfig | 1 +
> arch/powerpc/include/asm/book3s/32/pgtable.h | 8 +++-
> arch/powerpc/include/asm/book3s/64/pgtable.h | 44 ++++++++++++++++----
> arch/powerpc/include/asm/nohash/32/pgtable.h | 7 +++-
> arch/powerpc/include/asm/nohash/64/pgtable.h | 8 +++-
> arch/powerpc/include/asm/pgtable.h | 10 ++++-
> arch/powerpc/mm/book3s64/hash_pgtable.c | 2 +-
> arch/powerpc/mm/book3s64/pgtable.c | 16 ++++---
> arch/powerpc/mm/book3s64/radix_pgtable.c | 10 ++---
> arch/powerpc/mm/nohash/book3e_pgtable.c | 2 +-
> arch/powerpc/mm/pgtable_32.c | 2 +-
> 11 files changed, 83 insertions(+), 27 deletions(-)
>
> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> index 2c9cdf1d8761..2474e2699037 100644
> --- a/arch/powerpc/Kconfig
> +++ b/arch/powerpc/Kconfig
> @@ -154,6 +154,7 @@ config PPC
> select ARCH_STACKWALK
> select ARCH_SUPPORTS_ATOMIC_RMW
> select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x
> + select ARCH_SUPPORTS_PAGE_TABLE_CHECK
> select ARCH_USE_BUILTIN_BSWAP
> select ARCH_USE_CMPXCHG_LOCKREF if PPC64
> select ARCH_USE_MEMTEST
> diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h
> b/arch/powerpc/include/asm/book3s/32/pgtable.h
> index afd672e84791..8850b4fb22a4 100644
> --- a/arch/powerpc/include/asm/book3s/32/pgtable.h
> +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
> @@ -53,6 +53,8 @@
>
> #ifndef __ASSEMBLY__
>
> +#include <linux/page_table_check.h>
> +
> static inline bool pte_user(pte_t pte)
> {
> return pte_val(pte) & _PAGE_USER;
> @@ -338,7 +340,11 @@ static inline int __ptep_test_and_clear_young(struct
> mm_struct *mm,
> static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long
> addr,
> pte_t *ptep)
> {
> - return __pte(pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, 0, 0));
> + pte_t old_pte = __pte(pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, 0, 0));
> +
> + page_table_check_pte_clear(mm, addr, old_pte);
> +
> + return old_pte;
> }
>
> #define __HAVE_ARCH_PTEP_SET_WRPROTECT
> diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h
> b/arch/powerpc/include/asm/book3s/64/pgtable.h
> index a6ed93d01da1..0c6838875720 100644
> --- a/arch/powerpc/include/asm/book3s/64/pgtable.h
> +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
> @@ -162,6 +162,8 @@
> #define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX)
>
> #ifndef __ASSEMBLY__
> +#include <linux/page_table_check.h>
> +
> /*
> * page table defines
> */
> @@ -431,8 +433,11 @@ static inline void huge_ptep_set_wrprotect(struct
> mm_struct *mm,
> static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
> unsigned long addr, pte_t *ptep)
> {
> - unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
> - return __pte(old);
> + pte_t old_pte = __pte(pte_update(mm, addr, ptep, ~0UL, 0, 0));
> +
> + page_table_check_pte_clear(mm, addr, old_pte);
> +
> + return old_pte;
> }
>
> #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
> @@ -441,11 +446,16 @@ static inline pte_t ptep_get_and_clear_full(struct
> mm_struct *mm,
> pte_t *ptep, int full)
> {
> if (full && radix_enabled()) {
> + pte_t old_pte;
> +
> /*
> * We know that this is a full mm pte clear and
> * hence can be sure there is no parallel set_pte.
> */
> - return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
> + old_pte = radix__ptep_get_and_clear_full(mm, addr, ptep, full);
> + page_table_check_pte_clear(mm, addr, old_pte);
> +
> + return old_pte;
> }
> return ptep_get_and_clear(mm, addr, ptep);
> }
> @@ -1249,17 +1259,33 @@ extern int pmdp_test_and_clear_young(struct
> vm_area_struct *vma,
> static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
> unsigned long addr, pmd_t *pmdp)
> {
> - if (radix_enabled())
> - return radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
> - return hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
> + pmd_t old_pmd;
> +
> + if (radix_enabled()) {
> + old_pmd = radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
> + } else {
> + old_pmd = hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
> + }
> +
> + page_table_check_pmd_clear(mm, addr, old_pmd);
> +
> + return old_pmd;
> }
>
> static inline pmd_t __pmdp_collapse_flush(struct vm_area_struct *vma,
> struct mm_struct *mm,
> unsigned long address, pmd_t *pmdp)
> {
> - if (radix_enabled())
> - return radix__pmdp_collapse_flush(vma, address, pmdp);
> - return hash__pmdp_collapse_flush(vma, address, pmdp);
> + pmd_t old_pmd;
> +
> + if (radix_enabled()) {
> + old_pmd = radix__pmdp_collapse_flush(vma, address, pmdp);
> + } else {
> + old_pmd = hash__pmdp_collapse_flush(vma, address, pmdp);
> + }
> +
> + page_table_check_pmd_clear(mm, address, old_pmd);
> +
> + return old_pmd;
> }
> #define pmdp_collapse_flush(__vma, __addr, __pmdp) \
> ({ \
> diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h
> b/arch/powerpc/include/asm/nohash/32/pgtable.h
> index d953533c56ff..e9c77054fe0b 100644
> --- a/arch/powerpc/include/asm/nohash/32/pgtable.h
> +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
> @@ -166,6 +166,7 @@ void unmap_kernel_page(unsigned long va);
> #define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED |
> _PAGE_SPECIAL)
>
> #ifndef __ASSEMBLY__
> +#include <linux/page_table_check.h>
>
> #define pte_clear(mm, addr, ptep) \
> do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0)
> @@ -316,7 +317,11 @@ static inline int __ptep_test_and_clear_young(struct
> mm_struct *mm,
> static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long
> addr,
> pte_t *ptep)
> {
> - return __pte(pte_update(mm, addr, ptep, ~0, 0, 0));
> + pte_t old_pte = __pte(pte_update(mm, addr, ptep, ~0, 0, 0));
> +
> + page_table_check_pte_clear(mm, addr, old_pte);
> +
> + return old_pte;
> }
>
> #define __HAVE_ARCH_PTEP_SET_WRPROTECT
> diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h
> b/arch/powerpc/include/asm/nohash/64/pgtable.h
> index 14e69ebad31f..d88b22c753d3 100644
> --- a/arch/powerpc/include/asm/nohash/64/pgtable.h
> +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
> @@ -83,6 +83,7 @@
> #define H_PAGE_4K_PFN 0
>
> #ifndef __ASSEMBLY__
> +#include <linux/page_table_check.h>
> /* pte_clear moved to later in this file */
>
> static inline pte_t pte_mkwrite(pte_t pte)
> @@ -259,8 +260,11 @@ static inline void huge_ptep_set_wrprotect(struct
> mm_struct *mm,
> static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
> unsigned long addr, pte_t *ptep)
> {
> - unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
> - return __pte(old);
> + pte_t old_pte = __pte(pte_update(mm, addr, ptep, ~0UL, 0, 0));
> +
> + page_table_check_pte_clear(mm, addr, old_pte);
> +
> + return old_pte;
> }
>
> static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
> diff --git a/arch/powerpc/include/asm/pgtable.h
> b/arch/powerpc/include/asm/pgtable.h
> index b76fdb80b6c9..5715350e603a 100644
> --- a/arch/powerpc/include/asm/pgtable.h
> +++ b/arch/powerpc/include/asm/pgtable.h
> @@ -5,6 +5,7 @@
> #ifndef __ASSEMBLY__
> #include <linux/mmdebug.h>
> #include <linux/mmzone.h>
> +#include <linux/page_table_check.h>
> #include <asm/processor.h> /* For TASK_SIZE */
> #include <asm/mmu.h>
> #include <asm/page.h>
> @@ -48,7 +49,7 @@ struct mm_struct;
> /* Keep these as a macros to avoid include dependency mess */
> #define pte_page(x) pfn_to_page(pte_pfn(x))
> #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
> -#define set_pte_at set_pte
> +
> /*
> * Select all bits except the pfn
> */
> @@ -153,6 +154,13 @@ struct seq_file;
> void arch_report_meminfo(struct seq_file *m);
> #endif /* CONFIG_PPC64 */
>
> +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
> + pte_t *ptep, pte_t pte)
> +{
> + page_table_check_pte_set(mm, addr, ptep, pte);
> + set_pte(mm, addr, ptep, pte);
> +}
> +
> /*
> * Currently only consumed by page_table_check_pud_{set,clear}. Since clears
> * and sets to page table entries at any level are done through
> diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c
> b/arch/powerpc/mm/book3s64/hash_pgtable.c
> index 51f48984abca..a92a8a7c9199 100644
> --- a/arch/powerpc/mm/book3s64/hash_pgtable.c
> +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c
> @@ -165,7 +165,7 @@ int hash__map_kernel_page(unsigned long ea, unsigned long
> pa, pgprot_t prot)
> ptep = pte_alloc_kernel(pmdp, ea);
> if (!ptep)
> return -ENOMEM;
> - set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
> + set_pte(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
> } else {
> /*
> * If the mm subsystem is not fully up, we cannot create a
> diff --git a/arch/powerpc/mm/book3s64/pgtable.c
> b/arch/powerpc/mm/book3s64/pgtable.c
> index 85c84e89e3ea..d95be1d08b79 100644
> --- a/arch/powerpc/mm/book3s64/pgtable.c
> +++ b/arch/powerpc/mm/book3s64/pgtable.c
> @@ -9,6 +9,7 @@
> #include <linux/memremap.h>
> #include <linux/pkeys.h>
> #include <linux/debugfs.h>
> +#include <linux/page_table_check.h>
> #include <misc/cxl-base.h>
>
> #include <asm/pgalloc.h>
> @@ -87,7 +88,10 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
> WARN_ON(!(pmd_large(pmd)));
> #endif
> trace_hugepage_set_pmd(addr, pmd_val(pmd));
> - return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
> +
> + page_table_check_pmd_set(mm, addr, pmdp, pmd);
> +
> + return set_pte(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
> }
>
> static void do_serialize(void *arg)
> @@ -122,11 +126,13 @@ void serialize_against_pte_lookup(struct mm_struct *mm)
> pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
> pmd_t *pmdp)
> {
> - unsigned long old_pmd;
> + pmd_t old_pmd;
>
> - old_pmd = pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT,
> _PAGE_INVALID);
> + old_pmd = __pmd(pmd_hugepage_update(vma->vm_mm, address, pmdp,
> _PAGE_PRESENT, _PAGE_INVALID));
> flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
> - return __pmd(old_pmd);
> + page_table_check_pmd_clear(vma->vm_mm, address, old_pmd);
> +
> + return old_pmd;
> }
>
> pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
> @@ -460,7 +466,7 @@ void ptep_modify_prot_commit(struct vm_area_struct *vma,
> unsigned long addr,
> if (radix_enabled())
> return radix__ptep_modify_prot_commit(vma, addr,
> ptep, old_pte, pte);
> - set_pte_at(vma->vm_mm, addr, ptep, pte);
> + set_pte(vma->vm_mm, addr, ptep, pte);
> }
>
> /*
> diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c
> b/arch/powerpc/mm/book3s64/radix_pgtable.c
> index 4e46e001c3c3..9359e3589107 100644
> --- a/arch/powerpc/mm/book3s64/radix_pgtable.c
> +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
> @@ -110,7 +110,7 @@ static int early_map_kernel_page(unsigned long ea,
> unsigned long pa,
> ptep = pte_offset_kernel(pmdp, ea);
>
> set_the_pte:
> - set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
> + set_pte(&init_mm, ea, ptep, pfn_pte(pfn, flags));
> asm volatile("ptesync": : :"memory");
> return 0;
> }
> @@ -170,7 +170,7 @@ static int __map_kernel_page(unsigned long ea, unsigned
> long pa,
> return -ENOMEM;
>
> set_the_pte:
> - set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
> + set_pte(&init_mm, ea, ptep, pfn_pte(pfn, flags));
> asm volatile("ptesync": : :"memory");
> return 0;
> }
> @@ -1094,7 +1094,7 @@ void radix__ptep_modify_prot_commit(struct
> vm_area_struct *vma,
> (atomic_read(&mm->context.copros) > 0))
> radix__flush_tlb_page(vma, addr);
>
> - set_pte_at(mm, addr, ptep, pte);
> + set_pte(mm, addr, ptep, pte);
> }
>
> int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
> @@ -1105,7 +1105,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t
> prot)
> if (!radix_enabled())
> return 0;
>
> - set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
> + set_pte(&init_mm, 0 /* radix unused */, ptep, new_pud);
>
> return 1;
> }
> @@ -1152,7 +1152,7 @@ int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t
> prot)
> if (!radix_enabled())
> return 0;
>
> - set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
> + set_pte(&init_mm, 0 /* radix unused */, ptep, new_pmd);
>
> return 1;
> }
> diff --git a/arch/powerpc/mm/nohash/book3e_pgtable.c
> b/arch/powerpc/mm/nohash/book3e_pgtable.c
> index b80fc4a91a53..e50d22c6f983 100644
> --- a/arch/powerpc/mm/nohash/book3e_pgtable.c
> +++ b/arch/powerpc/mm/nohash/book3e_pgtable.c
> @@ -111,7 +111,7 @@ int __ref map_kernel_page(unsigned long ea, unsigned long
> pa, pgprot_t prot)
> }
> ptep = pte_offset_kernel(pmdp, ea);
> }
> - set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
> + set_pte(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
>
> smp_wmb();
> return 0;
> diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
> index 5c02fd08d61e..a86a16be24ea 100644
> --- a/arch/powerpc/mm/pgtable_32.c
> +++ b/arch/powerpc/mm/pgtable_32.c
> @@ -89,7 +89,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa,
> pgprot_t prot)
> * hash table
> */
> BUG_ON((pte_present(*pg) | pte_hashpte(*pg)) &&
> pgprot_val(prot));
> - set_pte_at(&init_mm, va, pg, pfn_pte(pa >> PAGE_SHIFT, prot));
> + set_pte(&init_mm, va, pg, pfn_pte(pa >> PAGE_SHIFT, prot));
> }
> smp_wmb();
> return err;
