From: Junaid Shahid <juna...@google.com>
This adds custom allocation and free functions for ASI page tables.
The alloc functions support allocating memory using different GFP
reclaim flags, in order to be able to support non-sensitive allocations
from both standard and atomic contexts. They also install the page
tables locklessly, which makes it slightly simpler to handle
non-sensitive allocations from interrupts/exceptions.
checkpatch.pl MACRO_ARG_UNUSED,SPACING is false positive. COMPLEX_MACRO - I
dunno, suggestions welcome.
Checkpatch-args: --ignore=MACRO_ARG_UNUSED,SPACING,COMPLEX_MACRO
Signed-off-by: Junaid Shahid <juna...@google.com>
Signed-off-by: Brendan Jackman <jackm...@google.com>
---
arch/x86/mm/asi.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 59 insertions(+)
diff --git a/arch/x86/mm/asi.c b/arch/x86/mm/asi.c
index
8d060c633be68b508847e2c1c111761df1da92af..b15d043acedc9f459f17e86564a15061650afc3a
100644
--- a/arch/x86/mm/asi.c
+++ b/arch/x86/mm/asi.c
@@ -73,6 +73,65 @@ const char *asi_class_name(enum asi_class_id class_id)
return asi_class_names[class_id];
}
+#ifndef mm_inc_nr_p4ds
+#define mm_inc_nr_p4ds(mm) do {} while (false)
+#endif
+
+#ifndef mm_dec_nr_p4ds
+#define mm_dec_nr_p4ds(mm) do {} while (false)
+#endif
+
+#define pte_offset pte_offset_kernel
+
+/*
+ * asi_p4d_alloc, asi_pud_alloc, asi_pmd_alloc, asi_pte_alloc.
+ *
+ * These are like the normal xxx_alloc functions, but:
+ *
+ * - They use atomic operations instead of taking a spinlock; this allows them
+ * to be used from interrupts. This is necessary because we use the page
+ * allocator from interrupts and the page allocator ultimately calls this
+ * code.
+ * - They support customizing the allocation flags.
+ *
+ * On the other hand, they do not use the normal page allocation
infrastructure,
+ * that means that PTE pages do not have the PageTable type nor the PagePgtable
+ * flag and we don't increment the meminfo stat (NR_PAGETABLE) as they do.
+ */
+static_assert(!IS_ENABLED(CONFIG_PARAVIRT));
+#define DEFINE_ASI_PGTBL_ALLOC(base, level) \
+__maybe_unused \
+static level##_t * asi_##level##_alloc(struct asi *asi,
\
+ base##_t *base, ulong addr, \
+ gfp_t flags) \
+{ \
+ if (unlikely(base##_none(*base))) { \
+ ulong pgtbl = get_zeroed_page(flags); \
+ phys_addr_t pgtbl_pa; \
+ \
+ if (!pgtbl) \
+ return NULL; \
+ \
+ pgtbl_pa = __pa(pgtbl); \
+ \
+ if (cmpxchg((ulong *)base, 0, \
+ pgtbl_pa | _PAGE_TABLE) != 0) { \
+ free_page(pgtbl); \
+ goto out; \
+ } \
+ \
+ mm_inc_nr_##level##s(asi->mm); \
+ } \
+out: \
+ VM_BUG_ON(base##_leaf(*base)); \
+ return level##_offset(base, addr); \
+}
+
+DEFINE_ASI_PGTBL_ALLOC(pgd, p4d)
+DEFINE_ASI_PGTBL_ALLOC(p4d, pud)
+DEFINE_ASI_PGTBL_ALLOC(pud, pmd)
+DEFINE_ASI_PGTBL_ALLOC(pmd, pte)
+
void __init asi_check_boottime_disable(void)
{
bool enabled =
IS_ENABLED(CONFIG_MITIGATION_ADDRESS_SPACE_ISOLATION_DEFAULT_ON);
--
2.47.1.613.gc27f4b7a9f-goog
