Replace memzero_expclit with our custom wrapper that additionally flushes the target address ranges from CPU caches.
On ARM64, while memzero_explicit is already reasonably reliable for wiping secrets from memory during kdump, it can theoretically leave residue in DRAM when the last memzero_explicit writes are still in flight in caches when shutting down the caches in machine_kexec. We need to swap every memzero_explicit call because ARM64 do not have a wholesale "flush all caches" primitive and only support flushes targeted to a particular address range. Architectures other than ARM64, notably x86-64, are not affected by this cache flush issue. Signed-off-by: Jan Sebastian Götte <[email protected]> --- arch/arm64/kernel/machine_kexec.c | 20 ++++++++++++++++++++ include/linux/crash_core.h | 21 +++++++++++++++++++++ kernel/crash_core.c | 23 +++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index c5693a32e49b..12bf3c90f519 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -6,6 +6,7 @@ * Copyright (C) Huawei Futurewei Technologies. */ +#include <linux/crash_core.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/kernel.h> @@ -15,6 +16,7 @@ #include <linux/set_memory.h> #include <linux/smp.h> +#include <asm/barrier.h> #include <asm/cacheflush.h> #include <asm/cpu_ops.h> #include <asm/daifflags.h> @@ -221,6 +223,24 @@ void machine_crash_shutdown(struct pt_regs *regs) pr_info("Starting crashdump kernel...\n"); } +#ifdef CONFIG_CRASH_WIPE_SECRETS +/* + * Queue each wiped range for cleaning to the point of coherency. Deferring + * the barrier until the notifier chain is complete avoids one DSB per range. + */ +void arch_crash_wipe_range(void *addr, size_t size) +{ + unsigned long start = (unsigned long)addr; + + dcache_clean_poc_nosync(start, start + size); +} + +void arch_crash_wipe_flush(void) +{ + dsb(sy); +} +#endif + #if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_HIBERNATION) /* * To preserve the crash dump kernel image, the relevant memory segments diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index 4230463f3faa..d3e4192b7e6d 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -6,6 +6,7 @@ #include <linux/elfcore.h> #include <linux/elf.h> #include <linux/notifier.h> +#include <linux/string.h> struct kimage; @@ -15,6 +16,26 @@ struct crash_mem { struct range ranges[] __counted_by(max_nr_ranges); }; +#ifdef CONFIG_CRASH_WIPE_SECRETS +/* + * Record a range that has already been wiped, or wipe and record it in one + * operation. Architectures may use the ranges to push the wipes out to + * memory before kexec disables the caches. + */ +void crash_wipe_cache_range(void *addr, size_t size); +void crash_wipe_memzero(void *addr, size_t size); + +void arch_crash_wipe_range(void *addr, size_t size); +void arch_crash_wipe_flush(void); +#else +static inline void crash_wipe_cache_range(void *addr, size_t size) { } + +static inline void crash_wipe_memzero(void *addr, size_t size) +{ + memzero_explicit(addr, size); +} +#endif + #ifdef CONFIG_CRASH_DUMP int crash_shrink_memory(unsigned long new_size); diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 95f5c0415e60..42faf8d0a4c4 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -51,10 +51,33 @@ int crash_wipe_secrets_unregister(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(crash_wipe_secrets_unregister); +/* + * Some kexec paths disable the data cache without first cleaning it. Give + * architectures valid virtual ranges for the wiped data, then let them defer + * any completion barrier until all crash-wipe callbacks have run. + */ +void __weak arch_crash_wipe_range(void *addr, size_t size) { } +void __weak arch_crash_wipe_flush(void) { } + +void crash_wipe_cache_range(void *addr, size_t size) +{ + if (size) + arch_crash_wipe_range(addr, size); +} +EXPORT_SYMBOL_GPL(crash_wipe_cache_range); + +void crash_wipe_memzero(void *addr, size_t size) +{ + memzero_explicit(addr, size); + crash_wipe_cache_range(addr, size); +} +EXPORT_SYMBOL_GPL(crash_wipe_memzero); + static void crash_wipe_secrets(void) { pr_info("Wiping sensitive secrets...\n"); atomic_notifier_call_chain(&crash_wipe_secrets_notifier_list, 0, NULL); + arch_crash_wipe_flush(); pr_info("Done wiping secrets.\n"); } #else -- 2.53.0
