mmap() returns MAP_FAILED, which is defined as (void *)-1, on error, not NULL. Several selftests incorrectly check the return value of mmap() using !ptr or ptr == NULL, which would erroneously treat MAP_FAILED as a valid pointer since MAP_FAILED is non-zero and non-NULL. This can lead to segfaults when mmap() actually fails under memory pressure.
Signed-off-by: Hongfu Li <[email protected]> Reviewed-by: Dev Jain <[email protected]> --- v2: - Add missing mmap() return value checks in pkey_sighandler_tests.c and protection_keys.c --- tools/testing/selftests/mm/ksm_tests.c | 2 +- tools/testing/selftests/mm/madv_populate.c | 2 +- tools/testing/selftests/mm/pkey_sighandler_tests.c | 2 ++ tools/testing/selftests/mm/protection_keys.c | 1 + tools/testing/selftests/mm/soft-dirty.c | 4 ++-- tools/testing/selftests/mm/vm_util.c | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a0b48b839d54..ed481f817282 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -174,7 +174,7 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_ { void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); - if (!map_ptr) { + if (map_ptr == MAP_FAILED) { perror("mmap"); return NULL; } diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c index 88050e0f829a..7fce5d0b622b 100644 --- a/tools/testing/selftests/mm/madv_populate.c +++ b/tools/testing/selftests/mm/madv_populate.c @@ -34,7 +34,7 @@ static void sense_support(void) addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (!addr) + if (addr == MAP_FAILED) ksft_exit_fail_msg("mmap failed\n"); ret = madvise(addr, pagesize, MADV_POPULATE_READ); diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 302fef54049c..4637809192f9 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -317,6 +317,7 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; @@ -490,6 +491,7 @@ static void test_pkru_sigreturn(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index 2085982dba69..580bf1668c71 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -1217,6 +1217,7 @@ static void arch_force_pkey_reg_init(void) * doing the XSAVE size enumeration dance. */ buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + pkey_assert(buf != (void *)-1); /* These __builtins require compiling with -mxsave */ diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index bcfcac99b436..67c26c265880 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) if (anon) { map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - if (!map) + if (map == MAP_FAILED) ksft_exit_fail_msg("anon mmap failed\n"); } else { test_fd = open(fname, O_RDWR | O_CREAT, 0664); @@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) ftruncate(test_fd, pagesize); map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_SHARED, test_fd, 0); - if (!map) + if (map == MAP_FAILED) ksft_exit_fail_msg("file mmap failed\n"); } diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index db94564f4431..63aaa2d9ec0b 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -463,7 +463,7 @@ bool softdirty_supported(void) /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */ addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (!addr) + if (addr == MAP_FAILED) ksft_exit_fail_msg("mmap failed\n"); supported = check_vmflag(addr, "sd"); -- 2.25.1
