Hi folks, I had this unsubmitted patch in my local filesystem. It makes Linux detect ELF32 AMD64 binaries and sets a flag to restrict them to 32-bit address space.
It's not rocket science but can save you some work in case you haven't implemented this already. Best regards -- Robert Millan
diff -Nur linux-2.6-2.6.26-libre2.old/arch/x86/kernel/sys_x86_64.c linux-2.6-2.6.26-libre2/arch/x86/kernel/sys_x86_64.c --- linux-2.6-2.6.26-libre2.old/arch/x86/kernel/sys_x86_64.c 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/arch/x86/kernel/sys_x86_64.c 2009-05-29 22:57:41.000000000 +0200 @@ -48,7 +48,7 @@ static void find_start_end(unsigned long flags, unsigned long *begin, unsigned long *end) { - if (!test_thread_flag(TIF_IA32) && (flags & MAP_32BIT)) { + if ((!test_thread_flag(TIF_IA32) && (flags & MAP_32BIT)) || test_thread_flag(TIF_AMD32)) { unsigned long new_begin; /* This is usually used needed to map code in small model, so it needs to be in the first 31bit. Limit @@ -94,7 +94,7 @@ (!vma || addr + len <= vma->vm_start)) return addr; } - if (((flags & MAP_32BIT) || test_thread_flag(TIF_IA32)) + if (((flags & MAP_32BIT) || test_thread_flag(TIF_IA32) || test_thread_flag(TIF_AMD32)) && len <= mm->cached_hole_size) { mm->cached_hole_size = 0; mm->free_area_cache = begin; @@ -150,8 +150,8 @@ if (flags & MAP_FIXED) return addr; - /* for MAP_32BIT mappings we force the legact mmap base */ - if (!test_thread_flag(TIF_IA32) && (flags & MAP_32BIT)) + /* for MAP_32BIT mappings we force the legacy mmap base */ + if ((!test_thread_flag(TIF_IA32) && (flags & MAP_32BIT)) || test_thread_flag(TIF_AMD32)) goto bottomup; /* requesting a specific address */ @@ -232,5 +232,7 @@ up_read(&uts_sem); if (personality(current->personality) == PER_LINUX32) err |= copy_to_user(&name->machine, "i686", 5); + else if (test_thread_flag(TIF_AMD32)) + err |= copy_to_user(&name->machine, "amd32", 6); return err ? -EFAULT : 0; } diff -Nur linux-2.6-2.6.26-libre2.old/arch/x86/mm/mmap.c linux-2.6-2.6.26-libre2/arch/x86/mm/mmap.c --- linux-2.6-2.6.26-libre2.old/arch/x86/mm/mmap.c 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/arch/x86/mm/mmap.c 2009-05-26 14:30:53.000000000 +0200 @@ -53,6 +53,15 @@ return 0; } +static int mmap_is_32bit(void) +{ + if (mmap_is_ia32 ()) + return 1; + if (test_thread_flag(TIF_AMD32)) + return 1; + return 0; +} + static int mmap_is_legacy(void) { if (current->personality & ADDR_COMPAT_LAYOUT) @@ -73,7 +82,7 @@ * 28 bits of randomness in 64bit mmaps, 40 address space bits */ if (current->flags & PF_RANDOMIZE) { - if (mmap_is_ia32()) + if (mmap_is_32bit()) rnd = (long)get_random_int() % (1<<8); else rnd = (long)(get_random_int() % (1<<28)); diff -Nur linux-2.6-2.6.26-libre2.old/fs/binfmt_elf_amd32.c linux-2.6-2.6.26-libre2/fs/binfmt_elf_amd32.c --- linux-2.6-2.6.26-libre2.old/fs/binfmt_elf_amd32.c 1970-01-01 01:00:00.000000000 +0100 +++ linux-2.6-2.6.26-libre2/fs/binfmt_elf_amd32.c 2009-05-26 14:26:24.000000000 +0200 @@ -0,0 +1,46 @@ +/* + * Support for loading AMD32 binaries + * Copyright (C) 2009 Robert Millan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/elfcore-compat.h> + +#undef ELF_CLASS +#define ELF_CLASS ELFCLASS32 + +#undef elfhdr +#define elfhdr elf32_hdr +#undef elf_phdr +#define elf_phdr elf32_phdr +#undef elf_note +#define elf_note elf32_note +#undef elf_addr_t +#define elf_addr_t Elf32_Addr + +#undef ELF_PLATFORM +#define ELF_PLATFORM ("amd32") + +#undef elf_check_arch +#define elf_check_arch(x) ((x)->e_machine == EM_X86_64 && (x)->e_ident[EI_CLASS] == ELFCLASS32) + +#undef SET_PERSONALITY +#define SET_PERSONALITY(ex, ibcs2) do { set_personality_64bit(); set_thread_flag(TIF_AMD32); current->personality |= force_personality32; } while (0) + +/* Use the generic 32-bit definition from asm/elf.h */ +#undef ELF_ET_DYN_BASE +#define ELF_ET_DYN_BASE COMPAT_ELF_ET_DYN_BASE + +#include "binfmt_elf.c" diff -Nur linux-2.6-2.6.26-libre2.old/fs/Kconfig.binfmt linux-2.6-2.6.26-libre2/fs/Kconfig.binfmt --- linux-2.6-2.6.26-libre2.old/fs/Kconfig.binfmt 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/fs/Kconfig.binfmt 2009-05-25 00:26:10.000000000 +0200 @@ -23,6 +23,13 @@ ld.so (check the file <file:Documentation/Changes> for location and latest version). +config BINFMT_ELF_AMD32 + bool "Kernel support for AMD32 binaries" + depends on X86_64 + default y + help + Support for ELF32 binaries on AMD x86-64. + config COMPAT_BINFMT_ELF bool depends on COMPAT && MMU diff -Nur linux-2.6-2.6.26-libre2.old/fs/Makefile linux-2.6-2.6.26-libre2/fs/Makefile --- linux-2.6-2.6.26-libre2.old/fs/Makefile 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/fs/Makefile 2009-05-25 00:24:37.000000000 +0200 @@ -39,6 +39,7 @@ obj-y += binfmt_script.o obj-$(CONFIG_BINFMT_ELF) += binfmt_elf.o +obj-$(CONFIG_BINFMT_ELF_AMD32) += binfmt_elf_amd32.o obj-$(CONFIG_COMPAT_BINFMT_ELF) += compat_binfmt_elf.o obj-$(CONFIG_BINFMT_ELF_FDPIC) += binfmt_elf_fdpic.o obj-$(CONFIG_BINFMT_SOM) += binfmt_som.o diff -Nur linux-2.6-2.6.26-libre2.old/include/asm-x86/elf.h linux-2.6-2.6.26-libre2/include/asm-x86/elf.h --- linux-2.6-2.6.26-libre2.old/include/asm-x86/elf.h 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/include/asm-x86/elf.h 2009-05-26 14:03:51.000000000 +0200 @@ -298,7 +298,7 @@ #define VDSO_HIGH_BASE 0xffffe000U /* CONFIG_COMPAT_VDSO address */ /* 1GB for 64bit, 8MB for 32bit */ -#define STACK_RND_MASK (test_thread_flag(TIF_IA32) ? 0x7ff : 0x3fffff) +#define STACK_RND_MASK ((test_thread_flag(TIF_IA32) || test_thread_flag(TIF_AMD32)) ? 0x7ff : 0x3fffff) #define ARCH_DLINFO \ do { \ diff -Nur linux-2.6-2.6.26-libre2.old/include/asm-x86/processor.h linux-2.6-2.6.26-libre2/include/asm-x86/processor.h --- linux-2.6-2.6.26-libre2.old/include/asm-x86/processor.h 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/include/asm-x86/processor.h 2009-05-26 14:06:49.000000000 +0200 @@ -881,9 +881,9 @@ #define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \ 0xc0000000 : 0xFFFFe000) -#define TASK_SIZE (test_thread_flag(TIF_IA32) ? \ +#define TASK_SIZE ((test_thread_flag(TIF_IA32) || test_thread_flag(TIF_AMD32)) ? \ IA32_PAGE_OFFSET : TASK_SIZE64) -#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_IA32)) ? \ +#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_IA32) || test_tsk_thread_flag(child, TIF_AMD32)) ? \ IA32_PAGE_OFFSET : TASK_SIZE64) #define STACK_TOP TASK_SIZE diff -Nur linux-2.6-2.6.26-libre2.old/include/asm-x86/thread_info_64.h linux-2.6-2.6.26-libre2/include/asm-x86/thread_info_64.h --- linux-2.6-2.6.26-libre2.old/include/asm-x86/thread_info_64.h 2008-07-13 23:51:29.000000000 +0200 +++ linux-2.6-2.6.26-libre2/include/asm-x86/thread_info_64.h 2009-05-26 14:28:33.000000000 +0200 @@ -111,8 +111,8 @@ #define TIF_SECCOMP 8 /* secure computing */ #define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */ #define TIF_HRTICK_RESCHED 11 /* reprogram hrtick timer */ -/* 16 free */ -#define TIF_IA32 17 /* 32bit process */ +#define TIF_AMD32 16 /* 32bit AMD process */ +#define TIF_IA32 17 /* 32bit Intel process */ #define TIF_FORK 18 /* ret_from_fork */ #define TIF_ABI_PENDING 19 #define TIF_MEMDIE 20 @@ -134,6 +134,7 @@ #define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) #define _TIF_HRTICK_RESCHED (1 << TIF_HRTICK_RESCHED) +#define _TIF_AMD32 (1 << TIF_AMD32) #define _TIF_IA32 (1 << TIF_IA32) #define _TIF_FORK (1 << TIF_FORK) #define _TIF_ABI_PENDING (1 << TIF_ABI_PENDING)