While theoretically we could still use the same call gate as for
32-bit userspace, it doesn't seem very common, and gcc seems to not
encode properly the instruction. Instead we use syscall/sysret as
other kernels (e.g. XNU,Linux). This version still has some
limitations, but should be enough to start working on the 64-bit user
space.
* i386/i386/i386asm.sym: add more constants to fill pcb->iss
* i386/i386/ldt.c: configure 64-bit syscall entry point. We can just
check for the SEP bit as MSR are always available on x86_64.
* i386/i386/ldt.h: swap CS/DS segments order if !USER32 as required by
sysret
* i386/i386/locore.h: add syscall64 and MSR definitions
* i386/include/mach/i386/syscall_sw.h: remove old BSD_TRAP
* x86_64/Makefrag.am: install syscall_sw.h depending on USER32
* x86_64/include/syscall_sw.h: add entry point from user space
* x86_64/locore.S: implement syscall64 entry point
---
i386/i386/i386asm.sym | 11 +++
i386/i386/ldt.c | 15 ++-
i386/i386/ldt.h | 9 +-
i386/i386/locore.h | 29 ++++++
i386/include/mach/i386/syscall_sw.h | 12 +--
x86_64/Makefrag.am | 7 +-
x86_64/include/syscall_sw.h | 40 ++++++++
x86_64/locore.S | 136 ++++++++++++++++++++++++++++
8 files changed, 244 insertions(+), 15 deletions(-)
create mode 100644 x86_64/include/syscall_sw.h
diff --git a/i386/i386/i386asm.sym b/i386/i386/i386asm.sym
index 8317db6c..733cc4eb 100644
--- a/i386/i386/i386asm.sym
+++ b/i386/i386/i386asm.sym
@@ -52,6 +52,8 @@ expr CALL_SINGLE_FUNCTION_BASE
offset ApicLocalUnit lu apic_id APIC_ID
+offset pcb pcb iss
+
offset thread th pcb
offset thread th task
offset thread th recover
@@ -82,9 +84,15 @@ size i386_kernel_state iks
size i386_exception_link iel
+offset i386_saved_state r gs
+offset i386_saved_state r fs
offset i386_saved_state r cs
offset i386_saved_state r uesp
offset i386_saved_state r eax
+offset i386_saved_state r ebx
+offset i386_saved_state r ecx
+offset i386_saved_state r edx
+offset i386_saved_state r ebp
offset i386_saved_state r trapno
offset i386_saved_state r err
offset i386_saved_state r efl R_EFLAGS
@@ -92,6 +100,9 @@ offset i386_saved_state r eip
offset i386_saved_state r cr2
offset i386_saved_state r edi
#ifdef __x86_64__
+offset i386_saved_state r r12
+offset i386_saved_state r r13
+offset i386_saved_state r r14
offset i386_saved_state r r15
#endif
diff --git a/i386/i386/ldt.c b/i386/i386/ldt.c
index b86a0e3c..8b7add38 100644
--- a/i386/i386/ldt.c
+++ b/i386/i386/ldt.c
@@ -31,6 +31,7 @@
#include <mach/xen.h>
#include <intel/pmap.h>
+#include <kern/debug.h>
#include "vm_param.h"
#include "seg.h"
@@ -65,10 +66,22 @@ ldt_fill(struct real_descriptor *myldt, struct
real_descriptor *mygdt)
ACC_PL_K|ACC_LDT, 0);
#endif /* MACH_PV_DESCRIPTORS */
- /* Initialize the 32bit LDT descriptors. */
+ /* Initialize the syscall entry point */
+#if defined(__x86_64__) && ! defined(USER32)
+ if (!CPU_HAS_FEATURE(CPU_FEATURE_SEP))
+ panic("syscall support is missing on 64 bit");
+ /* Enable 64-bit syscalls */
+ wrmsr(MSR_REG_EFER, rdmsr(MSR_REG_EFER) | MSR_EFER_SCE);
+ wrmsr(MSR_REG_LSTAR, (vm_offset_t)syscall64);
+ wrmsr(MSR_REG_STAR, ((((long)USER_CS - 16) << 16) | (long)KERNEL_CS)
<< 32);
+ wrmsr(MSR_REG_FMASK, 0); // ?
+#else /* defined(__x86_64__) && ! defined(USER32) */
fill_ldt_gate(myldt, USER_SCALL,
(vm_offset_t)&syscall, KERNEL_CS,
ACC_PL_U|ACC_CALL_GATE, 0);
+#endif /* defined(__x86_64__) && ! defined(USER32) */
+
+ /* Initialize the 32bit LDT descriptors. */
fill_ldt_descriptor(myldt, USER_CS,
VM_MIN_USER_ADDRESS,
VM_MAX_USER_ADDRESS-VM_MIN_USER_ADDRESS-4096,
diff --git a/i386/i386/ldt.h b/i386/i386/ldt.h
index b15f11a5..51867f47 100644
--- a/i386/i386/ldt.h
+++ b/i386/i386/ldt.h
@@ -43,11 +43,16 @@
* User descriptors for Mach - 32-bit flat address space
*/
#define USER_SCALL 0x07 /* system call gate */
-#ifdef __x86_64__
+#if defined(__x86_64__) && ! defined(USER32)
/* Call gate needs two entries */
-#endif
+
+/* The sysret instruction puts some constraints on the user segment indexes */
+#define USER_CS 0x1f /* user code segment */
+#define USER_DS 0x17 /* user data segment */
+#else
#define USER_CS 0x17 /* user code segment */
#define USER_DS 0x1f /* user data segment */
+#endif
#define LDTSZ 4
diff --git a/i386/i386/locore.h b/i386/i386/locore.h
index a8807dbf..74aca25d 100644
--- a/i386/i386/locore.h
+++ b/i386/i386/locore.h
@@ -57,6 +57,7 @@ extern int inst_fetch (int eip, int cs);
extern void cpu_shutdown (void);
extern int syscall (void);
+extern int syscall64 (void);
extern unsigned int cpu_features[2];
@@ -93,5 +94,33 @@ extern unsigned int cpu_features[2];
#define CPU_HAS_FEATURE(feature) (cpu_features[(feature) / 32] & (1 <<
((feature) % 32)))
+#define MSR_REG_EFER 0xC0000080
+#define MSR_REG_STAR 0xC0000081
+#define MSR_REG_LSTAR 0xC0000082
+#define MSR_REG_CSTAR 0xC0000083
+#define MSR_REG_FMASK 0xC0000084
+
+#define MSR_EFER_SCE 0x00000001
+
+static inline void wrmsr(uint32_t regaddr, uint64_t value)
+{
+ uint32_t low = (uint32_t) value, high = ((uint32_t) (value >> 32));
+ asm volatile("wrmsr"
+ :
+ : "c" (regaddr), "a" (low), "d" (high)
+ : "memory" /* wrmsr is a serializing instruction */
+ );
+}
+
+static inline uint64_t rdmsr(uint32_t regaddr)
+{
+ uint32_t low, high;
+ asm volatile("rdmsr"
+ : "=a" (low), "=d" (high)
+ : "c" (regaddr)
+ );
+ return ((uint64_t)high << 32) | low;
+}
+
#endif /* _MACHINE__LOCORE_H_ */
diff --git a/i386/include/mach/i386/syscall_sw.h
b/i386/include/mach/i386/syscall_sw.h
index 86f6ff2f..9eeb2939 100644
--- a/i386/include/mach/i386/syscall_sw.h
+++ b/i386/include/mach/i386/syscall_sw.h
@@ -29,21 +29,11 @@
#include <mach/machine/asm.h>
-#if BSD_TRAP
-#define kernel_trap(trap_name,trap_number,number_args) \
+#define kernel_trap(trap_name,trap_number,number_args) \
ENTRY(trap_name) \
movl $ trap_number,%eax; \
SVC; \
- jb LCL(cerror); \
ret; \
END(trap_name)
-#else
-#define kernel_trap(trap_name,trap_number,number_args) \
-ENTRY(trap_name) \
- movl $ trap_number,%eax; \
- SVC; \
- ret; \
-END(trap_name)
-#endif
#endif /* _MACH_I386_SYSCALL_SW_H_ */
diff --git a/x86_64/Makefrag.am b/x86_64/Makefrag.am
index d3735890..fb225aa5 100644
--- a/x86_64/Makefrag.am
+++ b/x86_64/Makefrag.am
@@ -175,11 +175,16 @@ include_mach_x86_64_HEADERS = \
i386/include/mach/i386/mach_i386_types.h \
i386/include/mach/i386/machine_types.defs \
i386/include/mach/i386/multiboot.h \
- i386/include/mach/i386/syscall_sw.h \
i386/include/mach/i386/thread_status.h \
i386/include/mach/i386/trap.h \
i386/include/mach/i386/vm_param.h \
i386/include/mach/i386/vm_types.h
+
+if enable_user32
+include_mach_x86_64_HEADERS += i386/include/mach/i386/syscall_sw.h
+else
+include_mach_x86_64_HEADERS += x86_64/include/syscall_sw.h
+endif
#
# Platform specific parts.
diff --git a/x86_64/include/syscall_sw.h b/x86_64/include/syscall_sw.h
new file mode 100644
index 00000000..4e03f28c
--- /dev/null
+++ b/x86_64/include/syscall_sw.h
@@ -0,0 +1,40 @@
+/*
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
+ * All Rights Reserved.
+ *
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ *
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ *
+ * Carnegie Mellon requests users of this software to return to
+ *
+ * Software Distribution Coordinator or software.distribut...@cs.cmu.edu
+ * School of Computer Science
+ * Carnegie Mellon University
+ * Pittsburgh PA 15213-3890
+ *
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#ifndef _MACH_X86_64_SYSCALL_SW_H_
+#define _MACH_X86_64_SYSCALL_SW_H_
+
+#include <mach/machine/asm.h>
+
+#define kernel_trap(trap_name,trap_number,number_args) \
+ENTRY(trap_name) \
+ movq $ trap_number,%rax; \
+ movq %rcx,%r10; \
+ syscall; \
+ ret; \
+END(trap_name)
+
+#endif /* _MACH_X86_64_SYSCALL_SW_H_ */
diff --git a/x86_64/locore.S b/x86_64/locore.S
index 47d9085c..564fd2ed 100644
--- a/x86_64/locore.S
+++ b/x86_64/locore.S
@@ -1281,6 +1281,142 @@ DATA(cpu_features_ecx)
END(syscall)
+
+/* Entry point for 64-bit syscalls.
+ * On entry we're still on the user stack, so better not use it. Instead we
+ * save the thread state immediately in thread->pcb->iss, then try to invoke
+ * the syscall.
+ * Note: emulated syscalls seem to not be used anymore in GNU/Hurd, so they
+ * are not handled here.
+ * TODO:
+ - for now we assume the return address is canonical, but apparently there
+ can be cases where it's not (see how Linux handles this). Does it apply
+ here?
+ - do we need to check for ast on syscalls? Maybe on interrupts is enough
+ - check that the case where a task is suspended, and later returns via
+ iretq from return_from_trap, works fine in all combinations
+ */
+ENTRY(syscall64)
+ /* RFLAGS[32:63] are reserved, so combine syscall num (32 bit) and
+ * eflags in RAX to allow using r11 as temporary register */
+ shlq $32,%r11
+ shlq $32,%rax /* make sure bits 32:63 of %rax are zero */
+ shrq $32,%rax
+ or %r11,%rax
+
+ /* Save thread state in pcb->iss, as on exception entry.
+ * Since this is triggered synchronously from userspace, we can
+ * save only the callee-preserved status according to the C ABI,
+ * plus RIP and EFLAGS for sysret */
+ CPU_NUMBER(%r11)
+ movq CX(EXT(active_threads),%r11),%r11 /* point to current thread */
+ movq TH_PCB(%r11),%r11 /* point to pcb */
+ addq $ PCB_ISS,%r11 /* point to saved state */
+
+ mov %gs,R_GS(%r11)
+ mov %fs,R_FS(%r11)
+ mov %rsp,R_UESP(%r11) /* callee-preserved register */
+ mov %rcx,R_EIP(%r11) /* syscall places user RIP in RCX */
+ mov %rbx,R_EBX(%r11) /* callee-preserved register */
+ mov %rax,%rbx /* Now we can unpack eflags again */
+ shr $32,%rbx
+ mov %rbx,R_EFLAGS(%r11) /* ... and save them in pcb as well */
+ mov %rbp,R_EBP(%r11) /* callee-preserved register */
+ mov %r12,R_R12(%r11) /* callee-preserved register */
+ mov %r13,R_R13(%r11) /* callee-preserved register */
+ mov %r14,R_R14(%r11) /* callee-preserved register */
+ mov %r15,R_R15(%r11) /* callee-preserved register */
+ mov %r11,%rbx /* prepare for error handling */
+ mov %r10,%rcx /* fix arg3 location according to C ABI
*/
+
+ /* switch to kernel stack */
+ CPU_NUMBER(%r11)
+ movq CX(EXT(kernel_stack),%r11),%rsp
+
+ /* Now we have saved state and args 1-6 are in place.
+ * Before invoking the syscall we do some bound checking and,
+ * if we have more that 6 arguments, we need to copy the
+ * remaining ones to the kernel stack, handling page faults when
+ * accessing the user stack.
+ */
+ shlq $32,%rax /* make sure bits 32:63 of %rax are
zero */
+ shrq $32,%rax
+ negl %eax /* get system call number */
+ jl _syscall64_range /* out of range if it was positive */
+ cmpl EXT(mach_trap_count),%eax /* check system call table
bounds */
+ jg _syscall64_range /* error if out of range */
+ shll $5,%eax /* manual indexing of mach_trap_t */
+
+ /* check if we need to place some arguments on the stack */
+_syscall64_args_stack:
+ mov EXT(mach_trap_table)(%rax),%r10 /* get number of arguments */
+ subq $6,%r10 /* the first 6 args are already in
place */
+ jle _syscall64_call /* skip argument copy if >6 args */
+
+ movq R_UESP(%rbx),%r11 /* get user stack pointer */
+ addq $8,%r11 /* Skip user return address */
+
+ mov $USER_DS,%r12 /* use user data segment for accesses */
+ mov %r12,%fs
+
+ lea (%r11,%r10,8),%r11 /* point past last argument */
+
+0: subq $8,%r11
+ RECOVER(_syscall64_addr_push)
+ mov %fs:(%r11),%r12
+ pushq %r12 /* push argument on stack */
+ dec %r10
+ jnz 0b /* loop for all remaining arguments */
+
+_syscall64_call:
+ call *EXT(mach_trap_table)+8(%rax) /* call procedure */
+ // XXX: check ast on exit?
+
+ /* avoid leaking information in callee-clobbered registers */
+ xorq $0,%rdi
+ xorq $0,%rsi
+ xorq $0,%rdx
+ xorq $0,%r10
+ xorq $0,%r9
+ xorq $0,%r8
+
+ /* restore thread state and return to user using sysret */
+ CPU_NUMBER(%r11)
+ movq CX(EXT(active_threads),%r11),%r11 /* point to current thread */
+ movq TH_PCB(%r11),%r11 /* point to pcb */
+ addq $ PCB_ISS,%r11 /* point to saved state */
+
+ mov R_GS(%r11),%gs
+ mov R_FS(%r11),%fs
+ mov R_UESP(%r11),%rsp /* callee-preserved register,
+ * switch to user stack */
+ mov R_EIP(%r11),%rcx /* sysret convention */
+ mov R_EBX(%r11),%rbx /* callee-preserved register */
+ mov R_EBP(%r11),%rbp /* callee-preserved register */
+ mov R_R12(%r11),%r12 /* callee-preserved register */
+ mov R_R13(%r11),%r13 /* callee-preserved register */
+ mov R_R14(%r11),%r14 /* callee-preserved register */
+ mov R_R15(%r11),%r15 /* callee-preserved register */
+ mov R_EFLAGS(%r11),%r11 /* sysret convention */
+
+ sysretq /* fast return to user-space, the thread didn't block */
+
+/* Error handling fragments, from here we jump directly to the trap handler */
+_syscall64_addr_push:
+ movq %rbx,%rsp /* clean parameters from stack */
+ movq %r11,R_CR2(%rbx) /* set fault address */
+ movq $(T_PAGE_FAULT),R_TRAPNO(%rbx) /* set page-fault trap */
+ movq $(T_PF_USER),R_ERR(%rbx) /* set error code - read user space */
+ jmp _take_trap /* treat as a trap */
+
+_syscall64_range:
+ movq $(T_INVALID_OPCODE),R_TRAPNO(%rbx)
+ /* set invalid-operation trap */
+ movq $0,R_ERR(%rbx) /* clear error code */
+ jmp _take_trap /* treat as a trap */
+
+END(syscall64)
+
/* Discover what kind of cpu we have; return the family number
(3, 4, 5, 6, for 386, 486, 586, 686 respectively). */
ENTRY(discover_x86_cpu_type)
--
2.30.2
