This speeds up smp again, by storing the struct processor
in a percpu area and avoiding an expensive cpu_number every call
of current_processor(), as well as getting the cpu_number by
an offset into the percpu area. Untested on 64 bit
and work remains to use other percpu arrays.
TESTED: on -smp 4 boots to INIT
TESTED: uniprocessor version still compiles/boots
---
i386/Makefrag.am | 2 ++
i386/i386/cpu_number.c | 8 ++++-
i386/i386/cpu_number.h | 16 +++++++++
i386/i386/cpuboot.S | 2 +-
i386/i386/gdt.c | 9 ++++-
i386/i386/gdt.h | 11 +++---
i386/i386/i386asm.sym | 9 ++---
i386/i386/locore.S | 27 +++++++++------
i386/i386/mp_desc.c | 6 ++--
i386/i386/mp_desc.h | 4 +--
i386/i386/percpu.c | 30 ++++++++++++++++
i386/i386/percpu.h | 76 +++++++++++++++++++++++++++++++++++++++++
i386/i386at/model_dep.c | 1 +
kern/cpu_number.h | 3 +-
kern/processor.c | 7 ++--
kern/processor.h | 18 ++++------
kern/startup.c | 2 ++
x86_64/Makefrag.am | 2 ++
x86_64/locore.S | 10 +++---
19 files changed, 191 insertions(+), 52 deletions(-)
create mode 100644 i386/i386/percpu.c
create mode 100644 i386/i386/percpu.h
diff --git a/i386/Makefrag.am b/i386/Makefrag.am
index 274e8695..c1724cea 100644
--- a/i386/Makefrag.am
+++ b/i386/Makefrag.am
@@ -108,6 +108,8 @@ libkernel_a_SOURCES += \
i386/i386/irq.c \
i386/i386/irq.h \
i386/i386/msr.h \
+ i386/i386/percpu.c \
+ i386/i386/percpu.h \
i386/i386/pit.c \
i386/i386/pit.h
diff --git a/i386/i386/cpu_number.c b/i386/i386/cpu_number.c
index ef19e11f..241015b5 100644
--- a/i386/i386/cpu_number.c
+++ b/i386/i386/cpu_number.c
@@ -20,11 +20,17 @@
#include <i386/smp.h>
#include <i386/cpu.h>
#include <i386/mp_desc.h>
+#include <i386/percpu.h>
#include <kern/printf.h>
#if NCPUS > 1
-int cpu_number(void)
+int cpu_number_slow(void)
{
return cpu_id_lut[apic_get_current_cpu()];
}
+
+int cpu_number(void)
+{
+ return *((int *)percpu_ptr(int, cpu_id));
+}
#endif
diff --git a/i386/i386/cpu_number.h b/i386/i386/cpu_number.h
index 479a847a..c551b073 100644
--- a/i386/i386/cpu_number.h
+++ b/i386/i386/cpu_number.h
@@ -30,6 +30,9 @@
#ifndef _I386_CPU_NUMBER_H_
#define _I386_CPU_NUMBER_H_
+#define SMP_COMPLETE (-1)
+#define MY(stm) %gs:PERCPU_##stm
+
#if NCPUS > 1
#ifdef __i386__
@@ -63,14 +66,27 @@
movl %esi, reg ;\
popl %esi ;\
+/* Never call CPU_NUMBER_GS(%esi) */
+#define CPU_NUMBER_GS(reg) \
+ movl %cs:bspdone, reg ;\
+ cmpl $SMP_COMPLETE, reg ;\
+ je 8f ;\
+ CPU_NUMBER(reg) ;\
+ jmp 9f ;\
+8: ;\
+ movl MY(CPU_ID), reg ;\
+9:
+
#ifndef __ASSEMBLER__
#include "kern/cpu_number.h"
+int cpu_number_slow(void);
int cpu_number(void);
#endif
#else /* NCPUS == 1 */
#define CPU_NUMBER_NO_STACK(reg)
+#define CPU_NUMBER_GS(reg)
#define CPU_NUMBER(reg)
#define CX(addr,reg) addr
diff --git a/i386/i386/cpuboot.S b/i386/i386/cpuboot.S
index 4a5823be..7d1e815c 100644
--- a/i386/i386/cpuboot.S
+++ b/i386/i386/cpuboot.S
@@ -119,7 +119,7 @@ _apboot:
wrmsr
/* Load int_stack_top[cpu] -> esp */
- CPU_NUMBER(%edx)
+ CPU_NUMBER_NO_STACK(%edx)
movl CX(EXT(int_stack_top), %edx), %esp
/* Ensure stack alignment */
diff --git a/i386/i386/gdt.c b/i386/i386/gdt.c
index ddda603b..e335de50 100644
--- a/i386/i386/gdt.c
+++ b/i386/i386/gdt.c
@@ -35,6 +35,7 @@
#include <kern/assert.h>
#include <intel/pmap.h>
+#include <machine/percpu.h>
#include "vm_param.h"
#include "seg.h"
@@ -73,6 +74,11 @@ gdt_fill(struct real_descriptor *mygdt)
0xffffffff,
ACC_PL_K|ACC_DATA_W, SZ_32);
#endif /* MACH_PV_DESCRIPTORS */
+ vm_offset_t thiscpu = kvtolin(&percpu_array[cpu_number_slow()]);
+ _fill_gdt_descriptor(mygdt, PERCPU_DS,
+ thiscpu,
+ thiscpu + sizeof(struct percpu) - 1,
+ ACC_PL_K|ACC_DATA_W, SZ_32);
#endif
#ifdef MACH_PV_DESCRIPTORS
@@ -119,8 +125,9 @@ reload_segs(void)
"movw %w1,%%ds\n"
"movw %w1,%%es\n"
+ "movw %w3,%%gs\n"
"movw %w1,%%ss\n"
- : : "i" (KERNEL_CS), "r" (KERNEL_DS), "r" (0));
+ : : "i" (KERNEL_CS), "r" (KERNEL_DS), "r" (0), "r"
(PERCPU_DS));
#endif
}
diff --git a/i386/i386/gdt.h b/i386/i386/gdt.h
index 5def73cb..c7da012a 100644
--- a/i386/i386/gdt.h
+++ b/i386/i386/gdt.h
@@ -77,11 +77,11 @@
/* 0x58 used by user TSS in 64bit mode */
-#ifdef __x86_64__
-#define GDTSZ sel_idx(0x60)
-#else
-#define GDTSZ sel_idx(0x58)
-#endif
+#define PERCPU_DS 0x68 /* per-cpu data mapping */
+
+#define GDTSZ sel_idx(0x70)
+
+#ifndef __ASSEMBLER__
extern struct real_descriptor gdt[GDTSZ];
@@ -117,4 +117,5 @@ extern struct real_descriptor gdt[GDTSZ];
extern void gdt_init(void);
extern void ap_gdt_init(int cpu);
+#endif /* __ASSEMBLER__ */
#endif /* _I386_GDT_ */
diff --git a/i386/i386/i386asm.sym b/i386/i386/i386asm.sym
index 436e296a..620e8f37 100644
--- a/i386/i386/i386asm.sym
+++ b/i386/i386/i386asm.sym
@@ -53,6 +53,8 @@ expr CALL_PMAP_UPDATE
offset ApicLocalUnit lu apic_id APIC_ID
+offset percpu pc cpu_id PERCPU_CPU_ID
+
offset pcb pcb iss
offset thread th pcb
@@ -154,17 +156,10 @@ expr NPTES
PTES_PER_PAGE
expr INTEL_PTE_VALID|INTEL_PTE_WRITE INTEL_PTE_KERNEL
expr IDTSZ
-expr GDTSZ
-expr LDTSZ
expr KERNEL_RING
-
expr KERNEL_CS
expr KERNEL_DS
-expr KERNEL_TSS
-#ifndef MACH_PV_DESCRIPTORS
-expr KERNEL_LDT
-#endif /* MACH_PV_DESCRIPTORS */
expr (VM_MIN_KERNEL_ADDRESS>>PDESHIFT)*sizeof(pt_entry_t) KERNELBASEPDE
diff --git a/i386/i386/locore.S b/i386/i386/locore.S
index 55aa9d60..ff59615b 100644
--- a/i386/i386/locore.S
+++ b/i386/i386/locore.S
@@ -33,6 +33,7 @@
#include <i386/proc_reg.h>
#include <i386/trap.h>
#include <i386/seg.h>
+#include <i386/gdt.h>
#include <i386/ldt.h>
#include <i386/i386asm.h>
#include <i386/cpu_number.h>
@@ -468,7 +469,8 @@ trap_push_segs:
mov %ax,%ds /* (same as kernel stack segment) */
mov %ax,%es
mov %ax,%fs
- mov %ax,%gs
+ mov $(PERCPU_DS),%ax
+ movw %ax,%gs
trap_set_segs:
cld /* clear direction flag */
@@ -479,7 +481,7 @@ trap_set_segs:
jz trap_from_kernel /* kernel trap if not */
trap_from_user:
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
TIME_TRAP_UENTRY
movl CX(EXT(kernel_stack),%edx),%ebx
@@ -501,7 +503,7 @@ _take_trap:
*/
_return_from_trap:
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
cmpl $0,CX(EXT(need_ast),%edx)
jz _return_to_user /* if we need an AST: */
@@ -686,9 +688,10 @@ ENTRY(all_intrs)
mov %dx,%ds
mov %dx,%es
mov %dx,%fs
- mov %dx,%gs
+ mov $(PERCPU_DS),%dx
+ movw %dx,%gs
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
movl CX(EXT(int_stack_top),%edx),%ecx
@@ -704,7 +707,7 @@ ENTRY(all_intrs)
#endif
#ifdef MACH_LDEBUG
- CPU_NUMBER(%ecx)
+ CPU_NUMBER_GS(%ecx)
incl CX(EXT(in_interrupt),%ecx)
#endif
@@ -712,7 +715,7 @@ ENTRY(all_intrs)
.globl EXT(return_to_iret) /* ( label for kdb_kintr and hardclock
*/
LEXT(return_to_iret) /* to find the return from calling
interrupt) */
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
#ifdef MACH_LDEBUG
decl CX(EXT(in_interrupt),%edx)
#endif
@@ -792,9 +795,10 @@ ast_from_interrupt:
mov %dx,%ds
mov %dx,%es
mov %dx,%fs
- mov %dx,%gs
+ mov $(PERCPU_DS),%dx
+ movw %dx,%gs
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
TIME_TRAP_UENTRY
movl CX(EXT(kernel_stack),%edx),%esp
@@ -1051,7 +1055,8 @@ syscall_entry_2:
mov %dx,%ds
mov %dx,%es
mov %dx,%fs
- mov %dx,%gs
+ mov $(PERCPU_DS),%dx
+ movw %dx,%gs
/*
* Shuffle eflags,eip,cs into proper places
@@ -1064,7 +1069,7 @@ syscall_entry_2:
movl %edx,R_CS(%esp) /* fix cs */
movl %ebx,R_EFLAGS(%esp) /* fix eflags */
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
TIME_TRAP_SENTRY
movl CX(EXT(kernel_stack),%edx),%ebx
diff --git a/i386/i386/mp_desc.c b/i386/i386/mp_desc.c
index f1a1f989..0f9bee3b 100644
--- a/i386/i386/mp_desc.c
+++ b/i386/i386/mp_desc.c
@@ -93,12 +93,13 @@ interrupt_stack_alloc(void)
}
}
-#if NCPUS > 1
/*
* Flag to mark SMP init by BSP complete
*/
int bspdone;
+#if NCPUS > 1
+
extern void *apboot, *apbootend;
extern volatile ApicLocalUnit* lapic;
@@ -238,6 +239,7 @@ cpu_setup(int cpu)
flush_instr_queue();
printf("AP=(%u) paging done\n", cpu);
+ init_percpu(cpu);
mp_desc_init(cpu);
printf("AP=(%u) mpdesc done\n", cpu);
@@ -275,7 +277,7 @@ cpu_setup(int cpu)
void
cpu_ap_main()
{
- int cpu = cpu_number();
+ int cpu = cpu_number_slow();
do {
cpu_pause();
diff --git a/i386/i386/mp_desc.h b/i386/i386/mp_desc.h
index fea42cd3..9859468c 100644
--- a/i386/i386/mp_desc.h
+++ b/i386/i386/mp_desc.h
@@ -76,8 +76,6 @@ extern struct real_descriptor *mp_gdt[NCPUS];
extern uint8_t solid_intstack[];
-extern int bspdone;
-
/*
* Each CPU calls this routine to set up its descriptor tables.
*/
@@ -89,6 +87,8 @@ extern void interrupt_processor(int cpu);
#endif /* MULTIPROCESSOR */
+extern int bspdone;
+
extern void start_other_cpus(void);
extern kern_return_t cpu_start(int cpu);
diff --git a/i386/i386/percpu.c b/i386/i386/percpu.c
new file mode 100644
index 00000000..b2b8afa7
--- /dev/null
+++ b/i386/i386/percpu.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2023 Free Software Foundation, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <i386/smp.h>
+#include <i386/apic.h>
+#include <i386/percpu.h>
+
+struct percpu percpu_array[NCPUS] __aligned(0x8000) = {0};
+
+void init_percpu(int cpu)
+{
+ int apic_id = apic_get_current_cpu();
+
+ percpu_array[cpu].self = &percpu_array[cpu];
+ percpu_array[cpu].apic_id = apic_id;
+ percpu_array[cpu].cpu_id = cpu;
+}
diff --git a/i386/i386/percpu.h b/i386/i386/percpu.h
new file mode 100644
index 00000000..24690581
--- /dev/null
+++ b/i386/i386/percpu.h
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2023 Free Software Foundation, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _PERCPU_H_
+#define _PERCPU_H_
+
+#include <kern/ast.h>
+#include <kern/processor.h>
+#include <kern/thread.h>
+#include <kern/timer.h>
+#include <i386/mp_desc.h>
+#include <i386/spl.h>
+#include <i386/cpu_number.h>
+#include <intel/pmap.h>
+#include <ipc/ipc_kmsg.h>
+
+#define percpu_assign(stm, val) \
+ asm("mov %[src], %%gs:%c[offs]" \
+ : /* No outputs */ \
+ : [src] "r" (val), [offs] "e" (__builtin_offsetof(struct percpu,
stm)) \
+ : );
+
+#define percpu_ptr(typ, stm) \
+MACRO_BEGIN \
+ typ *ptr_ = (typ *)__builtin_offsetof(struct percpu, stm); \
+ \
+ asm("add %%gs:0, %[pointer]" \
+ : [pointer] "+r" (ptr_) \
+ : /* No inputs */ \
+ : ); \
+ \
+ ptr_; \
+MACRO_END
+
+struct percpu {
+ struct percpu *self;
+ int apic_id;
+ int cpu_id;
+ struct processor processor;
+/*
+ struct machine_slot machine_slot;
+ struct mp_desc_table mp_desc_table;
+ thread_t active_thread;
+ vm_offset_t active_stack;
+ vm_offset_t int_stack_top;
+ vm_offset_t int_stack_base;
+ ast_t need_ast;
+ ipc_kmsg_t ipc_kmsg_cache;
+ pmap_update_list cpu_update_list;
+ spl_t saved_ipl;
+ spl_t curr_ipl;
+ timer_data_t kernel_timer;
+ timer_t current_timer;
+ unsigned long in_interrupt;
+*/
+};
+
+extern struct percpu percpu_array[NCPUS];
+
+void init_percpu(int cpu);
+
+#endif /* _PERCPU_H_ */
diff --git a/i386/i386at/model_dep.c b/i386/i386at/model_dep.c
index f83214b1..97acfdd6 100644
--- a/i386/i386at/model_dep.c
+++ b/i386/i386at/model_dep.c
@@ -462,6 +462,7 @@ i386at_init(void)
ldt_init();
ktss_init();
+ init_percpu(0);
#if NCPUS > 1
/* Initialize SMP structures in the master processor */
mp_desc_init(0);
diff --git a/kern/cpu_number.h b/kern/cpu_number.h
index 0be2d338..1abe3dbb 100644
--- a/kern/cpu_number.h
+++ b/kern/cpu_number.h
@@ -37,7 +37,8 @@ extern int master_cpu; /* 'master' processor - keeps
time */
#if (NCPUS == 1)
/* cpu number is always 0 on a single processor system */
-#define cpu_number() (0)
+#define cpu_number() (0)
+#define cpu_number_slow() (0)
#endif /* NCPUS == 1 */
diff --git a/kern/processor.c b/kern/processor.c
index 2cd6d46c..76735381 100644
--- a/kern/processor.c
+++ b/kern/processor.c
@@ -60,14 +60,12 @@ struct kmem_cache pset_cache;
int master_cpu;
struct processor_set default_pset;
-struct processor processor_array[NCPUS];
queue_head_t all_psets;
int all_psets_count;
def_simple_lock_data(, all_psets_lock);
processor_t master_processor;
-processor_t processor_ptr[NCPUS];
/*
* Bootstrap the processor/pset system so the scheduler can run.
@@ -81,10 +79,9 @@ void pset_sys_bootstrap(void)
for (i = 0; i < NCPUS; i++) {
/*
* Initialize processor data structures.
- * Note that cpu_to_processor(i) is processor_ptr[i].
+ * Note that cpu_to_processor is processor_ptr.
*/
- processor_ptr[i] = &processor_array[i];
- processor_init(processor_ptr[i], i);
+ processor_init(processor_ptr(i), i);
}
master_processor = cpu_to_processor(master_cpu);
queue_init(&all_psets);
diff --git a/kern/processor.h b/kern/processor.h
index 17b784a3..d83cdf3c 100644
--- a/kern/processor.h
+++ b/kern/processor.h
@@ -112,6 +112,8 @@ struct processor {
typedef struct processor Processor;
extern struct processor processor_array[NCPUS];
+#include <machine/percpu.h>
+
/*
* Chain of all processor sets.
*/
@@ -195,23 +197,15 @@ extern processor_t master_processor;
#define PROCESSOR_ASSIGN 4 /* Assignment is changing */
#define PROCESSOR_SHUTDOWN 5 /* Being shutdown */
-/*
- * Use processor ptr array to find current processor's data structure.
- * This replaces a multiplication (index into processor_array) with
- * an array lookup and a memory reference. It also allows us to save
- * space if processor numbering gets too sparse.
- */
-
-extern processor_t processor_ptr[NCPUS];
-
-#define cpu_to_processor(i) (processor_ptr[i])
+#define processor_ptr(i) (&percpu_array[i].processor)
+#define cpu_to_processor processor_ptr
-#define current_processor() (processor_ptr[cpu_number()])
+#define current_processor() (percpu_ptr(struct processor, processor))
#define current_processor_set() (current_processor()->processor_set)
/* Compatibility -- will go away */
-#define cpu_state(slot_num) (processor_ptr[slot_num]->state)
+#define cpu_state(slot_num) (processor_ptr(slot_num)->state)
#define cpu_idle(slot_num) (cpu_state(slot_num) == PROCESSOR_IDLE)
/* Useful lock macros */
diff --git a/kern/startup.c b/kern/startup.c
index 2eb3a739..d97809f7 100644
--- a/kern/startup.c
+++ b/kern/startup.c
@@ -74,6 +74,7 @@ boolean_t reboot_on_panic = TRUE;
#if NCPUS > 1
#include <machine/mp_desc.h>
+#include <kern/smp.h>
#include <kern/machine.h>
#endif /* NCPUS > 1 */
@@ -265,6 +266,7 @@ void start_kernel_threads(void)
xprinit(); /* XXX */
#endif /* XPR_DEBUG */
+ bspdone = SMP_COMPLETE;
/*
* Become the pageout daemon.
*/
diff --git a/x86_64/Makefrag.am b/x86_64/Makefrag.am
index 008ac58f..0c67517c 100644
--- a/x86_64/Makefrag.am
+++ b/x86_64/Makefrag.am
@@ -103,6 +103,8 @@ libkernel_a_SOURCES += \
i386/i386/irq.c \
i386/i386/irq.h \
i386/i386/msr.h \
+ i386/i386/percpu.h \
+ i386/i386/percpu.c \
i386/i386/pit.c \
i386/i386/pit.h
diff --git a/x86_64/locore.S b/x86_64/locore.S
index c75feb23..88a5e41a 100644
--- a/x86_64/locore.S
+++ b/x86_64/locore.S
@@ -33,6 +33,7 @@
#include <i386/i386/proc_reg.h>
#include <i386/i386/trap.h>
#include <i386/i386/seg.h>
+#include <i386/i386/gdt.h>
#include <i386/i386/ldt.h>
#include <i386/i386/msr.h>
#include <i386/i386/i386asm.h>
@@ -139,6 +140,7 @@
mov %dx,%ds ;\
mov %dx,%es ;\
mov %dx,%fs ;\
+ mov $(PERCPU_DS),%dx ;\
mov %dx,%gs
#else
#define SET_KERNEL_SEGMENTS
@@ -593,7 +595,7 @@ trap_set_segs:
jz trap_from_kernel /* kernel trap if not */
trap_from_user:
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
TIME_TRAP_UENTRY
movq CX(EXT(kernel_stack),%rdx),%rbx
@@ -667,7 +669,7 @@ trap_from_kernel:
cmpq EXT(int_stack_base),%rdx
je 1f /* OK if so */
- CPU_NUMBER(%edx) /* get CPU number */
+ CPU_NUMBER_GS(%edx) /* get CPU number */
cmpq CX(EXT(kernel_stack),%rdx),%rsp
/* already on kernel stack? */
ja 0f
@@ -807,7 +809,7 @@ ENTRY(all_intrs)
SET_KERNEL_SEGMENTS
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
movq CX(EXT(int_stack_top),%rdx),%rcx
@@ -921,7 +923,7 @@ ast_from_interrupt:
pusha /* save general registers */
PUSH_SEGMENTS_ISR(%rdx)
SET_KERNEL_SEGMENTS
- CPU_NUMBER(%edx)
+ CPU_NUMBER_GS(%edx)
TIME_TRAP_UENTRY
movq CX(EXT(kernel_stack),%rdx),%rsp
--
2.40.1
