Author: br
Date: Wed Feb 24 16:50:34 2016
New Revision: 295972
URL: https://svnweb.freebsd.org/changeset/base/295972
Log:
Add support for symmetric multiprocessing (SMP).
Tested on Spike simulator with 2 and 16 cores (tlb enabled),
so set MAXCPU to 16 at this time.
This uses FDT data to get information about CPUs
(code based on arm64 mp_machdep).
Invalidate entire TLB cache as it is the only way yet.
Sponsored by: DARPA, AFRL
Sponsored by: HEIF5
Added:
head/sys/riscv/riscv/mp_machdep.c (contents, props changed)
Modified:
head/sys/boot/fdt/dts/riscv/spike.dts
head/sys/conf/files.riscv
head/sys/riscv/conf/GENERIC
head/sys/riscv/htif/htif.c
head/sys/riscv/htif/htif_block.c
head/sys/riscv/htif/htif_console.c
head/sys/riscv/include/intr.h
head/sys/riscv/include/param.h
head/sys/riscv/include/pcpu.h
head/sys/riscv/include/riscvreg.h
head/sys/riscv/include/smp.h
head/sys/riscv/riscv/cpufunc_asm.S
head/sys/riscv/riscv/exception.S
head/sys/riscv/riscv/genassym.c
head/sys/riscv/riscv/intr_machdep.c
head/sys/riscv/riscv/locore.S
head/sys/riscv/riscv/machdep.c
head/sys/riscv/riscv/pmap.c
head/sys/riscv/riscv/swtch.S
Modified: head/sys/boot/fdt/dts/riscv/spike.dts
==============================================================================
--- head/sys/boot/fdt/dts/riscv/spike.dts Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/boot/fdt/dts/riscv/spike.dts Wed Feb 24 16:50:34 2016
(r295972)
@@ -43,6 +43,23 @@
#size-cells = <1>;
#interrupt-cells = <1>;
+ cpus {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ cpu@0 {
+ device_type = "cpu";
+ compatible = "riscv,rv64i";
+ reg = <0x40002000>;
+ };
+
+ cpu@1 {
+ device_type = "cpu";
+ compatible = "riscv,rv64i";
+ reg = <0x4000a000>;
+ };
+ };
+
aliases {
console0 = &console0;
};
Modified: head/sys/conf/files.riscv
==============================================================================
--- head/sys/conf/files.riscv Wed Feb 24 15:15:46 2016 (r295971)
+++ head/sys/conf/files.riscv Wed Feb 24 16:50:34 2016 (r295972)
@@ -1,6 +1,7 @@
# $FreeBSD$
crypto/blowfish/bf_enc.c optional crypto | ipsec
crypto/des/des_enc.c optional crypto | ipsec | netsmb
+dev/ofw/ofw_cpu.c optional fdt
kern/kern_clocksource.c standard
kern/subr_dummy_vdso_tc.c standard
libkern/bcmp.c standard
@@ -29,8 +30,9 @@ riscv/riscv/intr_machdep.c standard
riscv/riscv/in_cksum.c optional inet | inet6
riscv/riscv/identcpu.c standard
riscv/riscv/locore.S standard no-obj
-riscv/riscv/minidump_machdep.c standard
riscv/riscv/machdep.c standard
+riscv/riscv/minidump_machdep.c standard
+riscv/riscv/mp_machdep.c optional smp
riscv/riscv/mem.c standard
riscv/riscv/nexus.c standard
riscv/riscv/pmap.c standard
Modified: head/sys/riscv/conf/GENERIC
==============================================================================
--- head/sys/riscv/conf/GENERIC Wed Feb 24 15:15:46 2016 (r295971)
+++ head/sys/riscv/conf/GENERIC Wed Feb 24 16:50:34 2016 (r295972)
@@ -70,7 +70,13 @@ options MAC # TrustedBSD MAC
Framewor
options RACCT # Resource accounting framework
options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default
options RCTL # Resource limits
-# options SMP
+options SMP
+
+# Uncomment for memory disk
+# options MD_ROOT
+# options MD_ROOT_SIZE=8192 # 8MB ram disk
+# makeoptions MFS_IMAGE=/path/to/img
+# options ROOTDEVNAME=\"ufs:/dev/md0\"
# Debugging support. Always need this:
# options KDB # Enable kernel debugger support.
Modified: head/sys/riscv/htif/htif.c
==============================================================================
--- head/sys/riscv/htif/htif.c Wed Feb 24 15:15:46 2016 (r295971)
+++ head/sys/riscv/htif/htif.c Wed Feb 24 16:50:34 2016 (r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -62,7 +62,7 @@ __FBSDID("$FreeBSD$");
#include "htif.h"
static struct resource_spec htif_spec[] = {
- { SYS_RES_IRQ, 0, RF_ACTIVE },
+ { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE},
{ -1, 0 }
};
@@ -126,9 +126,9 @@ htif_intr(void *arg)
sc = arg;
- htif_handle_entry(sc);
+ csr_clear(sip, SIP_SSIP);
- csr_clear(sip, SIE_SSIE);
+ htif_handle_entry(sc);
return (FILTER_HANDLED);
}
Modified: head/sys/riscv/htif/htif_block.c
==============================================================================
--- head/sys/riscv/htif/htif_block.c Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/htif/htif_block.c Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -117,6 +117,9 @@ htif_blk_intr(void *arg, uint64_t entry)
if (sc->curtag == data) {
sc->cmd_done = 1;
wakeup(&sc->intr_chan);
+ } else {
+ device_printf(sc->dev, "Unexpected tag %d (should be %d)\n",
+ data, sc->curtag);
}
}
@@ -212,6 +215,8 @@ htif_blk_task(void *arg)
HTIF_BLK_UNLOCK(sc);
if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
+ HTIF_BLK_LOCK(sc);
+
req.offset = (bp->bio_pblkno * sc->disk->d_sectorsize);
req.size = bp->bio_bcount;
paddr = vtophys(bp->bio_data);
@@ -233,7 +238,6 @@ htif_blk_task(void *arg)
htif_command(cmd);
/* Wait for interrupt */
- HTIF_BLK_LOCK(sc);
i = 0;
while (sc->cmd_done == 0) {
msleep(&sc->intr_chan, &sc->sc_mtx, PRIBIO,
"intr", hz/2);
Modified: head/sys/riscv/htif/htif_console.c
==============================================================================
--- head/sys/riscv/htif/htif_console.c Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/htif/htif_console.c Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -89,6 +89,7 @@ CONSOLE_DRIVER(riscv);
#define MAX_BURST_LEN 1
#define QUEUE_SIZE 256
#define CONSOLE_DEFAULT_ID 1ul
+#define SPIN_IN_MACHINE_MODE 1
struct queue_entry {
uint64_t data;
@@ -109,7 +110,12 @@ htif_putc(int c)
cmd |= (CONSOLE_DEFAULT_ID << HTIF_DEV_ID_SHIFT);
cmd |= c;
+#ifdef SPIN_IN_MACHINE_MODE
+ machine_command(ECALL_HTIF_LOWPUTC, cmd);
+#else
htif_command(cmd);
+#endif
+
}
static uint8_t
@@ -141,6 +147,7 @@ riscv_putc(int c)
htif_putc(c);
+#ifndef SPIN_IN_MACHINE_MODE
/* Wait for an interrupt */
__asm __volatile(
"li %0, 1\n" /* counter = 1 */
@@ -153,6 +160,7 @@ riscv_putc(int c)
"2:"
: "=&r"(counter), "=&r"(val) : "r"(cc)
);
+#endif
}
#ifdef EARLY_PRINTF
Modified: head/sys/riscv/include/intr.h
==============================================================================
--- head/sys/riscv/include/intr.h Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/include/intr.h Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -51,4 +51,16 @@ typedef unsigned long * riscv_intrcnt_t;
riscv_intrcnt_t riscv_intrcnt_create(const char *);
void riscv_intrcnt_setname(riscv_intrcnt_t, const char *);
+#ifdef SMP
+void riscv_setup_ipihandler(driver_filter_t *);
+void riscv_unmask_ipi(void);
+#endif
+
+enum {
+ IRQ_SOFTWARE,
+ IRQ_TIMER,
+ IRQ_HTIF,
+ NIRQS
+};
+
#endif /* !_MACHINE_INTR_MACHDEP_H_ */
Modified: head/sys/riscv/include/param.h
==============================================================================
--- head/sys/riscv/include/param.h Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/include/param.h Wed Feb 24 16:50:34 2016
(r295972)
@@ -51,7 +51,7 @@
#if defined(SMP) || defined(KLD_MODULE)
#ifndef MAXCPU
-#define MAXCPU 2
+#define MAXCPU 16
#endif
#else
#define MAXCPU 1
Modified: head/sys/riscv/include/pcpu.h
==============================================================================
--- head/sys/riscv/include/pcpu.h Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/include/pcpu.h Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,7 +1,16 @@
/*-
* Copyright (c) 1999 Luoqi Chen <lu...@freebsd.org>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -36,7 +45,10 @@
#define ALT_STACK_SIZE 128
#define PCPU_MD_FIELDS
\
- char __pad[129]
+ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \
+ uint64_t pc_sptbr; /* L0 page table base (VA) */ \
+ uint64_t pc_reg; /* CPU MMIO base (PA) */ \
+ char __pad[109]
#ifdef _KERNEL
Modified: head/sys/riscv/include/riscvreg.h
==============================================================================
--- head/sys/riscv/include/riscvreg.h Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/include/riscvreg.h Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -38,12 +38,16 @@
#define _MACHINE_RISCVREG_H_
/* Machine mode requests */
-#define ECALL_MTIMECMP 0x1
-#define ECALL_CLEAR_PENDING 0x2
-#define ECALL_HTIF_CMD 0x3
-#define ECALL_HTIF_GET_ENTRY 0x4
-#define ECALL_MCPUID_GET 0x5
-#define ECALL_MIMPID_GET 0x6
+#define ECALL_MTIMECMP 0x01
+#define ECALL_CLEAR_PENDING 0x02
+#define ECALL_HTIF_CMD 0x03
+#define ECALL_HTIF_GET_ENTRY 0x04
+#define ECALL_MCPUID_GET 0x05
+#define ECALL_MIMPID_GET 0x06
+#define ECALL_SEND_IPI 0x07
+#define ECALL_CLEAR_IPI 0x08
+#define ECALL_HTIF_LOWPUTC 0x09
+#define ECALL_MIE_SET 0x10
#define EXCP_SHIFT 0
#define EXCP_MASK (0xf << EXCP_SHIFT)
@@ -109,9 +113,14 @@
#define SIE_SSIE (1 << 1)
#define SIE_STIE (1 << 5)
-/* Note: sip register is not yet implement in Spike simulator */
+/* Note: sip register has no SIP_STIP bit in Spike simulator */
+#define SIP_SSIP (1 << 1)
#define SIP_STIP (1 << 5)
+#define NCSRS 4096
+#define CSR_IPI 0x783
+#define XLEN 8
+
#define CSR_ZIMM(val)
\
(__builtin_constant_p(val) && ((u_long)(val) < 32))
Modified: head/sys/riscv/include/smp.h
==============================================================================
--- head/sys/riscv/include/smp.h Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/include/smp.h Wed Feb 24 16:50:34 2016
(r295972)
@@ -1 +1,55 @@
-/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <b...@bsdpad.com>
+ * All rights reserved.
+ *
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_SMP_H_
+#define _MACHINE_SMP_H_
+
+#include <machine/pcb.h>
+
+#define IPI_AST (1 << 0)
+#define IPI_PREEMPT (1 << 1)
+#define IPI_RENDEZVOUS (1 << 2)
+#define IPI_STOP (1 << 3)
+#define IPI_STOP_HARD (1 << 4)
+#define IPI_HARDCLOCK (1 << 5)
+
+void ipi_all_but_self(u_int ipi);
+void ipi_cpu(int cpu, u_int ipi);
+void ipi_selected(cpuset_t cpus, u_int ipi);
+
+extern struct pcb stoppcbs[];
+
+#endif /* !_MACHINE_SMP_H_ */
Modified: head/sys/riscv/riscv/cpufunc_asm.S
==============================================================================
--- head/sys/riscv/riscv/cpufunc_asm.S Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/riscv/cpufunc_asm.S Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -64,7 +64,7 @@ END(riscv_tlb_flushID_SE)
* void riscv_dcache_wb_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_dcache_wb_range)
- /* RISCVTODO */
+ sfence.vm
ret
END(riscv_dcache_wb_range)
@@ -72,7 +72,7 @@ END(riscv_dcache_wb_range)
* void riscv_dcache_wbinv_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_dcache_wbinv_range)
- /* RISCVTODO */
+ sfence.vm
ret
END(riscv_dcache_wbinv_range)
@@ -80,7 +80,7 @@ END(riscv_dcache_wbinv_range)
* void riscv_dcache_inv_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_dcache_inv_range)
- /* RISCVTODO */
+ sfence.vm
ret
END(riscv_dcache_inv_range)
@@ -88,7 +88,8 @@ END(riscv_dcache_inv_range)
* void riscv_idcache_wbinv_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_idcache_wbinv_range)
- /* RISCVTODO */
+ fence.i
+ sfence.vm
ret
END(riscv_idcache_wbinv_range)
@@ -96,6 +97,6 @@ END(riscv_idcache_wbinv_range)
* void riscv_icache_sync_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_icache_sync_range)
- /* RISCVTODO */
+ fence.i
ret
END(riscv_icache_sync_range)
Modified: head/sys/riscv/riscv/exception.S
==============================================================================
--- head/sys/riscv/riscv/exception.S Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/riscv/exception.S Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -229,6 +229,7 @@ bad_trap:
j bad_trap
user_trap:
+ /* Save state */
csrrw sp, mscratch, sp
addi sp, sp, -64
sd t0, (8 * 0)(sp)
@@ -282,7 +283,31 @@ machine_interrupt:
j 1b
software_interrupt:
- /* Redirect to supervisor */
+ li t0, MIP_MSIP
+ csrc mip, t0
+ li t0, MIP_SSIP
+ csrs mip, t0
+
+ /* If PRV1 is PRV_U (user) then serve the trap */
+ csrr t0, mstatus
+ li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
+ and t0, t0, t1
+ beqz t0, 1f
+
+ /*
+ * If PRV1 is supervisor and interrupts were enabled,
+ * then serve the trap.
+ */
+ csrr t0, mstatus
+ li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
+ and t0, t0, t1
+ li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
+ beq t0, t1, 1f
+
+ j exit
+
+1:
+ /* Serve a trap in supervisor mode */
j exit_mrts
timer_interrupt:
@@ -294,17 +319,20 @@ timer_interrupt:
li t0, MIP_MTIP
csrc mip, t0
- /* Post supervisor software interrupt */
+ /* Post supervisor timer interrupt */
li t0, MIP_STIP
csrs mip, t0
- /* If PRV1 is PRV_U (user) then serve a trap */
+ /* If PRV1 is PRV_U (user) then serve the trap */
csrr t0, mstatus
li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
and t0, t0, t1
beqz t0, 1f
- /* If PRV1 is supervisor and interrupts were enabled, then serve a trap
*/
+ /*
+ * If PRV1 is supervisor and interrupts were enabled,
+ * then serve the trap.
+ */
csrr t0, mstatus
li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
and t0, t0, t1
@@ -332,25 +360,35 @@ htif_interrupt:
la t0, console_intr
li t1, 1
sd t1, 0(t0)
- j 3f
+
+ /* Check if there is any other pending event */
+ j 1b
2:
/* Save entry */
- la t0, htif_ring_cursor
- beqz t0, 3f /* not initialized */
- ld t0, 0(t0) /* load struct */
- sd t5, 0(t0) /* put entry */
+ la t0, htif_ring
+ csrr t1, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t1
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE)
+ add t0, t0, t4 /* t0 == htif_ring_cursor */
+
+ ld t1, 0(t0) /* load ptr to cursor */
+ sd t5, 0(t1) /* put entry */
li t4, 1
- sd t4, 8(t0) /* mark used */
- ld t4, 16(t0) /* take next */
+ sd t4, 8(t1) /* mark used */
+ ld t4, 16(t1) /* take next */
/* Update cursor */
- la t0, htif_ring_cursor
sd t4, 0(t0)
/* Post supervisor software interrupt */
li t0, MIP_SSIP
csrs mip, t0
+ /* Check if there is any other pending event */
+ j 1b
+
3:
j exit
@@ -369,6 +407,18 @@ supervisor_call:
beq t5, t4, mcpuid_get
li t4, ECALL_MIMPID_GET
beq t5, t4, mimpid_get
+ li t4, ECALL_SEND_IPI
+ beq t5, t4, send_ipi
+ li t4, ECALL_CLEAR_IPI
+ beq t5, t4, clear_ipi
+ li t4, ECALL_HTIF_LOWPUTC
+ beq t5, t4, htif_lowputc
+ li t4, ECALL_MIE_SET
+ beq t5, t4, mie_set
+ j exit_next_instr
+
+mie_set:
+ csrs mie, t6
j exit_next_instr
mcpuid_get:
@@ -379,28 +429,111 @@ mimpid_get:
csrr t6, mimpid
j exit_next_instr
+send_ipi:
+ /* CPU mmio base in t6 */
+ mv t0, t6
+ li t2, (CSR_IPI * XLEN)
+ add t0, t0, t2 /* t0 = CSR_IPI */
+ li t2, 1
+ sd t2, 0(t0)
+ j exit_next_instr
+
+clear_ipi:
+ /* Do only clear if there are no new entries in HTIF ring */
+ la t0, htif_ring
+ csrr t2, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t2
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE)
+ add t0, t0, t4 /* t0 == ptr to htif_ring_cursor */
+ ld t2, 8(t0) /* load htif_ring_last */
+ ld t2, 8(t2) /* load used */
+ bnez t2, 1f
+
+ /* Clear supervisor software interrupt pending bit */
+ li t0, MIP_SSIP
+ csrc mip, t0
+
+1:
+ j exit_next_instr
+
htif_get_entry:
+ /* Get a htif_ring for current core */
+ la t0, htif_ring
+ csrr t2, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t2
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE + 8)
+ add t0, t0, t4 /* t0 == htif_ring_last */
+
+ /* Check for new entries */
li t6, 0 /* preset return value */
- la t0, htif_ring_last
- ld t0, 0(t0) /* load struct */
- ld t4, 8(t0) /* get used */
- beqz t4, 1f
- ld t6, 0(t0) /* get entry */
+ ld t2, 0(t0) /* load ptr to last */
+ ld t4, 8(t2) /* get used */
+ beqz t4, 1f /* No new entries. Exit */
+
+ /* Get one */
+ ld t6, 0(t2) /* get entry */
li t4, 0
- sd t4, 8(t0) /* mark free */
- sd t4, 0(t0) /* free entry, just in case */
- ld t4, 16(t0) /* take next */
- la t0, htif_ring_last
- sd t4, 0(t0)
+ sd t4, 8(t2) /* mark free */
+ sd t4, 0(t2) /* free entry, just in case */
+ ld t4, 16(t2) /* take next */
+ sd t4, 0(t0) /* update ptr to last */
1:
/* Exit. Result is stored in t6 */
j exit_next_instr
htif_cmd:
+1:
mv t0, t6
+ csrrw t0, mtohost, t0
+ bnez t0, 1b
+ j exit_next_instr
+
+htif_lowputc:
1:
+ mv t0, t6
csrrw t0, mtohost, t0
bnez t0, 1b
+
+2:
+ li t4, 0
+ csrrw t5, mfromhost, t4
+ beqz t5, 2b
+
+ /* Console PUT intr ? */
+ mv t2, t5
+ srli t2, t2, 48
+ li t3, 0x0101
+ beq t2, t3, 3f
+
+ /* Not a console PUT, so save entry */
+ la t0, htif_ring
+ csrr t2, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t2
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE)
+ add t0, t0, t4 /* t0 == htif_ring_cursor */
+
+ ld t2, 0(t0) /* load ptr to cursor */
+ sd t5, 0(t2) /* put entry */
+ li t4, 1
+ sd t4, 8(t2) /* mark used */
+ ld t4, 16(t2) /* take next */
+ /* Update cursor */
+ sd t4, 0(t0)
+
+ /* Post supervisor software interrupt */
+ li t0, MIP_SSIP
+ csrs mip, t0
+
+ /* Wait for console intr again */
+ j 2b
+
+3:
j exit_next_instr
set_mtimecmp:
Modified: head/sys/riscv/riscv/genassym.c
==============================================================================
--- head/sys/riscv/riscv/genassym.c Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/riscv/genassym.c Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -75,6 +75,7 @@ ASSYM(PCB_A, offsetof(struct pcb, pcb_a)
ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
+ASSYM(PC_SPTBR, offsetof(struct pcpu, pc_sptbr));
ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
Modified: head/sys/riscv/riscv/intr_machdep.c
==============================================================================
--- head/sys/riscv/riscv/intr_machdep.c Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/riscv/intr_machdep.c Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -38,7 +38,9 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
+#include <sys/cpuset.h>
#include <sys/interrupt.h>
+#include <sys/smp.h>
#include <machine/clock.h>
#include <machine/cpu.h>
@@ -46,12 +48,9 @@ __FBSDID("$FreeBSD$");
#include <machine/frame.h>
#include <machine/intr.h>
-enum {
- IRQ_SOFTWARE,
- IRQ_TIMER,
- IRQ_HTIF,
- NIRQS
-};
+#ifdef SMP
+#include <machine/smp.h>
+#endif
u_long intrcnt[NIRQS];
size_t sintrcnt = sizeof(intrcnt);
@@ -159,8 +158,12 @@ riscv_setup_intr(const char *name, drive
riscv_unmask_irq((void*)(uintptr_t)irq);
}
- intr_event_add_handler(event, name, filt, handler, arg,
+ error = intr_event_add_handler(event, name, filt, handler, arg,
intr_priority(flags), flags, cookiep);
+ if (error) {
+ printf("Failed to setup intr: %d\n", irq);
+ return (error);
+ }
riscv_intrcnt_setname(riscv_intr_counters[irq],
event->ie_fullname);
@@ -221,3 +224,74 @@ riscv_cpu_intr(struct trapframe *frame)
critical_exit();
}
+
+#ifdef SMP
+void
+riscv_setup_ipihandler(driver_filter_t *filt)
+{
+
+ riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE,
+ INTR_TYPE_MISC, NULL);
+}
+
+void
+riscv_unmask_ipi(void)
+{
+
+ csr_set(sie, SIE_SSIE);
+}
+
+/* Sending IPI */
+static void
+ipi_send(struct pcpu *pc, int ipi)
+{
+
+ CTR3(KTR_SMP, "%s: cpu=%d, ipi=%x", __func__, pc->pc_cpuid, ipi);
+
+ atomic_set_32(&pc->pc_pending_ipis, ipi);
+ machine_command(ECALL_SEND_IPI, pc->pc_reg);
+
+ CTR1(KTR_SMP, "%s: sent", __func__);
+}
+
+void
+ipi_all_but_self(u_int ipi)
+{
+ cpuset_t other_cpus;
+
+ other_cpus = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &other_cpus);
+
+ CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+ ipi_selected(other_cpus, ipi);
+}
+
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ cpuset_t cpus;
+
+ CPU_ZERO(&cpus);
+ CPU_SET(cpu, &cpus);
+
+ CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi);
+ ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
+void
+ipi_selected(cpuset_t cpus, u_int ipi)
+{
+ struct pcpu *pc;
+
+ CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
+
+ STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
+ if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
+ CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc,
+ ipi);
+ ipi_send(pc, ipi);
+ }
+ }
+}
+
+#endif
Modified: head/sys/riscv/riscv/locore.S
==============================================================================
--- head/sys/riscv/riscv/locore.S Wed Feb 24 15:15:46 2016
(r295971)
+++ head/sys/riscv/riscv/locore.S Wed Feb 24 16:50:34 2016
(r295972)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <b...@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -43,8 +43,55 @@
#include <machine/riscvreg.h>
#include <machine/pte.h>
-#define HTIF_RING_SIZE (64)
-#define HTIF_RING_LAST (24 * (HTIF_RING_SIZE - 1))
+#define HTIF_RING_NENTRIES (512)
+#define HTIF_RING_ENTRY_SZ (24)
+#define HTIF_RING_SIZE (HTIF_RING_ENTRY_SZ *
HTIF_RING_NENTRIES)
+#define HW_STACK_SIZE (96)
+
+/*
+ * Event queue for each CPU core:
+ *
+ * struct htif_ring {
+ * uint64_t data;
+ * uint64_t used;
+ * uint64_t next;
+ * } htif_ring[HTIF_RING_NENTRIES];
+ * uint64_t htif_ring_cursor;
+ * uint64_t htif_ring_last;
+ */
+
+.macro build_ring
+ la t0, htif_ring
+#ifdef SMP
+ csrr a0, mhartid
+ li s0, (HTIF_RING_SIZE + 16)
+ mulw s0, a0, s0
+ add t0, t0, s0
+#endif
+ li t1, 0
+ sd t1, 0(t0) /* zero data */
+ sd t1, 8(t0) /* zero used */
+ mv t2, t0
+ mv t3, t0
+ li t5, (HTIF_RING_SIZE - HTIF_RING_ENTRY_SZ)
+ li t6, 0
+ add t4, t0, t5
+1:
+ addi t3, t3, 24 /* pointer to next */
+ beq t3, t4, 2f /* finish */
+ sd t3, 16(t2) /* store pointer */
+ addi t2, t2, 24 /* next entry */
+ addi t6, t6, 1 /* counter */
+ j 1b
+2:
+ sd t0, 16(t3) /* last -> first */
+
+ li t2, (HTIF_RING_SIZE)
+ add s0, t0, t2
+ sd t0, 0(s0) /* cursor */
+ sd t0, 8(s0) /* last */
+ /* finish building ring */
+.endm
.globl kernbase
.set kernbase, KERNBASE
@@ -74,33 +121,14 @@ mentry:
.align 8
.globl _start
_start:
- li s11, KERNBASE
+ /* Direct secondary cores to mpentry */
+ csrr a0, mhartid
+ bnez a0, mpentry
- /* Build ring */
- la t0, htif_ring
- li t1, 0
- sd t1, 0(t0) /* zero data */
- sd t1, 8(t0) /* zero used */
- mv t2, t0
- mv t3, t0
- li t5, HTIF_RING_LAST
- li t6, 0
- add t4, t0, t5
-1:
- addi t3, t3, 24 /* pointer to next */
- beq t3, t4, 2f /* finish */
- sd t3, 16(t2) /* store pointer */
- addi t2, t2, 24 /* next entry */
- addi t6, t6, 1 /* counter */
- j 1b
-2:
- sd t0, 16(t3) /* last -> first */
- la t1, htif_ring_cursor
- sd t0, 0(t1)
- la t1, htif_ring_last
- sd t0, 0(t1)
- /* finish building ring */
+ /* Build event queue for current core */
+ build_ring
+ /* Setup machine-mode stack for CPU 0 */
la t0, hardstack_end
csrw mscratch, t0
@@ -129,7 +157,14 @@ _start:
li a6, PTE_SIZE
mulw a5, a5, a6
add t0, s1, a5
+ /* Store it to pagetable_l0 for each cpu */
+ li t1, MAXCPU
+ li t2, PAGE_SIZE
+1:
sd t6, 0(t0)
+ add t0, t0, t2
+ addi t1, t1, -1
+ bnez t1, 1b
/* Level 1 */
la s1, pagetable_l1
@@ -177,8 +212,16 @@ _start:
(MSTATUS_PRV_U << MSTATUS_PRV2_SHIFT));
csrw mstatus, s0
+ /*
+ * Enable machine-mode software interrupts
+ * so we can deliver IPI to this core.
+ */
+ li t0, MIE_MSIE
+ csrs mie, t0
+
/* Exit from machine mode */
la t0, .Lmmu_on
+ li s11, KERNBASE
add t0, t0, s11
csrw mepc, t0
eret
@@ -213,24 +256,16 @@ initstack:
.space (PAGE_SIZE * KSTACK_PAGES)
initstack_end:
hardstack:
- .space (PAGE_SIZE)
+ .space (HW_STACK_SIZE * MAXCPU)
hardstack_end:
.globl htif_ring
htif_ring:
- .space (24 * 1024)
-
- .globl htif_ring_cursor
-htif_ring_cursor:
- .space (8)
-
- .globl htif_ring_last
-htif_ring_last:
- .space (8)
+ .space ((HTIF_RING_SIZE + 16) * MAXCPU)
.globl console_intr
console_intr:
- .space (8)
+ .space (8)
ENTRY(sigcode)
mv a0, sp
@@ -259,7 +294,7 @@ szsigcode:
.align 12
.globl pagetable_l0
pagetable_l0:
- .space PAGE_SIZE
+ .space (PAGE_SIZE * MAXCPU)
pagetable_l1:
.space PAGE_SIZE
pagetable_l2:
@@ -270,4 +305,95 @@ pagetable_end:
init_pt_va:
.quad pagetable_l2 /* XXX: Keep page tables VA */
+#ifndef SMP
+ENTRY(mpentry)
+1:
+ wfi
+ j 1b
+END(mpentry)
+#else
+/*
+ * mpentry(unsigned long)
+ *
+ * Called by a core when it is being brought online.
+ * The data in x0 is passed straight to init_secondary.
+ */
+ENTRY(mpentry)
+ /*
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
