Module Name: src
Committed By: jmcneill
Date: Sat Oct 30 18:44:24 UTC 2021
Modified Files:
src/sys/arch/aarch64/conf: files.aarch64
src/sys/arch/arm/cortex: gic_splfuncs.c
Added Files:
src/sys/arch/arm/cortex: gic_splfuncs_armv8.S
Log Message:
Implement gic_splraise and the gic_splx fast path in asm (armv8).
To generate a diff of this commit:
cvs rdiff -u -r1.34 -r1.35 src/sys/arch/aarch64/conf/files.aarch64
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/arm/cortex/gic_splfuncs.c
cvs rdiff -u -r0 -r1.1 src/sys/arch/arm/cortex/gic_splfuncs_armv8.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/conf/files.aarch64
diff -u src/sys/arch/aarch64/conf/files.aarch64:1.34 src/sys/arch/aarch64/conf/files.aarch64:1.35
--- src/sys/arch/aarch64/conf/files.aarch64:1.34 Sun Oct 10 07:15:25 2021
+++ src/sys/arch/aarch64/conf/files.aarch64 Sat Oct 30 18:44:24 2021
@@ -1,4 +1,4 @@
-# $NetBSD: files.aarch64,v 1.34 2021/10/10 07:15:25 skrll Exp $
+# $NetBSD: files.aarch64,v 1.35 2021/10/30 18:44:24 jmcneill Exp $
defflag opt_cpuoptions.h AARCH64_ALIGNMENT_CHECK
defflag opt_cpuoptions.h AARCH64_EL0_STACK_ALIGNMENT_CHECK
@@ -119,6 +119,9 @@ file arch/aarch64/aarch64/pmap_page.S
file uvm/pmap/pmap_tlb.c
file uvm/pmap/pmap_pvt.c
+# GIC
+file arch/arm/cortex/gic_splfuncs_armv8.S gic_splfuncs
+
# EFI runtime (machdep)
file arch/aarch64/aarch64/efi_machdep.c efi_runtime
Index: src/sys/arch/arm/cortex/gic_splfuncs.c
diff -u src/sys/arch/arm/cortex/gic_splfuncs.c:1.4 src/sys/arch/arm/cortex/gic_splfuncs.c:1.5
--- src/sys/arch/arm/cortex/gic_splfuncs.c:1.4 Sun Sep 26 20:55:15 2021
+++ src/sys/arch/arm/cortex/gic_splfuncs.c Sat Oct 30 18:44:24 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: gic_splfuncs.c,v 1.4 2021/09/26 20:55:15 jmcneill Exp $ */
+/* $NetBSD: gic_splfuncs.c,v 1.5 2021/10/30 18:44:24 jmcneill Exp $ */
/*-
* Copyright (c) 2021 Jared McNeill <jmcne...@invisible.ca>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.4 2021/09/26 20:55:15 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.5 2021/10/30 18:44:24 jmcneill Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -41,16 +41,12 @@ __KERNEL_RCSID(0, "$NetBSD: gic_splfuncs
#include <arm/cortex/gic_splfuncs.h>
-static int
-gic_splraise(int newipl)
-{
- struct cpu_info * const ci = curcpu();
- const int oldipl = ci->ci_cpl;
- if (__predict_true(newipl > oldipl)) {
- ci->ci_cpl = newipl;
- }
- return oldipl;
-}
+/* Prototypes for functions in gic_splfuncs_<arch>.S */
+int gic_splraise(int);
+void gic_splx(int);
+
+/* Local functions */
+void Xgic_splx(int);
static int
gic_spllower(int newipl)
@@ -72,8 +68,8 @@ gic_spllower(int newipl)
return oldipl;
}
-static void
-gic_splx(int newipl)
+void
+Xgic_splx(int newipl)
{
struct cpu_info *ci = curcpu();
register_t psw;
@@ -82,37 +78,6 @@ gic_splx(int newipl)
return;
}
- /*
- * Try to avoid touching any hardware registers (DAIF, PMR) as an
- * optimization for the common case of splraise followed by splx
- * with no interrupts in between.
- *
- * If an interrupt fires in this critical section, the vector
- * handler is responsible for returning to the address pointed
- * to by ci_splx_restart to restart the sequence.
- */
- if (__predict_true(ci->ci_intr_depth == 0)) {
- ci->ci_splx_savedipl = newipl;
- __insn_barrier();
- ci->ci_splx_restart = &&restart;
- __insn_barrier();
-checkhwpl:
- if (ci->ci_hwpl <= newipl) {
- ci->ci_cpl = newipl;
- __insn_barrier();
- ci->ci_splx_restart = NULL;
- goto dosoft;
- } else {
- ci->ci_splx_restart = NULL;
- goto dohard;
- }
-restart:
- ci = curcpu();
- newipl = ci->ci_splx_savedipl;
- goto checkhwpl;
- }
-
-dohard:
psw = DISABLE_INTERRUPT_SAVE();
ci->ci_intr_depth++;
pic_do_pending_ints(psw, newipl, NULL);
@@ -121,7 +86,6 @@ dohard:
ENABLE_INTERRUPT();
}
-dosoft:
cpu_dosoftints();
}
Added files:
Index: src/sys/arch/arm/cortex/gic_splfuncs_armv8.S
diff -u /dev/null src/sys/arch/arm/cortex/gic_splfuncs_armv8.S:1.1
--- /dev/null Sat Oct 30 18:44:24 2021
+++ src/sys/arch/arm/cortex/gic_splfuncs_armv8.S Sat Oct 30 18:44:24 2021
@@ -0,0 +1,118 @@
+/* $NetBSD: gic_splfuncs_armv8.S,v 1.1 2021/10/30 18:44:24 jmcneill Exp $ */
+
+/*-
+ * Copyright (c) 2021 Jared McNeill <jmcne...@invisible.ca>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+#include "assym.h"
+
+RCSID("$NetBSD: gic_splfuncs_armv8.S,v 1.1 2021/10/30 18:44:24 jmcneill Exp $")
+
+/*
+ * int
+ * gic_splraise(int newipl)
+ *
+ * w0 = newipl
+ */
+ .align 7 /* cacheline-aligned */
+ENTRY_NP(gic_splraise)
+ /* Save cpu_info pointer in x1 */
+ mrs x1, tpidr_el1 /* get curlwp */
+ ldr x1, [x1, #L_CPU] /* get curcpu */
+
+ /* If newipl > cpl, update cpl */
+ ldr w2, [x1, #CI_CPL]
+ cmp w0, w2
+ b.le .Lnoraise
+ str w0, [x1, #CI_CPL]
+
+.Lnoraise:
+ mov w0, w2 /* return oldipl */
+ ret
+END(gic_splraise)
+
+
+/*
+ * void
+ * gic_splx(int newipl)
+ *
+ * w0 = newipl
+ */
+ .align 7 /* cacheline-aligned */
+ENTRY_NP(gic_splx)
+ /* Save cpu_info pointer in x1 */
+ mrs x1, tpidr_el1 /* get curlwp */
+ ldr x1, [x1, #L_CPU] /* get curcpu */
+
+ /* If newipl >= cpl, just return */
+ ldr w2, [x1, #CI_CPL]
+ cmp w0, w2
+ b.hs .Ldone
+
+.Lagain:
+ /* Slow path if ci_intr_depth != 0 */
+ ldr w2, [x1, #CI_INTR_DEPTH]
+ cbnz w2, .Lslow
+
+ /* Save newipl and restart address in cpu info */
+ str w0, [x1, #CI_SPLX_SAVEDIPL]
+ adr x2, .Lrestart
+ str x2, [x1, #CI_SPLX_RESTART]
+
+ /* Slow path if hwpl > newipl */
+ ldr w2, [x1, #CI_HWPL]
+ cmp w2, w0
+ b.hi .Lrestore
+
+ /* Update cpl */
+ str w0, [x1, #CI_CPL]
+
+ /* Clear saved restart address from cpu info */
+ str xzr, [x1, #CI_SPLX_RESTART]
+
+ /* Check for pending softints */
+ ldr w2, [x1, #CI_SOFTINTS]
+ lsr w2, w2, w0
+ cbnz w2, _C_LABEL(dosoftints)
+
+.Ldone:
+ ret
+
+.Lrestart:
+ /* Reload w0 and x1 */
+ mrs x1, tpidr_el1 /* get curlwp */
+ ldr x1, [x1, #L_CPU] /* get curcpu */
+ ldr w0, [x1, #CI_SPLX_SAVEDIPL] /* get newipl */
+ b .Lagain
+
+.Lrestore:
+ /* Clear saved restart address from cpu info */
+ str xzr, [x1, #CI_SPLX_RESTART]
+
+.Lslow:
+ /* Jump to slow path */
+ b _C_LABEL(Xgic_splx)
+END(gic_splx)
