Module Name: src
Committed By: riastradh
Date: Thu Feb 23 14:56:00 UTC 2023
Modified Files:
src/sys/arch/evbmips/ingenic: cpu_startup.S
src/sys/arch/mips/include: asm.h
src/sys/arch/mips/mips: locore.S locore_mips3.S
Log Message:
mips: Add missing barriers in cpu_switchto.
Details in comments.
PR kern/57240
XXX pullup-8
XXX pullup-9
XXX pullup-10
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/evbmips/ingenic/cpu_startup.S
cvs rdiff -u -r1.73 -r1.74 src/sys/arch/mips/include/asm.h
cvs rdiff -u -r1.228 -r1.229 src/sys/arch/mips/mips/locore.S
cvs rdiff -u -r1.115 -r1.116 src/sys/arch/mips/mips/locore_mips3.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/evbmips/ingenic/cpu_startup.S
diff -u src/sys/arch/evbmips/ingenic/cpu_startup.S:1.1 src/sys/arch/evbmips/ingenic/cpu_startup.S:1.2
--- src/sys/arch/evbmips/ingenic/cpu_startup.S:1.1 Fri Jan 29 01:54:14 2016
+++ src/sys/arch/evbmips/ingenic/cpu_startup.S Thu Feb 23 14:56:00 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_startup.S,v 1.1 2016/01/29 01:54:14 macallan Exp $ */
+/* $NetBSD: cpu_startup.S,v 1.2 2023/02/23 14:56:00 riastradh Exp $ */
/*-
* Copyright (c) 2015 Michael Lorenz
@@ -33,7 +33,7 @@
#include <sys/endian.h>
#include <mips/asm.h>
-RCSID("$NetBSD: cpu_startup.S,v 1.1 2016/01/29 01:54:14 macallan Exp $");
+RCSID("$NetBSD: cpu_startup.S,v 1.2 2023/02/23 14:56:00 riastradh Exp $");
#ifdef MULTIPROCESSOR
@@ -56,6 +56,11 @@ NESTED_NOPROFILE(ingenic_trampoline, 0,
nop
beqz MIPS_CURLWP, 1b
nop
+ /*
+ * No membar needed because we're not switching from a
+ * previous lwp, and the idle lwp we're switching to can't be
+ * holding locks already; see cpu_switchto.
+ */
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(a0)
li v0, 0
Index: src/sys/arch/mips/include/asm.h
diff -u src/sys/arch/mips/include/asm.h:1.73 src/sys/arch/mips/include/asm.h:1.74
--- src/sys/arch/mips/include/asm.h:1.73 Mon Feb 20 13:30:47 2023
+++ src/sys/arch/mips/include/asm.h Thu Feb 23 14:56:00 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: asm.h,v 1.73 2023/02/20 13:30:47 riastradh Exp $ */
+/* $NetBSD: asm.h,v 1.74 2023/02/23 14:56:00 riastradh Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -633,6 +633,28 @@ _C_LABEL(x):
#define SYNC_PLUNGER /* nothing */
#endif
+/*
+ * Store-before-load barrier. Do not use this unless you know what
+ * you're doing.
+ */
+#ifdef MULTIPROCESSOR
+#define SYNC_DEKKER sync
+#else
+#define SYNC_DEKKER /* nothing */
+#endif
+
+/*
+ * Store-before-store and load-before-load barriers. These could be
+ * made weaker than release (load/store-before-store) and acquire
+ * (load-before-load/store) barriers, and newer MIPS does have
+ * instruction encodings for finer-grained barriers like this, but I
+ * dunno how to appropriately conditionalize their use or get the
+ * assembler to be happy with them, so we'll use these definitions for
+ * now.
+ */
+#define SYNC_PRODUCER SYNC_REL
+#define SYNC_CONSUMER SYNC_ACQ
+
/* CPU dependent hook for cp0 load delays */
#if defined(MIPS1) || defined(MIPS2) || defined(MIPS3)
#define MFC0_HAZARD sll $0,$0,1 /* super scalar nop */
Index: src/sys/arch/mips/mips/locore.S
diff -u src/sys/arch/mips/mips/locore.S:1.228 src/sys/arch/mips/mips/locore.S:1.229
--- src/sys/arch/mips/mips/locore.S:1.228 Sat May 29 12:35:27 2021
+++ src/sys/arch/mips/mips/locore.S Thu Feb 23 14:56:00 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.228 2021/05/29 12:35:27 simonb Exp $ */
+/* $NetBSD: locore.S,v 1.229 2023/02/23 14:56:00 riastradh Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -63,7 +63,7 @@
#include <mips/trap.h>
#include <mips/locore.h>
-RCSID("$NetBSD: locore.S,v 1.228 2021/05/29 12:35:27 simonb Exp $")
+RCSID("$NetBSD: locore.S,v 1.229 2023/02/23 14:56:00 riastradh Exp $")
#include "assym.h"
@@ -286,7 +286,28 @@ NESTED(cpu_switchto, CALLFRAME_SIZ, ra)
PTR_L t2, L_CPU(MIPS_CURLWP)
nop # patchable load delay slot
+
+ /*
+ * Issue barriers to coordinate mutex_exit on this CPU with
+ * mutex_vector_enter on another CPU.
+ *
+ * 1. Any prior mutex_exit by oldlwp must be visible to other
+ * CPUs before we set ci_curlwp := newlwp on this one,
+ * requiring a store-before-store barrier.
+ *
+ * 2. ci_curlwp := newlwp must be visible on all other CPUs
+ * before any subsequent mutex_exit by newlwp can even test
+ * whether there might be waiters, requiring a
+ * store-before-load barrier.
+ *
+ * See kern_mutex.c for details -- this is necessary for
+ * adaptive mutexes to detect whether the lwp is on the CPU in
+ * order to safely block without requiring atomic r/m/w in
+ * mutex_exit.
+ */
+ SYNC_PRODUCER /* XXX fixup to nop for uniprocessor boot */
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(t2)
+ SYNC_DEKKER /* XXX fixup to nop for uniprocessor boot */
/* Check for restartable atomic sequences (RAS) */
PTR_L a0, L_PROC(MIPS_CURLWP) # argument to ras_lookup
@@ -406,7 +427,9 @@ NESTED(softint_fast_dispatch, CALLFRAME_
move MIPS_CURLWP, a0 # switch to softint lwp
PTR_L s1, L_CPU(MIPS_CURLWP) # get curcpu()
nop # patchable load delay slot
+ SYNC_PRODUCER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(s1) # ...
+ SYNC_DEKKER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
move s2, sp # remember sp
move s3, t0 # remember curpcb
@@ -417,7 +440,9 @@ NESTED(softint_fast_dispatch, CALLFRAME_
move sp, s2 # restore stack
move MIPS_CURLWP, s0 # restore curlwp
+ SYNC_PRODUCER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(s1) # ....
+ SYNC_DEKKER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
REG_L ra, CALLFRAME_RA(sp) # load early since we use it
Index: src/sys/arch/mips/mips/locore_mips3.S
diff -u src/sys/arch/mips/mips/locore_mips3.S:1.115 src/sys/arch/mips/mips/locore_mips3.S:1.116
--- src/sys/arch/mips/mips/locore_mips3.S:1.115 Sun May 24 07:15:24 2020
+++ src/sys/arch/mips/mips/locore_mips3.S Thu Feb 23 14:56:00 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: locore_mips3.S,v 1.115 2020/05/24 07:15:24 simonb Exp $ */
+/* $NetBSD: locore_mips3.S,v 1.116 2023/02/23 14:56:00 riastradh Exp $ */
/*
* Copyright (c) 1997 Jonathan Stone (hereinafter referred to as the author)
@@ -92,7 +92,7 @@
#include <mips/asm.h>
#include <mips/cpuregs.h>
-RCSID("$NetBSD: locore_mips3.S,v 1.115 2020/05/24 07:15:24 simonb Exp $")
+RCSID("$NetBSD: locore_mips3.S,v 1.116 2023/02/23 14:56:00 riastradh Exp $")
#include "assym.h"
@@ -809,6 +809,11 @@ NESTED_NOPROFILE(cpu_trampoline, 0, ra)
nop
beqz MIPS_CURLWP, 1b
nop
+ /*
+ * No membar needed because we're not switching from a
+ * previous lwp, and the idle lwp we're switching to can't be
+ * holding locks already; see cpu_switchto.
+ */
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(a0)
#ifdef _LP64
