Module Name: src
Committed By: riastradh
Date: Sat Aug 6 21:31:33 UTC 2022
Modified Files:
src/common/lib/libc/arch/aarch64/atomic: Makefile.inc __aarch64_lse.S
Log Message:
aarch64: Implement __aarch64_casN_sync.
gcc generates calls to this symbol in programs that use
__sync_*_compare_and_swap, which require full sequential consistency
barriers, including store-before-load ordering on both sides of the
atomic; none of the release/acquire operations guarantee that, so we
have to insert explicit DMB instructions.
Note: gcc's own definition omits some of the DMB instructions, but I
can't prove that it's correct that way -- stores preceding the CAS
must complete before the load part of the CAS, and the store part of
the CAS must complete before loads following the CAS. Maybe there's
some way to prove that one of these orderings is guaranteed some
other way than a DMB but I'm not seeing it, and store-before-load
ordering is hard to understand.
Patch by skrll@ based on a patch by mrg@, soliloquy in commit message
by me.
To generate a diff of this commit:
cvs rdiff -u -r1.4 -r1.5 src/common/lib/libc/arch/aarch64/atomic/Makefile.inc
cvs rdiff -u -r1.6 -r1.7 \
src/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/common/lib/libc/arch/aarch64/atomic/Makefile.inc
diff -u src/common/lib/libc/arch/aarch64/atomic/Makefile.inc:1.4 src/common/lib/libc/arch/aarch64/atomic/Makefile.inc:1.5
--- src/common/lib/libc/arch/aarch64/atomic/Makefile.inc:1.4 Tue Apr 27 09:14:24 2021
+++ src/common/lib/libc/arch/aarch64/atomic/Makefile.inc Sat Aug 6 21:31:33 2022
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile.inc,v 1.4 2021/04/27 09:14:24 skrll Exp $
+# $NetBSD: Makefile.inc,v 1.5 2022/08/06 21:31:33 riastradh Exp $
.if defined(LIB) && (${LIB} == "kern" || ${LIB} == "c" || ${LIB} == "pthread" \
|| ${LIB} == "rump")
@@ -14,7 +14,7 @@ SRCS.atomic+= membar_ops.S
#and cas nand or sub swap xor
.for op in swp cas clr set eor add
.for sz in 1 2 4 8
-.for ar in _relax _acq _rel _acq_rel
+.for ar in _relax _acq _rel _acq_rel _sync
__aarch64_${op}${sz}${ar}.S: __aarch64_lse.S
${_MKTARGET_CREATE}
printf '#define OP ${op}\n#define OP_${op}\n#define SZ ${sz}\n#define AR ${ar}\n#define AR${ar}\n#include "__aarch64_lse.S"\n' > ${.TARGET}
@@ -23,7 +23,7 @@ SRCS.gen+= __aarch64_${op}${sz}${ar}.S
.endfor
.endfor
.for op in casp
-.for ar in _relax _acq _rel _acq_rel
+.for ar in _relax _acq _rel _acq_rel _sync
__aarch64_${op}${ar}.S: __aarch64_lse.S
${_MKTARGET_CREATE}
printf '#define OP ${op}\n#define OP_${op}\n#define AR ${ar}\n#define AR${ar}\n#include "__aarch64_lse.S"\n' > ${.TARGET}
Index: src/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S
diff -u src/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S:1.6 src/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S:1.7
--- src/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S:1.6 Sat Jul 23 13:30:15 2022
+++ src/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S Sat Aug 6 21:31:33 2022
@@ -1,4 +1,4 @@
-/* $NetBSD: __aarch64_lse.S,v 1.6 2022/07/23 13:30:15 skrll Exp $ */
+/* $NetBSD: __aarch64_lse.S,v 1.7 2022/08/06 21:31:33 riastradh Exp $ */
/*-
* Copyright (c) 2021 The NetBSD Foundation, Inc.
@@ -64,21 +64,31 @@
#if defined(AR_relax)
#define ACQ
#define REL
+#define DMB
#endif
#if defined(AR_acq)
#define ACQ a
#define REL
+#define DMB
#endif
#if defined(AR_rel)
#define ACQ
#define REL l
+#define DMB
#endif
#if defined(AR_acq_rel)
#define ACQ a
#define REL l
+#define DMB
+#endif
+
+#if defined(AR_sync)
+#define ACQ
+#define REL
+#define DMB dmb ish
#endif
#if defined(OP_clr)
@@ -134,14 +144,18 @@
ENTRY_NP(SWP_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
SWP R0, R0, [x1]
+ DMB
ret
99:
#endif
mov x4, x0 /* need x0 for return value */
+ DMB /* potential barrier */
1: LDXR R0, [x1] /* load old value */
STXR w3, R4, [x1] /* store new value */
cbnz w3, 2f /* succeed?? no, try again */
+ DMB /* potential barrier */
ret /* return old value */
2: b 1b
END(SWP_FUNC)
@@ -151,16 +165,20 @@ END(SWP_FUNC)
ENTRY_NP(CAS_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
CAS R0, R1, [x2]
+ DMB
ret
99:
#endif
mov x4, x0 /* need x0 for return value */
+ DMB /* potential barrier */
1: LDXR R0, [x2] /* load old value */
cmp R0, R4 /* compare */
b.ne 2f /* not equal? return */
STXR w3, R1, [x2] /* store new value */
cbnz w3, 3f /* succeed? nope, try again. */
+ DMB /* potential barrier */
2: ret /* return. */
3: b 1b
END(CAS_FUNC)
@@ -170,12 +188,15 @@ END(CAS_FUNC)
ENTRY_NP(CASP_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
CASP x0, x1, x2, x3, [x4]
+ DMB
ret
99:
#endif
mov x5, x0 /* need x0 for return value */
mov x6, x1 /* need x1 for return value */
+ DMB /* potential barrier */
1: LDXP x0, x1, [x4] /* load old value */
cmp x0, x5 /* compare */
b.ne 2f /* not equal? return */
@@ -183,6 +204,7 @@ ENTRY_NP(CASP_FUNC)
b.ne 2f /* not equal? return */
STXP w7, x2, x3, [x4] /* store new value */
cbnz w7, 3f /* succeed? nope, try again. */
+ DMB /* potential barrier */
2: ret /* return. */
3: b 1b
END(CASP_FUNC)
@@ -192,15 +214,19 @@ END(CASP_FUNC)
ENTRY_NP(INSN_FUNC)
#ifdef _HAVE_LSE
DO_LSE_INSN_IF_SUPPORTED(99f)
+ DMB
INSN R0, R0, [x1]
+ DMB
ret
99:
#endif
mov x4, x0 /* need x0 for return value */
+ DMB /* potential barrier */
1: LDXR R0, [x1] /* load old value */
INSNOP R4, R0, R4
STXR w3, R4, [x1] /* store new value */
cbnz w3, 2f /* succeed?? no, try again */
+ DMB /* potential barrier */
ret /* return old value */
2: b 1b
END(INSN_FUNC)
