Enable lock-free 128-bit atomics on AArch64. This is backwards compatible with
existing binaries, gives better performance than locking atomics and is what
most users expect.
Note 128-bit atomic loads use a load/store exclusive loop if LSE2 is not
supported.
This results in an implicit store which is invisible to software as long as the
given
address is writeable (which will be true when using atomics in actual code).
A simple test on an old Cortex-A72 showed 2.7x speedup of 128-bit atomics.
Passes regress, OK for commit?
libatomic/
PR target/110061
config/linux/aarch64/atomic_16.S: Implement lock-free ARMv8.0 atomics.
config/linux/aarch64/host-config.h: Use atomic_16.S for baseline v8.0.
State we have lock-free atomics.
---
diff --git a/libatomic/config/linux/aarch64/atomic_16.S
b/libatomic/config/linux/aarch64/atomic_16.S
index
05439ce394b9653c9bcb582761ff7aaa7c8f9643..0485c284117edf54f41959d2fab9341a9567b1cf
100644
--- a/libatomic/config/linux/aarch64/atomic_16.S
+++ b/libatomic/config/linux/aarch64/atomic_16.S
@@ -22,6 +22,21 @@
<http://www.gnu.org/licenses/>. */
+/* AArch64 128-bit lock-free atomic implementation.
+
+ 128-bit atomics are now lock-free for all AArch64 architecture versions.
+ This is backwards compatible with existing binaries and gives better
+ performance than locking atomics.
+
+ 128-bit atomic loads use a exclusive loop if LSE2 is not supported.
+ This results in an implicit store which is invisible to software as long
+ as the given address is writeable. Since all other atomics have explicit
+ writes, this will be true when using atomics in actual code.
+
+ The libat_<op>_16 entry points are ARMv8.0.
+ The libat_<op>_16_i1 entry points are used when LSE2 is available. */
+
+
.arch armv8-a+lse
#define ENTRY(name) \
@@ -37,6 +52,10 @@ name: \
.cfi_endproc; \
.size name, .-name;
+#define ALIAS(alias,name) \
+ .global alias; \
+ .set alias, name;
+
#define res0 x0
#define res1 x1
#define in0 x2
@@ -70,6 +89,24 @@ name: \
#define SEQ_CST 5
+ENTRY (libat_load_16)
+ mov x5, x0
+ cbnz w1, 2f
+
+ /* RELAXED. */
+1: ldxp res0, res1, [x5]
+ stxp w4, res0, res1, [x5]
+ cbnz w4, 1b
+ ret
+
+ /* ACQUIRE/CONSUME/SEQ_CST. */
+2: ldaxp res0, res1, [x5]
+ stxp w4, res0, res1, [x5]
+ cbnz w4, 2b
+ ret
+END (libat_load_16)
+
+
ENTRY (libat_load_16_i1)
cbnz w1, 1f
@@ -93,6 +130,23 @@ ENTRY (libat_load_16_i1)
END (libat_load_16_i1)
+ENTRY (libat_store_16)
+ cbnz w4, 2f
+
+ /* RELAXED. */
+1: ldxp xzr, tmp0, [x0]
+ stxp w4, in0, in1, [x0]
+ cbnz w4, 1b
+ ret
+
+ /* RELEASE/SEQ_CST. */
+2: ldxp xzr, tmp0, [x0]
+ stlxp w4, in0, in1, [x0]
+ cbnz w4, 2b
+ ret
+END (libat_store_16)
+
+
ENTRY (libat_store_16_i1)
cbnz w4, 1f
@@ -101,14 +155,14 @@ ENTRY (libat_store_16_i1)
ret
/* RELEASE/SEQ_CST. */
-1: ldaxp xzr, tmp0, [x0]
+1: ldxp xzr, tmp0, [x0]
stlxp w4, in0, in1, [x0]
cbnz w4, 1b
ret
END (libat_store_16_i1)
-ENTRY (libat_exchange_16_i1)
+ENTRY (libat_exchange_16)
mov x5, x0
cbnz w4, 2f
@@ -126,22 +180,55 @@ ENTRY (libat_exchange_16_i1)
stxp w4, in0, in1, [x5]
cbnz w4, 3b
ret
-4:
- cmp w4, RELEASE
- b.ne 6f
- /* RELEASE. */
-5: ldxp res0, res1, [x5]
+ /* RELEASE/ACQ_REL/SEQ_CST. */
+4: ldaxp res0, res1, [x5]
stlxp w4, in0, in1, [x5]
- cbnz w4, 5b
+ cbnz w4, 4b
ret
+END (libat_exchange_16)
- /* ACQ_REL/SEQ_CST. */
-6: ldaxp res0, res1, [x5]
- stlxp w4, in0, in1, [x5]
- cbnz w4, 6b
+
+ENTRY (libat_compare_exchange_16)
+ ldp exp0, exp1, [x1]
+ cbz w4, 3f
+ cmp w4, RELEASE
+ b.hs 4f
+
+ /* ACQUIRE/CONSUME. */
+1: ldaxp tmp0, tmp1, [x0]
+ cmp tmp0, exp0
+ ccmp tmp1, exp1, 0, eq
+ bne 2f
+ stxp w4, in0, in1, [x0]
+ cbnz w4, 1b
+ mov x0, 1
ret
-END (libat_exchange_16_i1)
+
+2: stp tmp0, tmp1, [x1]
+ mov x0, 0
+ ret
+
+ /* RELAXED. */
+3: ldxp tmp0, tmp1, [x0]
+ cmp tmp0, exp0
+ ccmp tmp1, exp1, 0, eq
+ bne 2b
+ stxp w4, in0, in1, [x0]
+ cbnz w4, 3b
+ mov x0, 1
+ ret
+
+ /* RELEASE/ACQ_REL/SEQ_CST. */
+4: ldaxp tmp0, tmp1, [x0]
+ cmp tmp0, exp0
+ ccmp tmp1, exp1, 0, eq
+ bne 2b
+ stlxp w4, in0, in1, [x0]
+ cbnz w4, 4b
+ mov x0, 1
+ ret
+END (libat_compare_exchange_16)
ENTRY (libat_compare_exchange_16_i1)
@@ -180,7 +267,7 @@ ENTRY (libat_compare_exchange_16_i1)
END (libat_compare_exchange_16_i1)
-ENTRY (libat_fetch_add_16_i1)
+ENTRY (libat_fetch_add_16)
mov x5, x0
cbnz w4, 2f
@@ -199,10 +286,10 @@ ENTRY (libat_fetch_add_16_i1)
stlxp w4, tmp0, tmp1, [x5]
cbnz w4, 2b
ret
-END (libat_fetch_add_16_i1)
+END (libat_fetch_add_16)
-ENTRY (libat_add_fetch_16_i1)
+ENTRY (libat_add_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -221,10 +308,10 @@ ENTRY (libat_add_fetch_16_i1)
stlxp w4, res0, res1, [x5]
cbnz w4, 2b
ret
-END (libat_add_fetch_16_i1)
+END (libat_add_fetch_16)
-ENTRY (libat_fetch_sub_16_i1)
+ENTRY (libat_fetch_sub_16)
mov x5, x0
cbnz w4, 2f
@@ -243,10 +330,10 @@ ENTRY (libat_fetch_sub_16_i1)
stlxp w4, tmp0, tmp1, [x5]
cbnz w4, 2b
ret
-END (libat_fetch_sub_16_i1)
+END (libat_fetch_sub_16)
-ENTRY (libat_sub_fetch_16_i1)
+ENTRY (libat_sub_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -265,10 +352,10 @@ ENTRY (libat_sub_fetch_16_i1)
stlxp w4, res0, res1, [x5]
cbnz w4, 2b
ret
-END (libat_sub_fetch_16_i1)
+END (libat_sub_fetch_16)
-ENTRY (libat_fetch_or_16_i1)
+ENTRY (libat_fetch_or_16)
mov x5, x0
cbnz w4, 2f
@@ -287,10 +374,10 @@ ENTRY (libat_fetch_or_16_i1)
stlxp w4, tmp0, tmp1, [x5]
cbnz w4, 2b
ret
-END (libat_fetch_or_16_i1)
+END (libat_fetch_or_16)
-ENTRY (libat_or_fetch_16_i1)
+ENTRY (libat_or_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -309,10 +396,10 @@ ENTRY (libat_or_fetch_16_i1)
stlxp w4, res0, res1, [x5]
cbnz w4, 2b
ret
-END (libat_or_fetch_16_i1)
+END (libat_or_fetch_16)
-ENTRY (libat_fetch_and_16_i1)
+ENTRY (libat_fetch_and_16)
mov x5, x0
cbnz w4, 2f
@@ -331,10 +418,10 @@ ENTRY (libat_fetch_and_16_i1)
stlxp w4, tmp0, tmp1, [x5]
cbnz w4, 2b
ret
-END (libat_fetch_and_16_i1)
+END (libat_fetch_and_16)
-ENTRY (libat_and_fetch_16_i1)
+ENTRY (libat_and_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -353,10 +440,10 @@ ENTRY (libat_and_fetch_16_i1)
stlxp w4, res0, res1, [x5]
cbnz w4, 2b
ret
-END (libat_and_fetch_16_i1)
+END (libat_and_fetch_16)
-ENTRY (libat_fetch_xor_16_i1)
+ENTRY (libat_fetch_xor_16)
mov x5, x0
cbnz w4, 2f
@@ -375,10 +462,10 @@ ENTRY (libat_fetch_xor_16_i1)
stlxp w4, tmp0, tmp1, [x5]
cbnz w4, 2b
ret
-END (libat_fetch_xor_16_i1)
+END (libat_fetch_xor_16)
-ENTRY (libat_xor_fetch_16_i1)
+ENTRY (libat_xor_fetch_16)
mov x5, x0
cbnz w4, 2f
@@ -397,10 +484,10 @@ ENTRY (libat_xor_fetch_16_i1)
stlxp w4, res0, res1, [x5]
cbnz w4, 2b
ret
-END (libat_xor_fetch_16_i1)
+END (libat_xor_fetch_16)
-ENTRY (libat_fetch_nand_16_i1)
+ENTRY (libat_fetch_nand_16)
mov x5, x0
mvn in0, in0
mvn in1, in1
@@ -421,10 +508,10 @@ ENTRY (libat_fetch_nand_16_i1)
stlxp w4, tmp0, tmp1, [x5]
cbnz w4, 2b
ret
-END (libat_fetch_nand_16_i1)
+END (libat_fetch_nand_16)
-ENTRY (libat_nand_fetch_16_i1)
+ENTRY (libat_nand_fetch_16)
mov x5, x0
mvn in0, in0
mvn in1, in1
@@ -445,21 +532,38 @@ ENTRY (libat_nand_fetch_16_i1)
stlxp w4, res0, res1, [x5]
cbnz w4, 2b
ret
-END (libat_nand_fetch_16_i1)
+END (libat_nand_fetch_16)
-ENTRY (libat_test_and_set_16_i1)
- mov w2, 1
- cbnz w1, 2f
-
- /* RELAXED. */
- swpb w0, w2, [x0]
- ret
+/* __atomic_test_and_set is always inlined, so this entry is unused and
+ only required for completeness. */
+ENTRY (libat_test_and_set_16)
- /* ACQUIRE/CONSUME/RELEASE/ACQ_REL/SEQ_CST. */
-2: swpalb w0, w2, [x0]
+ /* RELAXED/ACQUIRE/CONSUME/RELEASE/ACQ_REL/SEQ_CST. */
+ mov x5, x0
+1: ldaxrb w0, [x5]
+ stlxrb w4, w2, [x5]
+ cbnz w4, 1b
ret
-END (libat_test_and_set_16_i1)
+END (libat_test_and_set_16)
+
+
+/* Alias entry points which are the same in baseline and LSE2. */
+
+ALIAS (libat_exchange_16_i1, libat_exchange_16)
+ALIAS (libat_fetch_add_16_i1, libat_fetch_add_16)
+ALIAS (libat_add_fetch_16_i1, libat_add_fetch_16)
+ALIAS (libat_fetch_sub_16_i1, libat_fetch_sub_16)
+ALIAS (libat_sub_fetch_16_i1, libat_sub_fetch_16)
+ALIAS (libat_fetch_or_16_i1, libat_fetch_or_16)
+ALIAS (libat_or_fetch_16_i1, libat_or_fetch_16)
+ALIAS (libat_fetch_and_16_i1, libat_fetch_and_16)
+ALIAS (libat_and_fetch_16_i1, libat_and_fetch_16)
+ALIAS (libat_fetch_xor_16_i1, libat_fetch_xor_16)
+ALIAS (libat_xor_fetch_16_i1, libat_xor_fetch_16)
+ALIAS (libat_fetch_nand_16_i1, libat_fetch_nand_16)
+ALIAS (libat_nand_fetch_16_i1, libat_nand_fetch_16)
+ALIAS (libat_test_and_set_16_i1, libat_test_and_set_16)
/* GNU_PROPERTY_AARCH64_* macros from elf.h for use in asm code. */
diff --git a/libatomic/config/linux/aarch64/host-config.h
b/libatomic/config/linux/aarch64/host-config.h
index
bea26825b4f75bb8ff348ab4b5fc45f4a5bd561e..851c78c01cd643318aaa52929ce4550266238b79
100644
--- a/libatomic/config/linux/aarch64/host-config.h
+++ b/libatomic/config/linux/aarch64/host-config.h
@@ -35,10 +35,19 @@
#endif
#define IFUNC_NCOND(N) (1)
-#if N == 16 && IFUNC_ALT != 0
+#endif /* HAVE_IFUNC */
+
+/* All 128-bit atomic functions are defined in aarch64/atomic_16.S. */
+#if N == 16
# define DONE 1
#endif
-#endif /* HAVE_IFUNC */
+/* State we have lock-free 128-bit atomics. */
+#undef FAST_ATOMIC_LDST_16
+#define FAST_ATOMIC_LDST_16 1
+#undef MAYBE_HAVE_ATOMIC_CAS_16
+#define MAYBE_HAVE_ATOMIC_CAS_16 1
+#undef MAYBE_HAVE_ATOMIC_EXCHANGE_16
+#define MAYBE_HAVE_ATOMIC_EXCHANGE_16 1
#include_next <host-config.h>
