Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/common/lib/libc/arch/arm/atomic Remove memory barriers from ...
details: https://anonhg.NetBSD.org/src/rev/1c36634021fd
branches: trunk
changeset: 984887:1c36634021fd
user: skrll <skrll%NetBSD.org@localhost>
date: Wed Jul 28 07:32:20 2021 +0000
description:
Remove memory barriers from the atomic_ops(3) atomic operations. They're
not needed for correctness.
Add the correct memory barriers to the gcc legacy __sync built-in
functions for atomic memory access. From the gcc documentation:
In most cases, these built-in functions are considered a full barrier.
That is, no memory operand is moved across the operation, either forward
or backward. Further, instructions are issued as necessary to prevent the
processor from speculating loads across the operation and from queuing
stores after the operation.
type __sync_lock_test_and_set (type *ptr, type value, ...)
This built-in function is not a full barrier, but rather an acquire
barrier. This means that references after the operation cannot move to
(or be speculated to) before the operation, but previous memory stores
may not be globally visible yet, and previous memory loads may not yet
be satisfied.
void __sync_lock_release (type *ptr, ...)
This built-in function is not a full barrier, but rather a release
barrier. This means that all previous memory stores are globally
visible, and all previous memory loads have been satisfied, but
following memory reads are not prevented from being speculated to
before the barrier.
diffstat:
common/lib/libc/arch/arm/atomic/atomic_add_16.S | 52 ++++++---
common/lib/libc/arch/arm/atomic/atomic_add_32.S | 51 ++++++--
common/lib/libc/arch/arm/atomic/atomic_add_64.S | 16 +-
common/lib/libc/arch/arm/atomic/atomic_add_8.S | 51 ++++++--
common/lib/libc/arch/arm/atomic/atomic_and_16.S | 32 +++--
common/lib/libc/arch/arm/atomic/atomic_and_32.S | 31 +++--
common/lib/libc/arch/arm/atomic/atomic_and_64.S | 16 +-
common/lib/libc/arch/arm/atomic/atomic_and_8.S | 32 +++--
common/lib/libc/arch/arm/atomic/atomic_cas_16.S | 17 +-
common/lib/libc/arch/arm/atomic/atomic_cas_32.S | 17 +-
common/lib/libc/arch/arm/atomic/atomic_cas_64.S | 17 +-
common/lib/libc/arch/arm/atomic/atomic_cas_8.S | 17 +-
common/lib/libc/arch/arm/atomic/atomic_dec_32.S | 13 +--
common/lib/libc/arch/arm/atomic/atomic_dec_64.S | 7 +-
common/lib/libc/arch/arm/atomic/atomic_inc_32.S | 14 +--
common/lib/libc/arch/arm/atomic/atomic_inc_64.S | 7 +-
common/lib/libc/arch/arm/atomic/atomic_nand_16.S | 32 +++--
common/lib/libc/arch/arm/atomic/atomic_nand_32.S | 32 +++--
common/lib/libc/arch/arm/atomic/atomic_nand_64.S | 16 +-
common/lib/libc/arch/arm/atomic/atomic_nand_8.S | 32 +++--
common/lib/libc/arch/arm/atomic/atomic_op_asm.h | 10 +-
common/lib/libc/arch/arm/atomic/atomic_or_16.S | 31 +++--
common/lib/libc/arch/arm/atomic/atomic_or_32.S | 32 +++--
common/lib/libc/arch/arm/atomic/atomic_or_64.S | 16 +-
common/lib/libc/arch/arm/atomic/atomic_or_8.S | 31 +++--
common/lib/libc/arch/arm/atomic/atomic_sub_64.S | 17 +-
common/lib/libc/arch/arm/atomic/atomic_swap.S | 38 ++----
common/lib/libc/arch/arm/atomic/atomic_swap_16.S | 20 +--
common/lib/libc/arch/arm/atomic/atomic_swap_64.S | 22 +--
common/lib/libc/arch/arm/atomic/atomic_xor_16.S | 31 +++--
common/lib/libc/arch/arm/atomic/atomic_xor_32.S | 31 +++--
common/lib/libc/arch/arm/atomic/atomic_xor_64.S | 7 +-
common/lib/libc/arch/arm/atomic/atomic_xor_8.S | 31 +++--
common/lib/libc/arch/arm/atomic/membar_ops.S | 18 +--
common/lib/libc/arch/arm/atomic/sync_bool_compare_and_swap_1.S | 8 +-
common/lib/libc/arch/arm/atomic/sync_bool_compare_and_swap_2.S | 8 +-
common/lib/libc/arch/arm/atomic/sync_bool_compare_and_swap_4.S | 8 +-
common/lib/libc/arch/arm/atomic/sync_bool_compare_and_swap_8.S | 8 +-
common/lib/libc/arch/arm/atomic/sync_fetch_and_add_8.S | 9 +-
common/lib/libc/arch/arm/atomic/sync_fetch_and_and_8.S | 9 +-
common/lib/libc/arch/arm/atomic/sync_fetch_and_nand_8.S | 9 +-
common/lib/libc/arch/arm/atomic/sync_fetch_and_or_8.S | 9 +-
common/lib/libc/arch/arm/atomic/sync_fetch_and_sub_8.S | 9 +-
common/lib/libc/arch/arm/atomic/sync_fetch_and_xor_8.S | 9 +-
44 files changed, 496 insertions(+), 427 deletions(-)
diffs (truncated from 2179 to 300 lines):
diff -r 18e9559ddf2c -r 1c36634021fd common/lib/libc/arch/arm/atomic/atomic_add_16.S
--- a/common/lib/libc/arch/arm/atomic/atomic_add_16.S Wed Jul 28 00:59:10 2021 +0000
+++ b/common/lib/libc/arch/arm/atomic/atomic_add_16.S Wed Jul 28 07:32:20 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_16.S,v 1.4 2021/07/10 06:53:40 skrll Exp $ */
+/* $NetBSD: atomic_add_16.S,v 1.5 2021/07/28 07:32:20 skrll Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -43,11 +43,6 @@
strexh r2, r3, [ip] /* try to store */
cmp r2, #0 /* succeed? */
bne 1b /* no, try again */
-#ifdef _ARM_ARCH_7
- dmb
-#else
- mcr p15, 0, r2, c7, c10, 5 /* data memory barrier */
-#endif
RET /* return old value */
END(_atomic_add_16)
END(_atomic_sub_16)
@@ -55,7 +50,6 @@
ATOMIC_OP_ALIAS(atomic_add_16,_atomic_add_16)
ATOMIC_OP_ALIAS(atomic_add_short,_atomic_add_16)
ATOMIC_OP_ALIAS(atomic_add_ushort,_atomic_add_16)
-CRT_ALIAS(__sync_fetch_and_add_2,_atomic_add_16)
CRT_ALIAS(__atomic_fetch_add_2,_atomic_add_16)
STRONG_ALIAS(_atomic_add_short,_atomic_add_16)
STRONG_ALIAS(_atomic_add_ushort,_atomic_add_16)
@@ -63,11 +57,27 @@
ATOMIC_OP_ALIAS(atomic_sub_16,_atomic_sub_16)
ATOMIC_OP_ALIAS(atomic_sub_short,_atomic_sub_16)
ATOMIC_OP_ALIAS(atomic_sub_ushort,_atomic_sub_16)
-CRT_ALIAS(__sync_fetch_and_sub_2,_atomic_sub_16)
CRT_ALIAS(__atomic_fetch_sub_2,_atomic_sub_16)
STRONG_ALIAS(_atomic_sub_short,_atomic_sub_16)
STRONG_ALIAS(_atomic_sub_ushort,_atomic_sub_16)
+ENTRY_NP(__sync_fetch_and_add_2)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_16
+ DMB
+ pop {r4, pc}
+END(__sync_fetch_and_add_2)
+
+ENTRY_NP(__sync_fetch_and_sub_2)
+ push {r4, lr}
+ DMB
+ bl _atomic_sub_16
+ DMB
+ pop {r4, pc}
+END(__sync_fetch_and_sub_2)
+
+
ENTRY_NP(_atomic_sub_16_nv)
negs r1, r1
/* FALLTHROUGH */
@@ -78,25 +88,35 @@
strexh r2, r0, [ip] /* try to store */
cmp r2, #0 /* succeed? */
bne 1b /* no, try again? */
-#ifdef _ARM_ARCH_7
- dmb
-#else
- mcr p15, 0, r2, c7, c10, 5 /* data memory barrier */
-#endif
RET /* return new value */
END(_atomic_add_16_nv)
END(_atomic_sub_16_nv)
+
ATOMIC_OP_ALIAS(atomic_add_16_nv,_atomic_add_16_nv)
ATOMIC_OP_ALIAS(atomic_add_short_nv,_atomic_add_16_nv)
ATOMIC_OP_ALIAS(atomic_add_ushort_nv,_atomic_add_16_nv)
-CRT_ALIAS(__sync_add_and_fetch_2,_atomic_add_16_nv)
STRONG_ALIAS(_atomic_add_short_nv,_atomic_add_16_nv)
STRONG_ALIAS(_atomic_add_ushort_nv,_atomic_add_16_nv)
-
ATOMIC_OP_ALIAS(atomic_sub_16_nv,_atomic_sub_16_nv)
ATOMIC_OP_ALIAS(atomic_sub_short_nv,_atomic_sub_16_nv)
ATOMIC_OP_ALIAS(atomic_sub_ushort_nv,_atomic_sub_16_nv)
-CRT_ALIAS(__sync_sub_and_fetch_2,_atomic_sub_16_nv)
STRONG_ALIAS(_atomic_sub_short_nv,_atomic_sub_16_nv)
STRONG_ALIAS(_atomic_sub_ushort_nv,_atomic_sub_16_nv)
+
+ENTRY_NP(__sync_add_and_fetch_2)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_16_nv
+ DMB
+ pop {r4, pc}
+END(__sync_add_and_fetch_2)
+
+ENTRY_NP(__sync_sub_and_fetch_2)
+ push {r4, lr}
+ DMB
+ bl _atomic_sub_16_nv
+ DMB
+ pop {r4, pc}
+END(__sync_sub_and_fetch_2)
+
#endif /* _ARM_ARCH_6 */
diff -r 18e9559ddf2c -r 1c36634021fd common/lib/libc/arch/arm/atomic/atomic_add_32.S
--- a/common/lib/libc/arch/arm/atomic/atomic_add_32.S Wed Jul 28 00:59:10 2021 +0000
+++ b/common/lib/libc/arch/arm/atomic/atomic_add_32.S Wed Jul 28 07:32:20 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_32.S,v 1.9 2021/07/10 06:53:40 skrll Exp $ */
+/* $NetBSD: atomic_add_32.S,v 1.10 2021/07/28 07:32:20 skrll Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -43,20 +43,31 @@
strex r2, r3, [ip] /* try to store */
cmp r2, #0 /* succeed? */
bne 1b /* no, try again */
-#ifdef _ARM_ARCH_7
- dmb
-#else
- mcr p15, 0, r2, c7, c10, 5 /* data memory barrier */
-#endif
RET /* return old value */
END(_atomic_add_32)
END(_atomic_sub_32)
+ENTRY_NP(__sync_fetch_and_add_4)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_32
+ DMB
+ pop {r4, pc}
+END(__sync_fetch_and_add_4)
+
+ENTRY_NP(__sync_fetch_and_sub_4)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_32
+ DMB
+ pop {r4, pc}
+END(__sync_fetch_and_sub_4)
+
+
ATOMIC_OP_ALIAS(atomic_add_32,_atomic_add_32)
ATOMIC_OP_ALIAS(atomic_add_int,_atomic_add_32)
ATOMIC_OP_ALIAS(atomic_add_long,_atomic_add_32)
ATOMIC_OP_ALIAS(atomic_add_ptr,_atomic_add_32)
-CRT_ALIAS(__sync_fetch_and_add_4,_atomic_add_32)
CRT_ALIAS(__atomic_fetch_add_4,_atomic_add_32)
STRONG_ALIAS(_atomic_add_int,_atomic_add_32)
STRONG_ALIAS(_atomic_add_long,_atomic_add_32)
@@ -66,7 +77,6 @@
ATOMIC_OP_ALIAS(atomic_sub_int,_atomic_sub_32)
ATOMIC_OP_ALIAS(atomic_sub_long,_atomic_sub_32)
ATOMIC_OP_ALIAS(atomic_sub_ptr,_atomic_sub_32)
-CRT_ALIAS(__sync_fetch_and_sub_4,_atomic_sub_32)
CRT_ALIAS(__atomic_fetch_sub_4,_atomic_sub_32)
STRONG_ALIAS(_atomic_sub_int,_atomic_sub_32)
STRONG_ALIAS(_atomic_sub_long,_atomic_sub_32)
@@ -82,19 +92,14 @@
strex r2, r0, [ip] /* try to store */
cmp r2, #0 /* succeed? */
bne 1b /* no, try again? */
-#ifdef _ARM_ARCH_7
- dmb
-#else
- mcr p15, 0, r2, c7, c10, 5 /* data memory barrier */
-#endif
RET /* return new value */
END(_atomic_add_32_nv)
END(_atomic_sub_32_nv)
+
ATOMIC_OP_ALIAS(atomic_add_32_nv,_atomic_add_32_nv)
ATOMIC_OP_ALIAS(atomic_add_int_nv,_atomic_add_32_nv)
ATOMIC_OP_ALIAS(atomic_add_long_nv,_atomic_add_32_nv)
ATOMIC_OP_ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv)
-CRT_ALIAS(__sync_add_and_fetch_4,_atomic_add_32_nv)
STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv)
STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv)
STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv)
@@ -103,8 +108,24 @@
ATOMIC_OP_ALIAS(atomic_sub_int_nv,_atomic_sub_32_nv)
ATOMIC_OP_ALIAS(atomic_sub_long_nv,_atomic_sub_32_nv)
ATOMIC_OP_ALIAS(atomic_sub_ptr_nv,_atomic_sub_32_nv)
-CRT_ALIAS(__sync_sub_and_fetch_4,_atomic_sub_32_nv)
STRONG_ALIAS(_atomic_sub_int_nv,_atomic_sub_32_nv)
STRONG_ALIAS(_atomic_sub_long_nv,_atomic_sub_32_nv)
STRONG_ALIAS(_atomic_sub_ptr_nv,_atomic_sub_32_nv)
+
+ENTRY_NP(__sync_add_and_fetch_4)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_32_nv
+ DMB
+ pop {r4, pc}
+END(__sync_add_and_fetch_4)
+
+ENTRY_NP(__sync_sub_and_fetch_4)
+ push {r4, lr}
+ DMB
+ bl _atomic_sub_32_nv
+ DMB
+ pop {r4, pc}
+END(__sync_sub_and_fetch_4)
+
#endif /* _ARM_ARCH_6 */
diff -r 18e9559ddf2c -r 1c36634021fd common/lib/libc/arch/arm/atomic/atomic_add_64.S
--- a/common/lib/libc/arch/arm/atomic/atomic_add_64.S Wed Jul 28 00:59:10 2021 +0000
+++ b/common/lib/libc/arch/arm/atomic/atomic_add_64.S Wed Jul 28 07:32:20 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_64.S,v 1.13 2021/07/10 06:53:40 skrll Exp $ */
+/* $NetBSD: atomic_add_64.S,v 1.14 2021/07/28 07:32:20 skrll Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -45,11 +45,6 @@
strexd r4, r0, r1, [ip] /* try to store */
cmp r4, #0 /* succeed? */
bne 1b /* no, try again? */
-#ifdef _ARM_ARCH_7
- dmb
-#else
- mcr p15, 0, r4, c7, c10, 5 /* data memory barrier */
-#endif
pop {r3, r4} /* restore temporary */
RET /* return new value */
END(_atomic_add_64_nv)
@@ -57,6 +52,13 @@
STRONG_ALIAS(_atomic_add_64,_atomic_add_64_nv)
ATOMIC_OP_ALIAS(atomic_add_64_nv,_atomic_add_64_nv)
ATOMIC_OP_ALIAS(atomic_add_64,_atomic_add_64)
-CRT_ALIAS(__sync_add_and_fetch_8,_atomic_add_64_nv)
+
+ENTRY_NP(__sync_add_and_fetch_8)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_64_nv
+ DMB
+ pop {r4, pc}
+END(__sync_add_and_fetch_8)
#endif /* _ARM_ARCH_6 */
diff -r 18e9559ddf2c -r 1c36634021fd common/lib/libc/arch/arm/atomic/atomic_add_8.S
--- a/common/lib/libc/arch/arm/atomic/atomic_add_8.S Wed Jul 28 00:59:10 2021 +0000
+++ b/common/lib/libc/arch/arm/atomic/atomic_add_8.S Wed Jul 28 07:32:20 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_8.S,v 1.4 2021/07/10 06:53:40 skrll Exp $ */
+/* $NetBSD: atomic_add_8.S,v 1.5 2021/07/28 07:32:20 skrll Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -43,11 +43,6 @@
strexb r2, r3, [ip] /* try to store */
cmp r2, #0 /* succeed? */
bne 1b /* no, try again */
-#ifdef _ARM_ARCH_7
- dmb
-#else
- mcr p15, 0, r2, c7, c10, 5 /* data memory barrier */
-#endif
RET /* return old value */
END(_atomic_add_8)
END(_atomic_sub_8)
@@ -55,19 +50,34 @@
ATOMIC_OP_ALIAS(atomic_add_8,_atomic_add_8)
ATOMIC_OP_ALIAS(atomic_add_char,_atomic_add_8)
ATOMIC_OP_ALIAS(atomic_add_uchar,_atomic_add_8)
-CRT_ALIAS(__sync_fetch_and_add_1,_atomic_add_8)
CRT_ALIAS(__atomic_fetch_add_1,_atomic_add_8)
STRONG_ALIAS(_atomic_add_char,_atomic_add_8)
STRONG_ALIAS(_atomic_add_uchar,_atomic_add_8)
+ENTRY_NP(__sync_fetch_and_add_1)
+ push {r4, lr}
+ DMB
+ bl _atomic_add_8
+ DMB
+ pop {r4, pc}
+END(__sync_fetch_and_add_1)
+
+ENTRY_NP(__sync_fetch_and_sub_1)
+ push {r4, lr}
+ DMB
+ bl _atomic_sub_8
+ DMB
+ pop {r4, pc}
+END(__sync_fetch_and_sub_1)
+
ATOMIC_OP_ALIAS(atomic_sub_8,_atomic_sub_8)
ATOMIC_OP_ALIAS(atomic_sub_char,_atomic_sub_8)
ATOMIC_OP_ALIAS(atomic_sub_uchar,_atomic_sub_8)
-CRT_ALIAS(__sync_fetch_and_sub_1,_atomic_sub_8)
CRT_ALIAS(__atomic_fetch_sub_1,_atomic_sub_8)
STRONG_ALIAS(_atomic_sub_char,_atomic_sub_8)
STRONG_ALIAS(_atomic_sub_uchar,_atomic_sub_8)
+
ENTRY_NP(_atomic_sub_8_nv)
negs r1, r1
/* FALLTHROUGH */
@@ -78,25 +88,36 @@
strexb r2, r0, [ip] /* try to store */
Home |
Main Index |
Thread Index |
Old Index