Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/common/lib/libc/arch/aarch64/atomic As we're providing the l...
details: https://anonhg.NetBSD.org/src/rev/6f602f7d3243
branches: trunk
changeset: 984899:6f602f7d3243
user: skrll <skrll%NetBSD.org@localhost>
date: Thu Jul 29 10:29:05 2021 +0000
description:
As we're providing the legacy gcc __sync built-in functions for atomic
memory access we might as well get the memory barriers right...
>From the gcc documentation:
In most cases, these built-in functions are considered a full barrier.
That is, no memory operand is moved across the operation, either forward
or backward. Further, instructions are issued as necessary to prevent the
processor from speculating loads across the operation and from queuing
stores after the operation.
type __sync_lock_test_and_set (type *ptr, type value, ...)
This built-in function is not a full barrier, but rather an acquire
barrier. This means that references after the operation cannot move to
(or be speculated to) before the operation, but previous memory stores
may not be globally visible yet, and previous memory loads may not yet
be satisfied.
void __sync_lock_release (type *ptr, ...)
This built-in function is not a full barrier, but rather a release
barrier. This means that all previous memory stores are globally
visible, and all previous memory loads have been satisfied, but
following memory reads are not prevented from being speculated to
before the barrier.
diffstat:
common/lib/libc/arch/aarch64/atomic/atomic_add_16.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_add_32.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_add_64.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_add_8.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_and_16.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_and_32.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_and_64.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_and_8.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S | 15 ++-
common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S | 16 ++-
common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S | 16 ++-
common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S | 16 ++-
common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S | 32 +++++-
common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S | 40 ++++++-
common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S | 32 +++++-
common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S | 33 +++++-
common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h | 98 +++++++++++++++++++-
common/lib/libc/arch/aarch64/atomic/atomic_or_16.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_or_32.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_or_64.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_or_8.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S | 12 ++-
common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S | 12 ++-
common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S | 12 ++-
common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S | 12 ++-
common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S | 8 +-
common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S | 8 +-
33 files changed, 410 insertions(+), 96 deletions(-)
diffs (truncated from 1220 to 300 lines):
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_16.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP16(add, add)
+SYNC_FETCH_OP16(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_16,_atomic_add_16)
ATOMIC_OP_ALIAS(atomic_add_short,_atomic_add_16)
-STRONG_ALIAS(__sync_fetch_and_add_2,_atomic_add_16)
STRONG_ALIAS(_atomic_add_short,_atomic_add_16)
ATOMIC_OP16_NV(add, add)
+SYNC_OP16_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_16_nv,_atomic_add_16_nv)
ATOMIC_OP_ALIAS(atomic_add_short_nv,_atomic_add_16_nv)
-STRONG_ALIAS(__sync_add_and_fetch_2,_atomic_add_16_nv)
STRONG_ALIAS(_atomic_add_short_nv,_atomic_add_16_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_32.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(add, add)
+SYNC_FETCH_OP32(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_32,_atomic_add_32)
ATOMIC_OP_ALIAS(atomic_add_int,_atomic_add_32)
-STRONG_ALIAS(__sync_fetch_and_add_4,_atomic_add_32)
STRONG_ALIAS(_atomic_add_int,_atomic_add_32)
ATOMIC_OP32_NV(add, add)
+SYNC_OP32_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_32_nv,_atomic_add_32_nv)
ATOMIC_OP_ALIAS(atomic_add_int_nv,_atomic_add_32_nv)
-STRONG_ALIAS(__sync_add_and_fetch_4,_atomic_add_32_nv)
STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_64.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,18 +33,20 @@
ATOMIC_OP64(add, add)
+SYNC_FETCH_OP64(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_64_nv,_atomic_add_64_nv)
ATOMIC_OP_ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv)
ATOMIC_OP_ALIAS(atomic_add_long_nv,_atomic_add_64_nv)
STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv)
STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv)
-STRONG_ALIAS(__sync_add_and_fetch_8,_atomic_add_64_nv)
ATOMIC_OP64_NV(add, add)
+SYNC_OP64_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_64,_atomic_add_64)
ATOMIC_OP_ALIAS(atomic_add_ptr,_atomic_add_64)
ATOMIC_OP_ALIAS(atomic_add_long,_atomic_add_64)
STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64)
STRONG_ALIAS(_atomic_add_long,_atomic_add_64)
-STRONG_ALIAS(__sync_fetch_and_add_8,_atomic_add_64)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_add_8.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_add_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_add_8.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(add, add)
+SYNC_FETCH_OP8(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_8,_atomic_add_8)
ATOMIC_OP_ALIAS(atomic_add_char,_atomic_add_8)
-STRONG_ALIAS(__sync_fetch_and_add_1,_atomic_add_8)
STRONG_ALIAS(_atomic_add_char,_atomic_add_8)
ATOMIC_OP8_NV(add, add)
+SYNC_OP8_FETCH(add, add)
+
ATOMIC_OP_ALIAS(atomic_add_8_nv,_atomic_add_8_nv)
ATOMIC_OP_ALIAS(atomic_add_char_nv,_atomic_add_8_nv)
-STRONG_ALIAS(__sync_add_and_fetch_1,_atomic_add_8_nv)
STRONG_ALIAS(_atomic_add_char_nv,_atomic_add_8_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_16.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP16(and, and)
+SYNC_FETCH_OP16(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_16,_atomic_and_16)
ATOMIC_OP_ALIAS(atomic_and_ushort,_atomic_and_16)
-STRONG_ALIAS(__sync_fetch_and_and_2,_atomic_and_16)
STRONG_ALIAS(_atomic_and_ushort,_atomic_and_16)
ATOMIC_OP16_NV(and, and)
+SYNC_OP16_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_16_nv,_atomic_and_16_nv)
ATOMIC_OP_ALIAS(atomic_and_ushort_nv,_atomic_and_16_nv)
-STRONG_ALIAS(__sync_and_and_fetch_2,_atomic_and_16_nv)
STRONG_ALIAS(_atomic_and_ushort_nv,_atomic_and_16_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_32.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP32(and, and)
+SYNC_FETCH_OP32(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_32,_atomic_and_32)
ATOMIC_OP_ALIAS(atomic_and_uint,_atomic_and_32)
-STRONG_ALIAS(__sync_fetch_and_and_4,_atomic_and_32)
STRONG_ALIAS(_atomic_and_uint,_atomic_and_32)
ATOMIC_OP32_NV(and, and)
+SYNC_OP32_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_32_nv,_atomic_and_32_nv)
ATOMIC_OP_ALIAS(atomic_and_uint_nv,_atomic_and_32_nv)
-STRONG_ALIAS(__sync_and_and_fetch_4,_atomic_and_32_nv)
STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_64.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP64(and, and)
+SYNC_FETCH_OP64(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_64,_atomic_and_64)
ATOMIC_OP_ALIAS(atomic_and_ulong,_atomic_and_64)
STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64)
-STRONG_ALIAS(__sync_fetch_and_and_8,_atomic_and_64)
ATOMIC_OP64_NV(and, and)
+SYNC_OP64_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_64_nv,_atomic_and_64_nv)
ATOMIC_OP_ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv)
STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv)
-STRONG_ALIAS(__sync_and_and_fetch_8,_atomic_and_64_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_and_8.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_and_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
+/* $NetBSD: atomic_and_8.S,v 1.2 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -33,14 +33,16 @@
ATOMIC_OP8(and, and)
+SYNC_FETCH_OP8(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_8,_atomic_and_8)
ATOMIC_OP_ALIAS(atomic_and_uchar,_atomic_and_8)
-STRONG_ALIAS(__sync_fetch_and_and_1,_atomic_and_8)
STRONG_ALIAS(_atomic_and_uchar,_atomic_and_8)
ATOMIC_OP8_NV(and, and)
+SYNC_OP8_FETCH(and, and)
+
ATOMIC_OP_ALIAS(atomic_and_8_nv,_atomic_and_8_nv)
ATOMIC_OP_ALIAS(atomic_and_uchar_nv,_atomic_and_8_nv)
-STRONG_ALIAS(__sync_and_and_fetch_1,_atomic_and_8_nv)
STRONG_ALIAS(_atomic_and_uchar_nv,_atomic_and_8_nv)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_16.S,v 1.3 2020/10/07 07:31:47 skrll Exp $ */
+/* $NetBSD: atomic_cas_16.S,v 1.4 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -46,3 +46,16 @@
STRONG_ALIAS(_atomic_cas_short,_atomic_cas_16)
STRONG_ALIAS(_atomic_cas_ushort,_atomic_cas_16)
STRONG_ALIAS(__sync_val_compare_and_swap_2,_atomic_cas_16)
+
+ENTRY_NP(__sync_val_compare_and_swap_2)
+ mov x4, x0 /* we need x0 for return value */
+ dmb ish
+1: ldxrh w0, [x4] /* load old value */
+ cmp w0, w1 /* compare? */
+ b.ne 2f
+ stxrh w3, w2, [x4] /* store new value */
+ cbnz w3, 3f /* succeed? nope, try again. */
+ dmb ish
+2: ret /* return. */
+3: b 1b
+END(__sync_val_compare_and_swap_2)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_32.S,v 1.3 2020/10/07 07:31:47 skrll Exp $ */
+/* $NetBSD: atomic_cas_32.S,v 1.4 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -46,7 +46,19 @@
ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_32_ni,_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_uint_ni,_atomic_cas_32)
-STRONG_ALIAS(__sync_val_compare_and_swap_4,_atomic_cas_32)
STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32)
STRONG_ALIAS(_atomic_cas_32_ni,_atomic_cas_32)
STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32)
+
+ENTRY_NP(__sync_val_compare_and_swap_4)
+ mov x4, x0 /* we need x0 for return value */
+ dmb ish
+1: ldxr w0, [x4] /* load old value */
+ cmp w0, w1 /* compare? */
+ b.ne 2f /* return if different */
+ stxr w3, w2, [x4] /* store new value */
+ cbnz w3, 3f /* succeed? nope, try again. */
+ dmb ish
+2: ret /* return. */
+3: b 1b
+END(__sync_val_compare_and_swap_4)
diff -r 964bfacc7d72 -r 6f602f7d3243 common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S
--- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S Thu Jul 29 06:35:20 2021 +0000
+++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S Thu Jul 29 10:29:05 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic_cas_64.S,v 1.5 2020/10/07 07:31:47 skrll Exp $ */
+/* $NetBSD: atomic_cas_64.S,v 1.6 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -48,9 +48,21 @@
ATOMIC_OP_ALIAS(atomic_cas_64_ni,_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_ulong_ni,_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_ptr_ni,_atomic_cas_64)
-STRONG_ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64)
STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64)
Home |
Main Index |
Thread Index |
Old Index