[src/trunk]: src/sys/arch For this case during build.sh:

[ad <ad%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/aaddbaf1a87d
branches:  trunk
changeset: 465455:aaddbaf1a87d
user:      ad <ad%NetBSD.org@localhost>
date:      Sat Nov 23 16:36:38 2019 +0000

description:
For this case during build.sh:

        rw_enter(lock, RW_READ);

Having instrumented it, it turns out that >99.5% of the time the lock is
completely unknowned.  Make this assumption in the assembly stub for
rw_enter(), and avoid the initial read of the lock word.  Where there are
existing read holds, we'll do an additional CMPXCHG but should already have
the cache line in the EXCLUSIVE state.

diffstat:

 sys/arch/amd64/amd64/lock_stubs.S |  20 +++++++++++---------
 sys/arch/i386/i386/lock_stubs.S   |  21 ++++++++++++---------
 2 files changed, 23 insertions(+), 18 deletions(-)

diffs (138 lines):

diff -r 4e61c92cf274 -r aaddbaf1a87d sys/arch/amd64/amd64/lock_stubs.S
--- a/sys/arch/amd64/amd64/lock_stubs.S Sat Nov 23 15:17:46 2019 +0000
+++ b/sys/arch/amd64/amd64/lock_stubs.S Sat Nov 23 16:36:38 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: lock_stubs.S,v 1.33 2019/11/14 16:23:52 maxv Exp $     */
+/*     $NetBSD: lock_stubs.S,v 1.34 2019/11/23 16:36:38 ad Exp $       */
 
 /*
  * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -191,13 +191,15 @@
  * Acquire one hold on a RW lock.
  */
 ENTRY(rw_enter)
-       cmpl    $RW_READER, %esi
+       xorl    %eax, %eax
+       testl   %esi, %esi      /* RW_READER = 0 */
        jne     2f
 
        /*
-        * Reader: this is the most common case.
+        * Reader, and no existing readers on the lock: this is a most
+        * common case.  Instead of reading from the lock word, use cmpxchg
+        * and get the cache line into the EXCLUSIVE state to begin with.
         */
-       movq    (%rdi), %rax
 0:
        testb   $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
        jnz     3f
@@ -213,7 +215,6 @@
         * Writer: if the compare-and-set fails, don't bother retrying.
         */
 2:     movq    CPUVAR(CURLWP), %rcx
-       xorq    %rax, %rax
        orq     $RW_WRITE_LOCKED, %rcx
        LOCK
        cmpxchgq %rcx, (%rdi)
@@ -268,13 +269,15 @@
  * Try to acquire one hold on a RW lock.
  */
 ENTRY(rw_tryenter)
-       cmpl    $RW_READER, %esi
+       xorl    %eax, %eax
+       testl   %esi, %esi      /* RW_READER = 0 */
        jne     2f
 
        /*
-        * Reader: this is the most common case.
+        * Reader, and no existing readers on the lock: this is a most
+        * common case.  Instead of reading from the lock word, use cmpxchg
+        * and get the cache line into the EXCLUSIVE state to begin with.
         */
-       movq    (%rdi), %rax
 0:
        testb   $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
        jnz     4f
@@ -291,7 +294,6 @@
         * Writer: if the compare-and-set fails, don't bother retrying.
         */
 2:     movq    CPUVAR(CURLWP), %rcx
-       xorq    %rax, %rax
        orq     $RW_WRITE_LOCKED, %rcx
        LOCK
        cmpxchgq %rcx, (%rdi)
diff -r 4e61c92cf274 -r aaddbaf1a87d sys/arch/i386/i386/lock_stubs.S
--- a/sys/arch/i386/i386/lock_stubs.S   Sat Nov 23 15:17:46 2019 +0000
+++ b/sys/arch/i386/i386/lock_stubs.S   Sat Nov 23 16:36:38 2019 +0000
@@ -1,7 +1,7 @@
-/*     $NetBSD: lock_stubs.S,v 1.30 2019/02/11 14:59:32 cherry Exp $   */
+/*     $NetBSD: lock_stubs.S,v 1.31 2019/11/23 16:36:38 ad Exp $       */
 
 /*-
- * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
+ * Copyright (c) 2006, 2007, 2008, 2009, 2019 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -35,7 +35,7 @@
  */
 
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.30 2019/02/11 14:59:32 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.31 2019/11/23 16:36:38 ad Exp $");
 
 #include "opt_lockdebug.h"
 
@@ -104,13 +104,15 @@
  */
 ENTRY(rw_enter)
        movl    4(%esp), %edx
+       xorl    %eax, %eax
        cmpl    $RW_READER, 8(%esp)
        jne     2f
 
        /*
-        * Reader
+        * Reader, and no existing readers on the lock: this is a most
+        * common case.  Instead of reading from the lock word, use cmpxchg
+        * and get the cache line into the EXCLUSIVE state to begin with.
         */
-       movl    (%edx), %eax
 0:
        testb   $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
        jnz     3f
@@ -125,7 +127,7 @@
        /*
         * Writer
         */
-2:     xorl    %eax, %eax
+2:
        movl    %fs:CPU_INFO_CURLWP(%eax), %ecx
        orl     $RW_WRITE_LOCKED, %ecx
        LOCK(3)
@@ -186,13 +188,15 @@
  */
 ENTRY(rw_tryenter)
        movl    4(%esp), %edx
+       xorl    %eax, %eax
        cmpl    $RW_READER, 8(%esp)
        jne     2f
 
        /*
-        * Reader
+        * Reader, and no existing readers on the lock: this is a most
+        * common case.  Instead of reading from the lock word, use cmpxchg
+        * and get the cache line into the EXCLUSIVE state to begin with.
         */
-       movl    (%edx), %eax
 0:
        testb   $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
        jnz     4f
@@ -209,7 +213,6 @@
         * Writer
         */
 2:
-       xorl    %eax, %eax
        movl    %fs:CPU_INFO_CURLWP(%eax), %ecx
        orl     $RW_WRITE_LOCKED, %ecx
        LOCK(13)