[src/trunk]: src/sys/arch gic_splx: performance optimizations

To: source-changes-hg%NetBSD.org@localhost
Subject: [src/trunk]: src/sys/arch gic_splx: performance optimizations
From: jmcneill <jmcneill%NetBSD.org@localhost>
Date: Sat, 18 Sep 2021 16:48:37 +0000

details:   https://anonhg.NetBSD.org/src/rev/5b664809d466
branches:  trunk
changeset: 986297:5b664809d466
user:      jmcneill <jmcneill%NetBSD.org@localhost>
date:      Sat Sep 18 12:25:06 2021 +0000

description:
gic_splx: performance optimizations

Avoid any kind of register access (DAIF, PMR, etc), barriers, and atomic
operations in the common case where no interrupt fires between spl being
raised and lowered.

This introduces a per-CPU return address (ci_splx_restart) used by the
vector handler to restart a sequence in splx that compares the new ipl
with the per-CPU hardware priority state stored in ci_hwpl.

diffstat:

 sys/arch/aarch64/aarch64/genassym.cf |   3 +-
 sys/arch/aarch64/aarch64/vectors.S   |  23 ++++++++++++++++++--
 sys/arch/aarch64/include/cpu.h       |   4 ++-
 sys/arch/arm/cortex/gic_splfuncs.c   |  40 +++++++++++++++++++++++++++++------
 sys/arch/evbarm/conf/std.generic64   |   3 +-
 5 files changed, 60 insertions(+), 13 deletions(-)

diffs (173 lines):

diff -r e4eea4039b02 -r 5b664809d466 sys/arch/aarch64/aarch64/genassym.cf
--- a/sys/arch/aarch64/aarch64/genassym.cf      Sat Sep 18 10:46:17 2021 +0000
+++ b/sys/arch/aarch64/aarch64/genassym.cf      Sat Sep 18 12:25:06 2021 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.33 2020/12/11 18:03:33 skrll Exp $
+# $NetBSD: genassym.cf,v 1.34 2021/09/18 12:25:06 jmcneill Exp $
 #-
 # Copyright (c) 2014 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -296,6 +296,7 @@
 define CI_SOFTINTS             offsetof(struct cpu_info, ci_softints)
 define CI_IDLELWP              offsetof(struct cpu_info, ci_data.cpu_idlelwp)
 define CI_CC_NINTR             offsetof(struct cpu_info, ci_data.cpu_nintr)
+define CI_SPLX_RESTART         offsetof(struct cpu_info, ci_splx_restart)
 
 define V_RESCHED_KPREEMPT      ilog2(RESCHED_KPREEMPT)
 
diff -r e4eea4039b02 -r 5b664809d466 sys/arch/aarch64/aarch64/vectors.S
--- a/sys/arch/aarch64/aarch64/vectors.S        Sat Sep 18 10:46:17 2021 +0000
+++ b/sys/arch/aarch64/aarch64/vectors.S        Sat Sep 18 12:25:06 2021 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: vectors.S,v 1.23 2021/08/30 23:20:00 jmcneill Exp $    */
+/*     $NetBSD: vectors.S,v 1.24 2021/09/18 12:25:06 jmcneill Exp $    */
 
 #include <aarch64/asm.h>
 #include <aarch64/locore.h>
@@ -9,8 +9,9 @@
 #include "opt_cpuoptions.h"
 #include "opt_ddb.h"
 #include "opt_dtrace.h"
+#include "opt_gic.h"
 
-RCSID("$NetBSD: vectors.S,v 1.23 2021/08/30 23:20:00 jmcneill Exp $")
+RCSID("$NetBSD: vectors.S,v 1.24 2021/09/18 12:25:06 jmcneill Exp $")
 
        ARMV8_DEFINE_OPTIONS
 
@@ -220,7 +221,23 @@
 
        unwind_x3_x30
 
-#if TF_PC + 8 == TF_SPSR
+#ifdef GIC_SPLFUNCS
+       mrs     x0, tpidr_el1           /* get curlwp */
+       ldr     x0, [x0, #L_CPU]        /* get curcpu */
+
+       /*
+        * If ci_intr_depth == 0 and ci_splx_restart != 0, return
+        * to splx restart. Otherwise return to exception pc.
+        */
+       ldr     w1, [x0, #CI_INTR_DEPTH]
+       cbnz    w1, 1f
+       ldr     x0, [x0, #CI_SPLX_RESTART]
+       cbnz    x0, 2f
+1:
+       ldr     x0, [sp, #TF_PC]
+2:
+       ldr     x1, [sp, #TF_SPSR]
+#elif TF_PC + 8 == TF_SPSR
        ldp     x0, x1, [sp, #TF_PC]
 #else
        ldr     x0, [sp, #TF_PC]
diff -r e4eea4039b02 -r 5b664809d466 sys/arch/aarch64/include/cpu.h
--- a/sys/arch/aarch64/include/cpu.h    Sat Sep 18 10:46:17 2021 +0000
+++ b/sys/arch/aarch64/include/cpu.h    Sat Sep 18 12:25:06 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.38 2021/08/14 17:51:18 ryo Exp $ */
+/* $NetBSD: cpu.h,v 1.39 2021/09/18 12:25:06 jmcneill Exp $ */
 
 /*-
  * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
@@ -108,6 +108,8 @@
        volatile uint32_t ci_blocked_pics;
        volatile uint32_t ci_pending_pics;
        volatile uint32_t ci_pending_ipls;
+       void *ci_splx_restart;
+       int ci_splx_savedipl;
 
        int ci_kfpu_spl;
 
diff -r e4eea4039b02 -r 5b664809d466 sys/arch/arm/cortex/gic_splfuncs.c
--- a/sys/arch/arm/cortex/gic_splfuncs.c        Sat Sep 18 10:46:17 2021 +0000
+++ b/sys/arch/arm/cortex/gic_splfuncs.c        Sat Sep 18 12:25:06 2021 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: gic_splfuncs.c,v 1.1 2021/08/10 15:33:09 jmcneill Exp $ */
+/* $NetBSD: gic_splfuncs.c,v 1.2 2021/09/18 12:25:07 jmcneill Exp $ */
 
 /*-
  * Copyright (c) 2021 Jared McNeill <jmcneill%invisible.ca@localhost>
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.1 2021/08/10 15:33:09 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gic_splfuncs.c,v 1.2 2021/09/18 12:25:07 jmcneill Exp $");
 
 #include <sys/param.h>
 #include <sys/atomic.h>
@@ -75,26 +75,52 @@
 static void
 gic_splx(int newipl)
 {
-       struct cpu_info * const ci = curcpu();
+       struct cpu_info *ci = curcpu();
+       register_t psw;
 
        if (newipl >= ci->ci_cpl) {
                return;
        }
 
-       if (ci->ci_hwpl <= newipl) {
-               ci->ci_cpl = newipl;
+       /*
+        * Try to avoid touching any hardware registers (DAIF, PMR) as an
+        * optimization for the common case of splraise followed by splx
+        * with no interrupts in between.
+        *
+        * If an interrupt fires in this critical section, the vector
+        * handler is responsible for returning to the address pointed
+        * to by ci_splx_restart to restart the sequence.
+        */
+       if (__predict_true(ci->ci_intr_depth == 0)) {
+               ci->ci_splx_savedipl = newipl;
+               ci->ci_splx_restart = &&restart;
+               __insn_barrier();
+checkhwpl:
                if (ci->ci_hwpl <= newipl) {
-                       return;
+                       ci->ci_cpl = newipl;
+                       __insn_barrier();
+                       ci->ci_splx_restart = NULL;
+                       goto dosoft;
+               } else {
+                       ci->ci_splx_restart = NULL;
+                       goto dohard;
                }
+restart:
+               ci = curcpu();
+               newipl = ci->ci_splx_savedipl;
+               goto checkhwpl;
        }
 
-       register_t psw = DISABLE_INTERRUPT_SAVE();
+dohard:
+       psw = DISABLE_INTERRUPT_SAVE();
        ci->ci_intr_depth++;
        pic_do_pending_ints(psw, newipl, NULL);
        ci->ci_intr_depth--;
        if ((psw & I32_bit) == 0) {
                ENABLE_INTERRUPT();
        }
+
+dosoft:
        cpu_dosoftints();
 }
 
diff -r e4eea4039b02 -r 5b664809d466 sys/arch/evbarm/conf/std.generic64
--- a/sys/arch/evbarm/conf/std.generic64        Sat Sep 18 10:46:17 2021 +0000
+++ b/sys/arch/evbarm/conf/std.generic64        Sat Sep 18 12:25:06 2021 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: std.generic64,v 1.14 2021/08/08 12:31:42 jmcneill Exp $
+#      $NetBSD: std.generic64,v 1.15 2021/09/18 12:25:07 jmcneill Exp $
 #
 #      generic NetBSD/evbarm64 with FDT support
 
@@ -16,6 +16,7 @@
 options        EVBARM_BOARDTYPE="FDT"
 options        FDT                             # Flattened Device Tree support
 options        FPU_VFP
+options        GIC_SPLFUNCS
 options        MODULAR
 options        MODULAR_DEFAULT_AUTOLOAD
 options        PCI_NETBSD_CONFIGURE

Prev by Date: [src/trunk]: src/usr.bin/xlint/lint1 use isfinite(3) instead of finite(3) for...
Next by Date: [src/trunk]: src/sys/arch/luna68k/dev Implement transmitting keyboard LED and...
Previous by Thread: [src/trunk]: src/usr.bin/xlint/lint1 use isfinite(3) instead of finite(3) for...
Next by Thread: [src/trunk]: src/sys/arch/luna68k/dev Implement transmitting keyboard LED and...
Indexes:

Home | Main Index | Thread Index | Old Index