[src/trunk]: src/sys/arch/aarch64 Switch the Icache sync operation to the nec...

[ryo <ryo%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/5170f7648b27
branches:  trunk
changeset: 973448:5170f7648b27
user:      ryo <ryo%NetBSD.org@localhost>
date:      Wed Jul 01 07:59:16 2020 +0000

description:
Switch the Icache sync operation to the necessary and sufficient one according to the CTR_EL0.DIC and CTR_EL0.IDC flags.

If CTR_EL0.DIC=1, Icache invalidation is not required.
If CTR_EL0.IDC=1, Dcache clean before Icache invalidation is not required.
CLIDR_EL1.LoC is 0, or CLIDR_EL1.LoUIS and CLIDR_EL1.LoUU are 0, Dcache clean is not required as well.

SEE ALSO ARMARM, "CTR_EL0 Cache Type Register", and "CLIDR_EL1 Cache Level ID Register"

diffstat:

 sys/arch/aarch64/aarch64/cpu.c               |  17 ++++++++++++-----
 sys/arch/aarch64/aarch64/cpufunc.c           |  24 +++++++++++++++++++++---
 sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S |  19 ++++++++++++++++++-
 sys/arch/aarch64/include/cpu.h               |   3 ++-
 sys/arch/aarch64/include/cpufunc.h           |   7 +++++--
 5 files changed, 58 insertions(+), 12 deletions(-)

diffs (180 lines):

diff -r 655bf85c995e -r 5170f7648b27 sys/arch/aarch64/aarch64/cpu.c
--- a/sys/arch/aarch64/aarch64/cpu.c    Wed Jul 01 07:54:24 2020 +0000
+++ b/sys/arch/aarch64/aarch64/cpu.c    Wed Jul 01 07:59:16 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.50 2020/06/29 23:56:30 riastradh Exp $ */
+/* $NetBSD: cpu.c,v 1.51 2020/07/01 07:59:16 ryo Exp $ */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.50 2020/06/29 23:56:30 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.51 2020/07/01 07:59:16 ryo Exp $");
 
 #include "locators.h"
 #include "opt_arm_debug.h"
@@ -239,7 +239,7 @@
 static void
 cpu_identify1(device_t self, struct cpu_info *ci)
 {
-       uint64_t ctr, sctlr;    /* for cache */
+       uint64_t ctr, clidr, sctlr;     /* for cache */
 
        /* SCTLR - System Control Register */
        sctlr = reg_sctlr_el1_read();
@@ -277,14 +277,21 @@
         * CTR - Cache Type Register
         */
        ctr = reg_ctr_el0_read();
+       clidr = reg_clidr_el1_read();
        aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B,"
            " Exclusives Reservation Granule %" PRIu64 "B\n",
            __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4,
            __SHIFTOUT(ctr, CTR_EL0_ERG_LINE) * 4);
 
-       aprint_verbose_dev(self, "Dcache line %ld, Icache line %ld\n",
+       aprint_verbose_dev(self, "Dcache line %ld, Icache line %ld"
+           ", DIC=%lu, IDC=%lu, LoUU=%lu, LoC=%lu, LoUIS=%lu\n",
            sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE),
-           sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_IMIN_LINE));
+           sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_IMIN_LINE),
+           __SHIFTOUT(ctr, CTR_EL0_DIC),
+           __SHIFTOUT(ctr, CTR_EL0_IDC),
+           __SHIFTOUT(clidr, CLIDR_LOUU),
+           __SHIFTOUT(clidr, CLIDR_LOC),
+           __SHIFTOUT(clidr, CLIDR_LOUIS));
 }
 
 
diff -r 655bf85c995e -r 5170f7648b27 sys/arch/aarch64/aarch64/cpufunc.c
--- a/sys/arch/aarch64/aarch64/cpufunc.c        Wed Jul 01 07:54:24 2020 +0000
+++ b/sys/arch/aarch64/aarch64/cpufunc.c        Wed Jul 01 07:59:16 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.c,v 1.20 2020/05/25 05:13:16 ryo Exp $ */
+/*     $NetBSD: cpufunc.c,v 1.21 2020/07/01 07:59:16 ryo Exp $ */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -30,7 +30,7 @@
 #include "opt_multiprocessor.h"
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.20 2020/05/25 05:13:16 ryo Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.21 2020/07/01 07:59:16 ryo Exp $");
 
 #include <sys/param.h>
 #include <sys/types.h>
@@ -426,13 +426,31 @@
 set_cpufuncs(void)
 {
        struct cpu_info * const ci = curcpu();
+       const uint64_t ctr = reg_ctr_el0_read();
+       const uint64_t clidr = reg_clidr_el1_read();
        const uint32_t midr __unused = reg_midr_el1_read();
 
        /* install default functions */
        ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0;
+       ci->ci_cpufuncs.cf_icache_sync_range = aarch64_icache_sync_range;
 
+       /*
+        * install core/cluster specific functions
+        */
 
-       /* install core/cluster specific functions */
+       /* Icache sync op */
+       if (__SHIFTOUT(ctr, CTR_EL0_DIC) == 1) {
+               /* Icache invalidation to the PoU is not required */
+               ci->ci_cpufuncs.cf_icache_sync_range =
+                   aarch64_icache_barrier_range;
+       } else if (__SHIFTOUT(ctr, CTR_EL0_IDC) == 1 ||
+           __SHIFTOUT(clidr, CLIDR_LOC) == 0 ||
+           (__SHIFTOUT(clidr, CLIDR_LOUIS) == 0 && __SHIFTOUT(clidr, CLIDR_LOUU) == 0)) {
+               /* Dcache clean to the PoU is not required for Icache */
+               ci->ci_cpufuncs.cf_icache_sync_range =
+                   aarch64_icache_inv_range;
+       }
+
 #ifdef CPU_THUNDERX
        /* Cavium erratum 27456 */
        if ((midr == CPU_ID_THUNDERXP1d0) ||
diff -r 655bf85c995e -r 5170f7648b27 sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S
--- a/sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S      Wed Jul 01 07:54:24 2020 +0000
+++ b/sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S      Wed Jul 01 07:59:16 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc_asm_armv8.S,v 1.5 2020/06/01 08:59:00 ryo Exp $        */
+/*     $NetBSD: cpufunc_asm_armv8.S,v 1.6 2020/07/01 07:59:16 ryo Exp $        */
 
 /*-
  * Copyright (c) 2014 Robin Randhawa
@@ -135,6 +135,23 @@
 END(aarch64_icache_sync_range)
 
 /*
+ * void aarch64_icache_inv_range(vaddr_t, vsize_t)
+ */
+ENTRY(aarch64_icache_inv_range)
+       cache_handle_range      icop = ivau
+       ret
+END(aarch64_icache_inv_range)
+
+/*
+ * void aarch64_icache_barrier_range(vaddr_t, vsize_t)
+ */
+ENTRY(aarch64_icache_barrier_range)
+       dsb     ishst
+       isb
+       ret
+END(aarch64_icache_barrier_range)
+
+/*
  * void aarch64_icache_inv_all(void)
  */
 ENTRY(aarch64_icache_inv_all)
diff -r 655bf85c995e -r 5170f7648b27 sys/arch/aarch64/include/cpu.h
--- a/sys/arch/aarch64/include/cpu.h    Wed Jul 01 07:54:24 2020 +0000
+++ b/sys/arch/aarch64/include/cpu.h    Wed Jul 01 07:59:16 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.23 2020/06/29 23:22:27 riastradh Exp $ */
+/* $NetBSD: cpu.h,v 1.24 2020/07/01 07:59:16 ryo Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -68,6 +68,7 @@
 
 struct aarch64_cpufuncs {
        void (*cf_set_ttbr0)(uint64_t);
+       void (*cf_icache_sync_range)(vaddr_t, vsize_t);
 };
 
 struct cpu_info {
diff -r 655bf85c995e -r 5170f7648b27 sys/arch/aarch64/include/cpufunc.h
--- a/sys/arch/aarch64/include/cpufunc.h        Wed Jul 01 07:54:24 2020 +0000
+++ b/sys/arch/aarch64/include/cpufunc.h        Wed Jul 01 07:59:16 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.h,v 1.15 2020/05/25 05:13:16 ryo Exp $ */
+/*     $NetBSD: cpufunc.h,v 1.16 2020/07/01 07:59:16 ryo Exp $ */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -80,6 +80,8 @@
 void aarch64_nullop(void);
 uint32_t aarch64_cpuid(void);
 void aarch64_icache_sync_range(vaddr_t, vsize_t);
+void aarch64_icache_inv_range(vaddr_t, vsize_t);
+void aarch64_icache_barrier_range(vaddr_t, vsize_t);
 void aarch64_idcache_wbinv_range(vaddr_t, vsize_t);
 void aarch64_dcache_wbinv_range(vaddr_t, vsize_t);
 void aarch64_dcache_inv_range(vaddr_t, vsize_t);
@@ -117,7 +119,8 @@
 #define cpu_dcache_inv_range(v,s)      aarch64_dcache_inv_range((v),(s))
 #define cpu_dcache_wb_range(v,s)       aarch64_dcache_wb_range((v),(s))
 #define cpu_idcache_wbinv_range(v,s)   aarch64_idcache_wbinv_range((v),(s))
-#define cpu_icache_sync_range(v,s)     aarch64_icache_sync_range((v),(s))
+#define cpu_icache_sync_range(v,s)     \
+       curcpu()->ci_cpufuncs.cf_icache_sync_range((v),(s))
 
 #define cpu_sdcache_wbinv_range(v,p,s) ((void)0)
 #define cpu_sdcache_inv_range(v,p,s)   ((void)0)