[src/trunk]: src/sys/arch/aarch64 add MULTIPROCESSOR support

[ryo <ryo%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/c6cea6b3b38f
branches:  trunk
changeset: 363045:c6cea6b3b38f
user:      ryo <ryo%NetBSD.org@localhost>
date:      Mon Jul 09 06:19:53 2018 +0000

description:
add MULTIPROCESSOR support

diffstat:

 sys/arch/aarch64/aarch64/cpu.c         |  180 +++++++++++---
 sys/arch/aarch64/aarch64/cpu_machdep.c |   38 +---
 sys/arch/aarch64/aarch64/db_machdep.c  |  123 +++++++++-
 sys/arch/aarch64/aarch64/genassym.cf   |    7 +-
 sys/arch/aarch64/aarch64/locore.S      |  376 ++++++++++++++++++++++++++------
 sys/arch/aarch64/include/cpu.h         |   14 +-
 sys/arch/aarch64/include/locore.h      |    7 +-
 sys/arch/aarch64/include/machdep.h     |    6 +-
 8 files changed, 573 insertions(+), 178 deletions(-)

diffs (truncated from 1131 to 300 lines):

diff -r 0eab3c7fa136 -r c6cea6b3b38f sys/arch/aarch64/aarch64/cpu.c
--- a/sys/arch/aarch64/aarch64/cpu.c    Mon Jul 09 06:14:38 2018 +0000
+++ b/sys/arch/aarch64/aarch64/cpu.c    Mon Jul 09 06:19:53 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.1 2018/04/01 04:35:03 ryo Exp $ */
+/* $NetBSD: cpu.c,v 1.2 2018/07/09 06:19:53 ryo Exp $ */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -27,13 +27,16 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.1 2018/04/01 04:35:03 ryo Exp $");
+__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.2 2018/07/09 06:19:53 ryo Exp $");
 
 #include "locators.h"
+#include "opt_arm_debug.h"
+#include "opt_fdt.h"
 #include "opt_multiprocessor.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/atomic.h>
 #include <sys/device.h>
 #include <sys/cpu.h>
 #include <sys/kmem.h>
@@ -43,71 +46,106 @@
 #include <aarch64/cpufunc.h>
 #include <aarch64/machdep.h>
 
+#ifdef FDT
+#include <arm/fdt/arm_fdtvar.h>
+#endif
+
 void cpu_attach(device_t, cpuid_t);
-static void cpu_identify(device_t self, struct cpu_info *);
+static void identify_aarch64_model(uint32_t, char *, size_t);
+static void cpu_identify(device_t self, struct cpu_info *, uint32_t, uint64_t);
+static void cpu_identify1(device_t self, struct cpu_info *);
 static void cpu_identify2(device_t self, struct cpu_info *);
 
+#ifdef MULTIPROCESSOR
+volatile u_int arm_cpu_hatched __cacheline_aligned = 0;
+volatile uint32_t arm_cpu_mbox __cacheline_aligned = 0;
+u_int arm_cpu_max = 1;
+
+/* stored by secondary processors (available when arm_cpu_hatched) */
+uint32_t cpus_midr[MAXCPUS];
+uint64_t cpus_mpidr[MAXCPUS];
+
+static kmutex_t cpu_hatch_lock;
+#endif /* MULTIPROCESSOR */
+
+/* Our exported CPU info; we can have only one. */
+struct cpu_info cpu_info_store __cacheline_aligned = {
+       .ci_cpl = IPL_HIGH,
+       .ci_curlwp = &lwp0
+};
+
+#ifdef MULTIPROCESSOR
+#define NCPUINFO       MAXCPUS
+#else
+#define NCPUINFO       1
+#endif /* MULTIPROCESSOR */
+
+struct cpu_info *cpu_info[NCPUINFO] = {
+       [0] = &cpu_info_store
+};
+
 void
 cpu_attach(device_t dv, cpuid_t id)
 {
        struct cpu_info *ci;
+       uint64_t mpidr;
+       uint32_t midr;
 
        if (id == 0) {
                ci = curcpu();
-
+               midr = reg_midr_el1_read();
+               mpidr = reg_mpidr_el1_read();
        } else {
 #ifdef MULTIPROCESSOR
-               //XXXAARCH64: notyet?
-
-               uint64_t mpidr = reg_mpidr_el1_read();
-
                KASSERT(cpu_info[id] == NULL);
                ci = kmem_zalloc(sizeof(*ci), KM_SLEEP);
                ci->ci_cpl = IPL_HIGH;
                ci->ci_cpuid = id;
-               if (mpidr & MPIDR_MT) {
-                       ci->ci_data.cpu_smt_id = mpidr & MPIDR_AFF0;
-                       ci->ci_data.cpu_core_id = mpidr & MPIDR_AFF1;
-                       ci->ci_data.cpu_package_id = mpidr & MPIDR_AFF2;
-               } else {
-                       ci->ci_data.cpu_core_id = mpidr & MPIDR_AFF0;
-                       ci->ci_data.cpu_package_id = mpidr & MPIDR_AFF1;
-               }
-               ci->ci_data.cpu_cc_freq = cpu_info_store.ci_data.cpu_cc_freq;
+
+               ci->ci_data.cpu_cc_freq = cpu_info[0]->ci_data.cpu_cc_freq;
                cpu_info[ci->ci_cpuid] = ci;
                if ((arm_cpu_hatched & (1 << id)) == 0) {
                        ci->ci_dev = dv;
                        dv->dv_private = ci;
 
                        aprint_naive(": disabled\n");
-                       aprint_normal(": disabled (uniprocessor kernel)\n");
+                       aprint_normal(": disabled (unresponsive)\n");
                        return;
                }
-#else
+
+               /* cpus_{midr,mpidr}[id] is stored by secondary processor */
+               midr = cpus_midr[id];
+               mpidr = cpus_mpidr[id];
+#else /* MULTIPROCESSOR */
                aprint_naive(": disabled\n");
                aprint_normal(": disabled (uniprocessor kernel)\n");
                return;
-#endif
+#endif /* MULTIPROCESSOR */
+       }
+
+       if (mpidr & MPIDR_MT) {
+               ci->ci_data.cpu_smt_id = mpidr & MPIDR_AFF0;
+               ci->ci_data.cpu_core_id = mpidr & MPIDR_AFF1;
+               ci->ci_data.cpu_package_id = mpidr & MPIDR_AFF2;
+       } else {
+               ci->ci_data.cpu_core_id = mpidr & MPIDR_AFF0;
+               ci->ci_data.cpu_package_id = mpidr & MPIDR_AFF1;
        }
 
        ci->ci_dev = dv;
        dv->dv_private = ci;
 
+       cpu_identify(ci->ci_dev, ci, midr, mpidr);
 #ifdef MULTIPROCESSOR
-       if (caa->caa_cpucore != 0) {
-               aprint_naive("\n");
-               aprint_normal(": %s\n", cpu_getmodel());
+       if (id != 0) {
                mi_cpu_attach(ci);
-
-               // XXXAARCH64
-               //pmap_tlb_info_attach();
-               panic("notyet");
+               return;
        }
-#endif
+#endif /* MULTIPROCESSOR */
 
        fpu_attach(ci);
 
-       cpu_identify(dv, ci);
+       cpu_identify1(dv, ci);
        cpu_identify2(dv, ci);
 }
 
@@ -224,30 +262,25 @@
 }
 
 static void
-cpu_identify(device_t self, struct cpu_info *ci)
+cpu_identify(device_t self, struct cpu_info *ci, uint32_t midr, uint64_t mpidr)
 {
-       uint64_t mpidr;
-       int level;
-       uint32_t cpuid;
-       uint32_t ctr, sctlr;    /* for cache */
        char model[128];
 
-       cpuid = reg_midr_el1_read();
-       identify_aarch64_model(cpuid, model, sizeof(model));
+       identify_aarch64_model(midr, model, sizeof(model));
        if (ci->ci_cpuid == 0)
                cpu_setmodel("%s", model);
 
        aprint_naive("\n");
        aprint_normal(": %s\n", model);
-
+       aprint_normal_dev(ci->ci_dev, "package %lu, core %lu, smt %lu\n",
+           ci->ci_package_id, ci->ci_core_id, ci->ci_smt_id);
+}
 
-       mpidr = reg_mpidr_el1_read();
-       aprint_normal_dev(self, "CPU Affinity %llu-%llu-%llu-%llu\n",
-           __SHIFTOUT(mpidr, MPIDR_AFF3),
-           __SHIFTOUT(mpidr, MPIDR_AFF2),
-           __SHIFTOUT(mpidr, MPIDR_AFF1),
-           __SHIFTOUT(mpidr, MPIDR_AFF0));
-
+static void
+cpu_identify1(device_t self, struct cpu_info *ci)
+{
+       int level;
+       uint32_t ctr, sctlr;    /* for cache */
 
        /* SCTLR - System Control Register */
        sctlr = reg_sctlr_el1_read();
@@ -448,3 +481,62 @@
 
        aprint_normal("\n");
 }
+
+#ifdef MULTIPROCESSOR
+void
+cpu_boot_secondary_processors(void)
+{
+       mutex_init(&cpu_hatch_lock, MUTEX_DEFAULT, IPL_NONE);
+
+#ifdef VERBOSE_INIT_ARM
+       printf("%s: writing mbox with %#x\n", __func__, arm_cpu_hatched);
+#endif
+
+       /* send mbox to have secondary processors do cpu_hatch() */
+       atomic_or_32(&arm_cpu_mbox, arm_cpu_hatched);
+       __asm __volatile ("sev; sev; sev");
+
+       /* wait all cpus have done cpu_hatch() */
+       while (arm_cpu_mbox) {
+               __asm __volatile ("wfe");
+       }
+
+#ifdef VERBOSE_INIT_ARM
+       printf("%s: secondary processors hatched\n", __func__);
+#endif
+
+       /* add available processors to kcpuset */
+       uint32_t mbox = arm_cpu_hatched;
+       kcpuset_export_u32(kcpuset_attached, &mbox, sizeof(mbox));
+}
+
+void
+cpu_hatch(struct cpu_info *ci)
+{
+       KASSERT(curcpu() == ci);
+
+       delay(1000 * ci->ci_index);     /* XXX: to attach cpu* in order */
+
+       mutex_enter(&cpu_hatch_lock);
+
+       fpu_attach(ci);
+
+       cpu_identify1(ci->ci_dev, ci);
+       cpu_identify2(ci->ci_dev, ci);
+
+       mutex_exit(&cpu_hatch_lock);
+
+       intr_cpu_init(ci);
+
+#ifdef FDT
+       arm_fdt_cpu_hatch(ci);
+#endif
+#ifdef MD_CPU_HATCH
+       MD_CPU_HATCH(ci);       /* for non-fdt arch? */
+#endif
+
+       /* clear my bit of arm_cpu_mbox to tell cpu_boot_secondary_processors() */
+       atomic_and_32(&arm_cpu_mbox, ~(1 << ci->ci_cpuid));
+       __asm __volatile ("sev; sev; sev");
+}
+#endif /* MULTIPROCESSOR */
diff -r 0eab3c7fa136 -r c6cea6b3b38f sys/arch/aarch64/aarch64/cpu_machdep.c
--- a/sys/arch/aarch64/aarch64/cpu_machdep.c    Mon Jul 09 06:14:38 2018 +0000
+++ b/sys/arch/aarch64/aarch64/cpu_machdep.c    Mon Jul 09 06:19:53 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_machdep.c,v 1.3 2018/04/01 04:35:03 ryo Exp $ */
+/* $NetBSD: cpu_machdep.c,v 1.4 2018/07/09 06:19:53 ryo Exp $ */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(1, "$NetBSD: cpu_machdep.c,v 1.3 2018/04/01 04:35:03 ryo Exp $");
+__KERNEL_RCSID(1, "$NetBSD: cpu_machdep.c,v 1.4 2018/07/09 06:19:53 ryo Exp $");
 
 #include "opt_multiprocessor.h"
 
@@ -52,30 +52,6 @@
 #include <aarch64/pcb.h>
 #include <aarch64/userret.h>
 
-#ifdef MULTIPROCESSOR
-/* for arm compatibility (referred from pic.c) */
-volatile u_int arm_cpu_hatched;
-u_int arm_cpu_max = 1;
-#endif
-
-/* Our exported CPU info; we can have only one. */
-struct cpu_info cpu_info_store __cacheline_aligned = {
-       .ci_cpl = IPL_HIGH,
-       .ci_curlwp = &lwp0
-};
-
-#ifdef MULTIPROCESSOR