Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/uvm - Many small tweaks to the SMT awareness in the sche...
details: https://anonhg.NetBSD.org/src/rev/ab36fa3a3fc2
branches: trunk
changeset: 1006311:ab36fa3a3fc2
user: ad <ad%NetBSD.org@localhost>
date: Thu Jan 09 16:35:03 2020 +0000
description:
- Many small tweaks to the SMT awareness in the scheduler. It does a much
better job now at keeping all physical CPUs busy, while using the extra
threads to help out. In particular, during preempt() if we're using SMT,
try to find a better CPU to run on and teleport curlwp there.
- Change the CPU topology stuff so it can work on asymmetric systems. This
mainly entails rearranging one of the CPU lists so it makes sense in all
configurations.
- Add a parameter to cpu_topology_set() to note that a CPU is "slow", for
where there are fast CPUs and slow CPUs, like with the Rockwell RK3399.
Extend the SMT awareness to try and handle that situation too (keep fast
CPUs busy, use slow CPUs as helpers).
diffstat:
sys/arch/aarch64/aarch64/cpufunc.c | 10 +-
sys/arch/arm/arm32/cpu.c | 10 +-
sys/arch/macppc/macppc/cpu.c | 6 +-
sys/arch/mips/mips/cpu_subr.c | 6 +-
sys/arch/x86/x86/cpu_topology.c | 10 +-
sys/kern/kern_runq.c | 315 +++++++++++++++++++++++++-----------
sys/kern/kern_synch.c | 6 +-
sys/kern/sched_4bsd.c | 13 +-
sys/kern/subr_cpu.c | 165 +++++++++++-------
sys/sys/cpu.h | 4 +-
sys/sys/cpu_data.h | 32 ++-
sys/sys/sched.h | 11 +-
sys/uvm/uvm_page.c | 18 +-
13 files changed, 396 insertions(+), 210 deletions(-)
diffs (truncated from 1083 to 300 lines):
diff -r 25029195c9e7 -r ab36fa3a3fc2 sys/arch/aarch64/aarch64/cpufunc.c
--- a/sys/arch/aarch64/aarch64/cpufunc.c Thu Jan 09 16:27:57 2020 +0000
+++ b/sys/arch/aarch64/aarch64/cpufunc.c Thu Jan 09 16:35:03 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.c,v 1.12 2019/12/20 21:05:33 ad Exp $ */
+/* $NetBSD: cpufunc.c,v 1.13 2020/01/09 16:35:03 ad Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -29,7 +29,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.12 2019/12/20 21:05:33 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.13 2020/01/09 16:35:03 ad Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -97,13 +97,15 @@
__SHIFTOUT(mpidr, MPIDR_AFF2),
__SHIFTOUT(mpidr, MPIDR_AFF1),
__SHIFTOUT(mpidr, MPIDR_AFF0),
- 0);
+ 0,
+ false);
} else {
cpu_topology_set(ci,
__SHIFTOUT(mpidr, MPIDR_AFF1),
__SHIFTOUT(mpidr, MPIDR_AFF0),
0,
- 0);
+ 0,
+ false);
}
}
diff -r 25029195c9e7 -r ab36fa3a3fc2 sys/arch/arm/arm32/cpu.c
--- a/sys/arch/arm/arm32/cpu.c Thu Jan 09 16:27:57 2020 +0000
+++ b/sys/arch/arm/arm32/cpu.c Thu Jan 09 16:35:03 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.138 2020/01/09 16:23:42 martin Exp $ */
+/* $NetBSD: cpu.c,v 1.139 2020/01/09 16:35:03 ad Exp $ */
/*
* Copyright (c) 1995 Mark Brinicombe.
@@ -46,7 +46,7 @@
#include "opt_multiprocessor.h"
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.138 2020/01/09 16:23:42 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.139 2020/01/09 16:35:03 ad Exp $");
#include <sys/param.h>
#include <sys/conf.h>
@@ -143,13 +143,15 @@
__SHIFTOUT(id, MPIDR_AFF2),
__SHIFTOUT(id, MPIDR_AFF1),
__SHIFTOUT(id, MPIDR_AFF0),
- 0);
+ 0,
+ false);
} else {
cpu_topology_set(ci,
__SHIFTOUT(id, MPIDR_AFF1),
__SHIFTOUT(id, MPIDR_AFF0),
0,
- 0);
+ 0,
+ false);
}
evcnt_attach_dynamic(&ci->ci_arm700bugcount, EVCNT_TYPE_MISC,
diff -r 25029195c9e7 -r ab36fa3a3fc2 sys/arch/macppc/macppc/cpu.c
--- a/sys/arch/macppc/macppc/cpu.c Thu Jan 09 16:27:57 2020 +0000
+++ b/sys/arch/macppc/macppc/cpu.c Thu Jan 09 16:35:03 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.69 2019/12/20 21:05:33 ad Exp $ */
+/* $NetBSD: cpu.c,v 1.70 2020/01/09 16:35:03 ad Exp $ */
/*-
* Copyright (c) 2001 Tsubai Masanari.
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.69 2019/12/20 21:05:33 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.70 2020/01/09 16:35:03 ad Exp $");
#include "opt_ppcparam.h"
#include "opt_multiprocessor.h"
@@ -175,7 +175,7 @@
core = package & 1;
package >>= 1;
}
- cpu_topology_set(ci, package, core, 0, 0);
+ cpu_topology_set(ci, package, core, 0, 0, false);
if (ci->ci_khz == 0) {
cpu_OFgetspeed(self, ci);
diff -r 25029195c9e7 -r ab36fa3a3fc2 sys/arch/mips/mips/cpu_subr.c
--- a/sys/arch/mips/mips/cpu_subr.c Thu Jan 09 16:27:57 2020 +0000
+++ b/sys/arch/mips/mips/cpu_subr.c Thu Jan 09 16:35:03 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_subr.c,v 1.44 2019/12/31 13:07:11 ad Exp $ */
+/* $NetBSD: cpu_subr.c,v 1.45 2020/01/09 16:35:03 ad Exp $ */
/*-
* Copyright (c) 2010, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.44 2019/12/31 13:07:11 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.45 2020/01/09 16:35:03 ad Exp $");
#include "opt_cputype.h"
#include "opt_ddb.h"
@@ -189,7 +189,7 @@
ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip;
ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count;
- cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id, 0);
+ cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id, 0, false);
pmap_md_alloc_ephemeral_address_space(ci);
diff -r 25029195c9e7 -r ab36fa3a3fc2 sys/arch/x86/x86/cpu_topology.c
--- a/sys/arch/x86/x86/cpu_topology.c Thu Jan 09 16:27:57 2020 +0000
+++ b/sys/arch/x86/x86/cpu_topology.c Thu Jan 09 16:35:03 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_topology.c,v 1.16 2019/12/20 21:05:34 ad Exp $ */
+/* $NetBSD: cpu_topology.c,v 1.17 2020/01/09 16:35:03 ad Exp $ */
/*-
* Copyright (c) 2009 Mindaugas Rasiukevicius <rmind at NetBSD org>,
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.16 2019/12/20 21:05:34 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_topology.c,v 1.17 2020/01/09 16:35:03 ad Exp $");
#include "acpica.h"
@@ -95,14 +95,14 @@
case CPUVENDOR_INTEL:
if (cpu_family < 6) {
cpu_topology_set(ci, package_id, core_id, smt_id,
- numa_id);
+ numa_id, false);
return;
}
break;
case CPUVENDOR_AMD:
if (cpu_family < 0xf) {
cpu_topology_set(ci, package_id, core_id, smt_id,
- numa_id);
+ numa_id, false);
return;
}
break;
@@ -210,5 +210,5 @@
smt_id = __SHIFTOUT(apic_id, smt_mask);
}
- cpu_topology_set(ci, package_id, core_id, smt_id, numa_id);
+ cpu_topology_set(ci, package_id, core_id, smt_id, numa_id, false);
}
diff -r 25029195c9e7 -r ab36fa3a3fc2 sys/kern/kern_runq.c
--- a/sys/kern/kern_runq.c Thu Jan 09 16:27:57 2020 +0000
+++ b/sys/kern/kern_runq.c Thu Jan 09 16:35:03 2020 +0000
@@ -1,7 +1,7 @@
-/* $NetBSD: kern_runq.c,v 1.56 2020/01/08 17:38:42 ad Exp $ */
+/* $NetBSD: kern_runq.c,v 1.57 2020/01/09 16:35:03 ad Exp $ */
/*-
- * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -56,7 +56,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.56 2020/01/08 17:38:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.57 2020/01/09 16:35:03 ad Exp $");
#include "opt_dtrace.h"
@@ -446,15 +446,89 @@
}
/*
+ * Find a CPU to run LWP "l". Look for the CPU with the lowest priority
+ * thread. In case of equal priority, prefer first class CPUs, and amongst
+ * the remainder choose the CPU with the fewest runqueue entries.
+ */
+static struct cpu_info * __noinline
+sched_bestcpu(struct lwp *l)
+{
+ struct cpu_info *bestci, *curci, *pivot, *next;
+ struct schedstate_percpu *bestspc, *curspc;
+ pri_t bestpri, curpri;
+
+ pivot = l->l_cpu;
+ curci = pivot;
+ bestci = NULL;
+ bestspc = NULL;
+ bestpri = PRI_COUNT;
+ do {
+ if ((next = cpu_lookup(cpu_index(curci) + 1)) == NULL) {
+ /* Reached the end, start from the beginning. */
+ next = cpu_lookup(0);
+ }
+ if (!sched_migratable(l, curci)){
+ continue;
+ }
+
+ curspc = &curci->ci_schedstate;
+ curpri = MAX(curspc->spc_curpriority, curspc->spc_maxpriority);
+
+ if (bestci == NULL) {
+ bestci = curci;
+ bestspc = curspc;
+ bestpri = curpri;
+ continue;
+ }
+ if (curpri > bestpri) {
+ continue;
+ }
+ if (curpri == bestpri) {
+ /* Prefer first class CPUs over others. */
+ if ((curspc->spc_flags & SPCF_1STCLASS) == 0 &&
+ (bestspc->spc_flags & SPCF_1STCLASS) != 0) {
+ continue;
+ }
+ /*
+ * Pick the least busy CPU. Make sure this is not
+ * <=, otherwise it defeats the above preference.
+ */
+ if (bestspc->spc_count < curspc->spc_count) {
+ continue;
+ }
+ }
+
+ bestpri = curpri;
+ bestci = curci;
+ bestspc = curspc;
+
+ /* If this CPU is idle and 1st class, we're done. */
+ if ((curspc->spc_flags & (SPCF_IDLE | SPCF_1STCLASS)) ==
+ (SPCF_IDLE | SPCF_1STCLASS)) {
+ break;
+ }
+
+ /*
+ * XXXAD After execve, likely still resident on the same
+ * package as the parent; should teleport to a different
+ * package to maximise bus bandwidth / cache availability.
+ * SMT & non-SMT cases are different.
+ */
+ } while (curci = next, curci != pivot);
+ return bestci;
+}
+
+/*
* Estimate the migration of LWP to the other CPU.
* Take and return the CPU, if migration is needed.
*/
struct cpu_info *
sched_takecpu(struct lwp *l)
{
- struct cpu_info *ci, *tci, *pivot, *next;
- struct schedstate_percpu *spc, *ici_spc;
- pri_t eprio, lpri, pri;
+ struct schedstate_percpu *spc, *tspc;
+ struct cpu_info *ci, *tci;
+ int flags;
+ pri_t eprio;
KASSERT(lwp_locked(l, NULL));
@@ -467,33 +541,28 @@
eprio = lwp_eprio(l);
/*
- * For new LWPs (LSIDL), l_cpu was inherited from the parent when
- * the LWP was created (and is probably still curcpu at this point).
- * The child will initially be in close communication with the
- * parent and share VM context and cache state. Look for an idle
- * SMT sibling to run it, and failing that run on the same CPU as
- * the parent.
+ * Look within the current CPU core.
+ *
+ * - For new LWPs (LSIDL), l_cpu was inherited from the parent when
+ * the LWP was created (and is probably still curcpu at this
+ * point). The child will initially be in close communication
+ * with the parent and share VM context and cache state. Look for
+ * an idle SMT sibling to run it, and failing that run on the same
Home |
Main Index |
Thread Index |
Old Index