Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Introduce per-cpu IDTs
details: https://anonhg.NetBSD.org/src/rev/8f1f659739b9
branches: trunk
changeset: 1011858:8f1f659739b9
user: yamaguchi <yamaguchi%NetBSD.org@localhost>
date: Tue Jul 14 00:45:52 2020 +0000
description:
Introduce per-cpu IDTs
This is realized by following modifications:
- Add IDT pages and its allocation maps for each cpu in "struct cpu_info"
- Load per-cpu IDTs at cpu_init_idt(struct cpu_info*)
- Copy the IDT entries for cpu0 to other CPUs at attach
- These are, for example, exceptions, db, system calls, etc.
And, added a kernel option named PCPU_IDT to enable the feature.
diffstat:
sys/arch/amd64/amd64/db_interface.c | 8 +-
sys/arch/amd64/amd64/machdep.c | 100 +++++++++++++++++++++++++++++++----
sys/arch/amd64/conf/ALL | 5 +-
sys/arch/amd64/include/segments.h | 17 +++--
sys/arch/i386/conf/ALL | 5 +-
sys/arch/i386/i386/db_interface.c | 11 ++-
sys/arch/i386/i386/machdep.c | 78 ++++++++++++++++++++++++---
sys/arch/i386/i386/trap.c | 10 ++-
sys/arch/i386/include/segments.h | 18 +++---
sys/arch/x86/conf/files.x86 | 4 +-
sys/arch/x86/include/cpu.h | 9 ++-
sys/arch/x86/include/cpuvar.h | 3 +-
sys/arch/x86/include/pmap.h | 4 +-
sys/arch/x86/x86/cpu.c | 7 +-
sys/arch/x86/x86/hyperv.c | 100 +++++++++++++++++++++++++++++++----
sys/arch/x86/x86/idt.c | 45 ++++++++++++---
sys/arch/x86/x86/intr.c | 65 +++++++++++++++++------
sys/arch/x86/x86/lapic.c | 23 ++++---
sys/arch/x86/x86/pmap.c | 8 +--
sys/arch/x86/x86/svs.c | 8 +-
sys/arch/xen/x86/cpu.c | 9 ++-
sys/arch/xen/xen/hypervisor.c | 10 ++-
sys/dev/hyperv/vmbusvar.h | 4 +-
sys/dev/nvmm/x86/nvmm_x86_vmx.c | 12 ++-
24 files changed, 433 insertions(+), 130 deletions(-)
diffs (truncated from 1510 to 300 lines):
diff -r 778dd29e471d -r 8f1f659739b9 sys/arch/amd64/amd64/db_interface.c
--- a/sys/arch/amd64/amd64/db_interface.c Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/amd64/db_interface.c Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: db_interface.c,v 1.37 2020/02/29 15:00:28 christos Exp $ */
+/* $NetBSD: db_interface.c,v 1.38 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Mach Operating System
@@ -33,7 +33,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.37 2020/02/29 15:00:28 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.38 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -111,12 +111,14 @@
#ifdef MULTIPROCESSOR
#ifndef XENPV
+ struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
vector *handler = &Xintr_ddbipi;
+ idt_descriptor_t *idt = iv->iv_idt;
#if NLAPIC > 0
if (lapic_is_x2apic())
handler = &Xintr_x2apic_ddbipi;
#endif
- ddb_vec = idt_vec_alloc(0xf0, 0xff);
+ ddb_vec = idt_vec_alloc(iv, 0xf0, 0xff);
set_idtgate(&idt[ddb_vec], handler, 1, SDT_SYS386IGT, SEL_KPL,
GSEL(GCODE_SEL, SEL_KPL));
#else
diff -r 778dd29e471d -r 8f1f659739b9 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/amd64/machdep.c Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.355 2020/05/10 06:30:57 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.356 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.355 2020/05/10 06:30:57 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.356 2020/07/14 00:45:52 yamaguchi Exp $");
#include "opt_modular.h"
#include "opt_user_ldt.h"
@@ -1392,11 +1392,15 @@
char *gdtstore;
void
-setgate(struct gate_descriptor *gd, void *func, int ist, int type, int dpl, int sel)
+setgate(struct gate_descriptor *gd, void *func,
+ int ist, int type, int dpl, int sel)
{
+ vaddr_t vaddr;
+
+ vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
kpreempt_disable();
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+ pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
gd->gd_looffset = (uint64_t)func & 0xffff;
gd->gd_selector = sel;
@@ -1410,20 +1414,23 @@
gd->gd_xx2 = 0;
gd->gd_xx3 = 0;
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
+ pmap_changeprot_local(vaddr, VM_PROT_READ);
kpreempt_enable();
}
void
unsetgate(struct gate_descriptor *gd)
{
+ vaddr_t vaddr;
+
+ vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
kpreempt_disable();
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+ pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
memset(gd, 0, sizeof (*gd));
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
+ pmap_changeprot_local(vaddr, VM_PROT_READ);
kpreempt_enable();
}
@@ -1470,10 +1477,12 @@
}
void
-cpu_init_idt(void)
+cpu_init_idt(struct cpu_info *ci)
{
struct region_descriptor region;
+ idt_descriptor_t *idt;
+ idt = ci->ci_idtvec.iv_idt;
setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1);
lidt(®ion);
}
@@ -1677,6 +1686,8 @@
extern void consinit(void);
struct region_descriptor region;
struct mem_segment_descriptor *ldt_segp;
+ struct idt_vec *iv;
+ idt_descriptor_t *idt;
int x;
struct pcb *pcb;
extern vaddr_t lwp0uarea;
@@ -1807,7 +1818,9 @@
pmap_update(pmap_kernel());
- idt = (idt_descriptor_t *)idt_vaddr;
+ iv = &(cpu_info_primary.ci_idtvec);
+ idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary));
+ idt = iv->iv_idt;
gdtstore = (char *)gdt_vaddr;
ldtstore = (char *)ldt_vaddr;
@@ -1872,7 +1885,7 @@
sel = SEL_KPL;
ist = 0;
- idt_vec_reserve(x);
+ idt_vec_reserve(iv, x);
switch (x) {
case 1: /* DB */
@@ -1902,7 +1915,7 @@
}
/* new-style interrupt gate for syscalls */
- idt_vec_reserve(128);
+ idt_vec_reserve(iv, 128);
set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL,
GSEL(GCODE_SEL, SEL_KPL));
@@ -1920,7 +1933,7 @@
panic("HYPERVISOR_set_callbacks() failed");
#endif /* XENPV */
- cpu_init_idt();
+ cpu_init_idt(&cpu_info_primary);
#ifdef XENPV
xen_init_ksyms();
@@ -1961,6 +1974,14 @@
void
cpu_reset(void)
{
+#ifndef XENPV
+ idt_descriptor_t *idt;
+ vaddr_t vaddr;
+
+ idt = cpu_info_primary.ci_idtvec.iv_idt;
+ vaddr = (vaddr_t)idt;
+#endif
+
x86_disable_intr();
#ifdef XENPV
@@ -1974,7 +1995,7 @@
* invalid and causing a fault.
*/
kpreempt_disable();
- pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+ pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
memset((void *)idt, 0, NIDT * sizeof(idt[0]));
kpreempt_enable();
breakpoint();
@@ -2321,3 +2342,56 @@
return false;
#endif
}
+
+static void
+idt_vec_copy(struct idt_vec *dst, struct idt_vec *src)
+{
+ idt_descriptor_t *idt_dst;
+
+ idt_dst = dst->iv_idt;
+
+ kpreempt_disable();
+ pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE);
+
+ memcpy(idt_dst, src->iv_idt, PAGE_SIZE);
+ memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap));
+
+ pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ);
+ kpreempt_enable();
+}
+
+void
+idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid)
+{
+ vaddr_t va;
+
+ if (cid != cpu_index(&cpu_info_primary) &&
+ idt_vec_is_pcpu()) {
+#ifdef __HAVE_PCPU_AREA
+ va = (vaddr_t)&pcpuarea->ent[cid].idt;
+#else
+ struct vm_page *pg;
+
+ va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
+ UVM_KMF_VAONLY);
+ pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
+ if (pg == NULL) {
+ panic("failed to allocate a page for IDT");
+ }
+ pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
+ VM_PROT_READ|VM_PROT_WRITE, 0);
+ pmap_update(pmap_kernel());
+#endif
+
+ memset((void *)va, 0, PAGE_SIZE);
+#ifndef XENPV
+ pmap_changeprot_local(va, VM_PROT_READ);
+#endif
+ pmap_update(pmap_kernel());
+
+ iv->iv_idt = (void *)va;
+ idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec));
+ } else {
+ iv->iv_idt = (void *)idt_vaddr;
+ }
+}
diff -r 778dd29e471d -r 8f1f659739b9 sys/arch/amd64/conf/ALL
--- a/sys/arch/amd64/conf/ALL Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/conf/ALL Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: ALL,v 1.155 2020/06/29 23:58:44 riastradh Exp $
+# $NetBSD: ALL,v 1.156 2020/07/14 00:45:52 yamaguchi Exp $
# From NetBSD: GENERIC,v 1.787 2006/10/01 18:37:54 bouyer Exp
#
# ALL machine description file
@@ -17,7 +17,7 @@
options INCLUDE_CONFIG_FILE # embed config file in kernel binary
-#ident "ALL-$Revision: 1.155 $"
+#ident "ALL-$Revision: 1.156 $"
maxusers 64 # estimated number of users
@@ -29,6 +29,7 @@
#options PAE # PAE mode (36 bits physical addressing)
makeoptions SPECTRE_V2_GCC_MITIGATION=1 # GCC Spectre variant 2
# migitation
+options PCPU_IDT # Per CPU IDT
# CPU features
acpicpu* at cpu? # ACPI CPU (including frequency scaling)
diff -r 778dd29e471d -r 8f1f659739b9 sys/arch/amd64/include/segments.h
--- a/sys/arch/amd64/include/segments.h Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/include/segments.h Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: segments.h,v 1.36 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: segments.h,v 1.37 2020/07/14 00:45:52 yamaguchi Exp $ */
/*
* Copyright (c) 1990 The Regents of the University of California.
@@ -238,7 +238,6 @@
#else
typedef struct gate_descriptor idt_descriptor_t;
#endif /* XENPV */
-extern idt_descriptor_t *idt;
extern char *gdtstore;
extern char *ldtstore;
@@ -251,14 +250,16 @@
int, int, int);
void set_mem_segment(struct mem_segment_descriptor *, void *, size_t,
int, int, int, int, int);
-void cpu_init_idt(void);
void update_descriptor(void *, void *);
-
-void idt_vec_reserve(int);
-int idt_vec_alloc(int, int);
-void idt_vec_set(int, void (*)(void));
-void idt_vec_free(int);
+struct idt_vec;
+void idt_vec_reserve(struct idt_vec *, int);
+int idt_vec_alloc(struct idt_vec *, int, int);
+void idt_vec_set(struct idt_vec *, int, void (*)(void));
+void idt_vec_free(struct idt_vec *, int);
+void idt_vec_init_cpu_md(struct idt_vec *, cpuid_t);
Home |
Main Index |
Thread Index |
Old Index