Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys - Add and use wrapper functions that take and acquire pa...
details: https://anonhg.NetBSD.org/src/rev/8b545ab204f0
branches: trunk
changeset: 968019:8b545ab204f0
user: ad <ad%NetBSD.org@localhost>
date: Tue Dec 31 22:42:50 2019 +0000
description:
- Add and use wrapper functions that take and acquire page interlocks, and pairs
of page interlocks. Require that the page interlock be held over calls to
uvm_pageactivate(), uvm_pagewire() and similar.
- Solve the concurrency problem with page replacement state. Rather than
updating the global state synchronously, set an intended state on
individual pages (active, inactive, enqueued, dequeued) while holding the
page interlock. After the interlock is released put the pages on a 128
entry per-CPU queue for their state changes to be made real in batch.
This results in in a ~400 fold decrease in contention on my test system.
Proposed on tech-kern but modified to use the page interlock rather than
atomics to synchronise as it's much easier to maintain that way, and
cheaper.
diffstat:
sys/kern/kern_idle.c | 8 +-
sys/miscfs/genfs/genfs_io.c | 12 +-
sys/ufs/lfs/lfs_pages.c | 8 +-
sys/ufs/lfs/lfs_vfsops.c | 8 +-
sys/ufs/lfs/ulfs_inode.c | 6 +-
sys/ufs/ufs/ufs_inode.c | 6 +-
sys/uvm/uvm.h | 8 +-
sys/uvm/uvm_anon.c | 6 +-
sys/uvm/uvm_aobj.c | 8 +-
sys/uvm/uvm_bio.c | 10 +-
sys/uvm/uvm_extern.h | 3 +-
sys/uvm/uvm_fault.c | 27 ++-
sys/uvm/uvm_glue.c | 24 ++-
sys/uvm/uvm_loan.c | 52 +++--
sys/uvm/uvm_map.c | 6 +-
sys/uvm/uvm_object.c | 8 +-
sys/uvm/uvm_page.c | 137 +++++++++++++--
sys/uvm/uvm_page.h | 24 ++-
sys/uvm/uvm_pager.c | 8 +-
sys/uvm/uvm_pdaemon.c | 12 +-
sys/uvm/uvm_pdpolicy.h | 18 +-
sys/uvm/uvm_pdpolicy_clock.c | 341 ++++++++++++++++++++++++++++++++++-----
sys/uvm/uvm_pdpolicy_clockpro.c | 131 +++++++++++---
23 files changed, 699 insertions(+), 172 deletions(-)
diffs (truncated from 2029 to 300 lines):
diff -r ef71d1baab2f -r 8b545ab204f0 sys/kern/kern_idle.c
--- a/sys/kern/kern_idle.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/kern/kern_idle.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_idle.c,v 1.28 2019/12/06 21:36:10 ad Exp $ */
+/* $NetBSD: kern_idle.c,v 1.29 2019/12/31 22:42:51 ad Exp $ */
/*-
* Copyright (c)2002, 2006, 2007 YAMAMOTO Takashi,
@@ -28,7 +28,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.28 2019/12/06 21:36:10 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.29 2019/12/31 22:42:51 ad Exp $");
#include <sys/param.h>
#include <sys/cpu.h>
@@ -39,7 +39,7 @@
#include <sys/proc.h>
#include <sys/atomic.h>
-#include <uvm/uvm.h> /* uvm_pageidlezero */
+#include <uvm/uvm.h> /* uvm_idle */
#include <uvm/uvm_extern.h>
void
@@ -81,7 +81,7 @@
sched_idle();
if (!sched_curcpu_runnable_p()) {
if ((spc->spc_flags & SPCF_OFFLINE) == 0) {
- uvm_pageidlezero();
+ uvm_idle();
}
if (!sched_curcpu_runnable_p()) {
cpu_idle();
diff -r ef71d1baab2f -r 8b545ab204f0 sys/miscfs/genfs/genfs_io.c
--- a/sys/miscfs/genfs/genfs_io.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/miscfs/genfs/genfs_io.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: genfs_io.c,v 1.82 2019/12/31 12:40:27 ad Exp $ */
+/* $NetBSD: genfs_io.c,v 1.83 2019/12/31 22:42:50 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.82 2019/12/31 12:40:27 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.83 2019/12/31 22:42:50 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -491,7 +491,9 @@
uvm_pagefree(pg);
continue;
}
+ uvm_pagelock(pg);
uvm_pageenqueue(pg);
+ uvm_pageunlock(pg);
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
}
@@ -1164,14 +1166,18 @@
if (tpg->offset < startoff || tpg->offset >= endoff)
continue;
if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
+ uvm_pagelock(tpg);
uvm_pagedeactivate(tpg);
+ uvm_pageunlock(tpg);
} else if (flags & PGO_FREE) {
pmap_page_protect(tpg, VM_PROT_NONE);
if (tpg->flags & PG_BUSY) {
tpg->flags |= freeflag;
if (pagedaemon) {
uvm_pageout_start(1);
+ uvm_pagelock(tpg);
uvm_pagedequeue(tpg);
+ uvm_pageunlock(tpg);
}
} else {
@@ -1603,7 +1609,9 @@
pg->flags |= PG_RELEASED;
} else {
pmap_clear_modify(pg);
+ uvm_pagelock(pg);
uvm_pageactivate(pg);
+ uvm_pageunlock(pg);
}
}
if (error) {
diff -r ef71d1baab2f -r 8b545ab204f0 sys/ufs/lfs/lfs_pages.c
--- a/sys/ufs/lfs/lfs_pages.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/lfs/lfs_pages.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lfs_pages.c,v 1.18 2019/12/20 20:54:48 ad Exp $ */
+/* $NetBSD: lfs_pages.c,v 1.19 2019/12/31 22:42:51 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.18 2019/12/20 20:54:48 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.19 2019/12/31 22:42:51 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_compat_netbsd.h"
@@ -338,7 +338,9 @@
* Wire the page so that
* pdaemon doesn't see it again.
*/
+ uvm_pagelock(pg);
uvm_pagewire(pg);
+ uvm_pageunlock(pg);
/* Suspended write flag */
pg->flags |= PG_DELWRI;
@@ -495,7 +497,9 @@
"lfsput2", 0);
mutex_enter(vp->v_interlock);
}
+ uvm_pagelock(pg);
uvm_pageactivate(pg);
+ uvm_pageunlock(pg);
}
ap->a_offlo = blkeof;
if (ap->a_offhi > 0 && ap->a_offhi <= ap->a_offlo) {
diff -r ef71d1baab2f -r 8b545ab204f0 sys/ufs/lfs/lfs_vfsops.c
--- a/sys/ufs/lfs/lfs_vfsops.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/lfs/lfs_vfsops.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: lfs_vfsops.c,v 1.366 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: lfs_vfsops.c,v 1.367 2019/12/31 22:42:51 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2007, 2007
@@ -61,7 +61,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lfs_vfsops.c,v 1.366 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lfs_vfsops.c,v 1.367 2019/12/31 22:42:51 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_lfs.h"
@@ -2054,7 +2054,9 @@
pgs[i]->flags |= PG_PAGEOUT;
uvm_pageout_start(1);
mutex_enter(vp->v_interlock);
+ uvm_pagelock(pgs[i]);
uvm_pageunwire(pgs[i]);
+ uvm_pageunlock(pgs[i]);
mutex_exit(vp->v_interlock);
}
}
@@ -2241,10 +2243,12 @@
if (pg->flags & PG_PAGEOUT)
uvm_pageout_done(1);
+ uvm_pagelock(pg);
if (pg->flags & PG_DELWRI) {
uvm_pageunwire(pg);
}
uvm_pageactivate(pg);
+ uvm_pageunlock(pg);
pg->flags &= ~(PG_CLEAN|PG_DELWRI|PG_PAGEOUT|PG_RELEASED);
DLOG((DLOG_PAGE, "pg[%d] = %p (vp %p off %" PRIx64 ")\n", i, pg,
vp, pg->offset));
diff -r ef71d1baab2f -r 8b545ab204f0 sys/ufs/lfs/ulfs_inode.c
--- a/sys/ufs/lfs/ulfs_inode.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/lfs/ulfs_inode.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ulfs_inode.c,v 1.22 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: ulfs_inode.c,v 1.23 2019/12/31 22:42:51 ad Exp $ */
/* from NetBSD: ufs_inode.c,v 1.95 2015/06/13 14:56:45 hannken Exp */
/*
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ulfs_inode.c,v 1.22 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ulfs_inode.c,v 1.23 2019/12/31 22:42:51 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_lfs.h"
@@ -243,7 +243,9 @@
}
pgs[i]->flags &= ~PG_CLEAN;
}
+ uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);
+ uvm_pageunlock(pgs[i]);
}
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
diff -r ef71d1baab2f -r 8b545ab204f0 sys/ufs/ufs/ufs_inode.c
--- a/sys/ufs/ufs/ufs_inode.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/ufs/ufs/ufs_inode.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ufs_inode.c,v 1.106 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: ufs_inode.c,v 1.107 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1991, 1993
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ufs_inode.c,v 1.106 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ufs_inode.c,v 1.107 2019/12/31 22:42:51 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_ffs.h"
@@ -279,7 +279,9 @@
}
pgs[i]->flags &= ~PG_CLEAN;
}
+ uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);
+ uvm_pageunlock(pgs[i]);
}
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);
diff -r ef71d1baab2f -r 8b545ab204f0 sys/uvm/uvm.h
--- a/sys/uvm/uvm.h Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/uvm/uvm.h Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm.h,v 1.72 2019/12/27 13:19:24 ad Exp $ */
+/* $NetBSD: uvm.h,v 1.73 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -87,6 +87,12 @@
/* entropy */
krndsource_t rs; /* entropy source */
+
+ /* uvmpdpol: queue of intended page status changes. */
+ struct vm_page **pdq; /* queue entries */
+ u_int pdqhead; /* current queue head */
+ u_int pdqtail; /* maximum number entries */
+ int pdqtime; /* last time queue cleared */
};
/*
diff -r ef71d1baab2f -r 8b545ab204f0 sys/uvm/uvm_anon.c
--- a/sys/uvm/uvm_anon.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/uvm/uvm_anon.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_anon.c,v 1.69 2019/12/13 20:10:22 ad Exp $ */
+/* $NetBSD: uvm_anon.c,v 1.70 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.69 2019/12/13 20:10:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.70 2019/12/31 22:42:51 ad Exp $");
#include "opt_uvmhist.h"
@@ -352,7 +352,9 @@
* Deactivate the page (to put it on a page queue).
*/
+ uvm_pagelock(pg);
uvm_pagedeactivate(pg);
+ uvm_pageunlock(pg);
if (pg->flags & PG_WANTED) {
pg->flags &= ~PG_WANTED;
wakeup(pg);
diff -r ef71d1baab2f -r 8b545ab204f0 sys/uvm/uvm_aobj.c
--- a/sys/uvm/uvm_aobj.c Tue Dec 31 21:35:25 2019 +0000
+++ b/sys/uvm/uvm_aobj.c Tue Dec 31 22:42:50 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uvm_aobj.c,v 1.132 2019/12/15 21:11:35 ad Exp $ */
+/* $NetBSD: uvm_aobj.c,v 1.133 2019/12/31 22:42:51 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.132 2019/12/15 21:11:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.133 2019/12/31 22:42:51 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@@ -738,7 +738,9 @@
case PGO_CLEANIT|PGO_DEACTIVATE:
case PGO_DEACTIVATE:
deactivate_it:
Home |
Main Index |
Thread Index |
Old Index