Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys - Fix uneven performance with "bursty" vmem arenas. Adj...
details: https://anonhg.NetBSD.org/src/rev/29ef1edc6e89
branches: trunk
changeset: 1009303:29ef1edc6e89
user: ad <ad%NetBSD.org@localhost>
date: Sun Apr 19 21:11:42 2020 +0000
description:
- Fix uneven performance with "bursty" vmem arenas. Adjust locking so that
the mutex is acquired and released only once in the happy path. Align
tags to cachelines. Size the hash table according to the maximum count of
boundary tags over the interval just gone, not the instantaneous count,
and decay that maximum value by 50%+1 after each rehash. Round up to the
next power of two to eliminate divisions. Do the rehash check unlocked.
- Hash bucket size is sizeof(vmem_hashlist), not size of a pointer to same.
diffstat:
sys/kern/subr_vmem.c | 166 ++++++++++++++++++++++++++++----------------------
sys/sys/vmem_impl.h | 4 +-
2 files changed, 96 insertions(+), 74 deletions(-)
diffs (truncated from 503 to 300 lines):
diff -r 30ffc4afc66a -r 29ef1edc6e89 sys/kern/subr_vmem.c
--- a/sys/kern/subr_vmem.c Sun Apr 19 20:53:20 2020 +0000
+++ b/sys/kern/subr_vmem.c Sun Apr 19 21:11:42 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_vmem.c,v 1.100 2019/12/21 14:50:34 ad Exp $ */
+/* $NetBSD: subr_vmem.c,v 1.101 2020/04/19 21:11:42 ad Exp $ */
/*-
* Copyright (c)2006,2007,2008,2009 YAMAMOTO Takashi,
@@ -46,7 +46,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.100 2019/12/21 14:50:34 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.101 2020/04/19 21:11:42 ad Exp $");
#if defined(_KERNEL) && defined(_KERNEL_OPT)
#include "opt_ddb.h"
@@ -205,6 +205,7 @@
/* ---- boundary tag */
static int bt_refill(vmem_t *vm);
+static int bt_refill_locked(vmem_t *vm);
static void *
pool_page_alloc_vmem_meta(struct pool *pp, int flags)
@@ -234,13 +235,13 @@
};
static int
-bt_refill(vmem_t *vm)
+bt_refill_locked(vmem_t *vm)
{
bt_t *bt;
- VMEM_LOCK(vm);
+ VMEM_ASSERT_LOCKED(vm);
+
if (vm->vm_nfreetags > BT_MINRESERVE) {
- VMEM_UNLOCK(vm);
return 0;
}
@@ -269,29 +270,40 @@
}
if (vm->vm_nfreetags <= BT_MINRESERVE) {
- VMEM_UNLOCK(vm);
return ENOMEM;
}
- VMEM_UNLOCK(vm);
-
if (kmem_meta_arena != NULL) {
+ VMEM_UNLOCK(vm);
(void)bt_refill(kmem_arena);
(void)bt_refill(kmem_va_meta_arena);
(void)bt_refill(kmem_meta_arena);
+ VMEM_LOCK(vm);
}
return 0;
}
+static int
+bt_refill(vmem_t *vm)
+{
+ int rv;
+
+ VMEM_LOCK(vm);
+ rv = bt_refill_locked(vm);
+ VMEM_UNLOCK(vm);
+ return rv;
+}
+
static bt_t *
bt_alloc(vmem_t *vm, vm_flag_t flags)
{
bt_t *bt;
- VMEM_LOCK(vm);
+
+ VMEM_ASSERT_LOCKED(vm);
+
while (vm->vm_nfreetags <= BT_MINRESERVE && (flags & VM_POPULATING) == 0) {
- VMEM_UNLOCK(vm);
- if (bt_refill(vm)) {
+ if (bt_refill_locked(vm)) {
if ((flags & VM_NOSLEEP) != 0) {
return NULL;
}
@@ -306,14 +318,12 @@
*/
vmem_kick_pdaemon();
- kpause("btalloc", false, 1, NULL);
+ kpause("btalloc", false, 1, &vm->vm_lock);
}
- VMEM_LOCK(vm);
}
bt = LIST_FIRST(&vm->vm_freetags);
LIST_REMOVE(bt, bt_freelist);
vm->vm_nfreetags--;
- VMEM_UNLOCK(vm);
return bt;
}
@@ -322,10 +332,10 @@
bt_free(vmem_t *vm, bt_t *bt)
{
- VMEM_LOCK(vm);
+ VMEM_ASSERT_LOCKED(vm);
+
LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
vm->vm_nfreetags++;
- VMEM_UNLOCK(vm);
}
static void
@@ -334,9 +344,10 @@
bt_t *t;
LIST_HEAD(, vmem_btag) tofree;
+ VMEM_ASSERT_LOCKED(vm);
+
LIST_INIT(&tofree);
- VMEM_LOCK(vm);
while (vm->vm_nfreetags > freelimit) {
bt_t *bt = LIST_FIRST(&vm->vm_freetags);
LIST_REMOVE(bt, bt_freelist);
@@ -423,7 +434,7 @@
unsigned int hash;
hash = hash32_buf(&addr, sizeof(addr), HASH32_BUF_INIT);
- list = &vm->vm_hashlist[hash % vm->vm_hashsize];
+ list = &vm->vm_hashlist[hash & vm->vm_hashmask];
return list;
}
@@ -463,7 +474,9 @@
list = bt_hashhead(vm, bt->bt_start);
LIST_INSERT_HEAD(list, bt, bt_hashlist);
- vm->vm_nbusytag++;
+ if (++vm->vm_nbusytag > vm->vm_maxbusytag) {
+ vm->vm_maxbusytag = vm->vm_nbusytag;
+ }
vm->vm_inuse += bt->bt_size;
}
@@ -676,8 +689,8 @@
uvm_km_kmem_alloc, uvm_km_kmem_free, kmem_va_meta_arena,
0, VM_NOSLEEP | VM_BOOTSTRAP, IPL_VM);
- pool_init(&vmem_btag_pool, sizeof(bt_t), 0, 0, PR_PHINPAGE,
- "vmembt", &pool_allocator_vmem_meta, IPL_VM);
+ pool_init(&vmem_btag_pool, sizeof(bt_t), coherency_unit, 0,
+ PR_PHINPAGE, "vmembt", &pool_allocator_vmem_meta, IPL_VM);
}
#endif /* defined(_KERNEL) */
@@ -688,6 +701,7 @@
bt_t *btspan;
bt_t *btfree;
+ VMEM_ASSERT_LOCKED(vm);
KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0);
KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0);
KASSERT(spanbttype == BT_TYPE_SPAN ||
@@ -711,12 +725,10 @@
btfree->bt_start = addr;
btfree->bt_size = size;
- VMEM_LOCK(vm);
bt_insseg_tail(vm, btspan);
bt_insseg(vm, btfree, btspan);
bt_insfree(vm, btfree);
vm->vm_size += size;
- VMEM_UNLOCK(vm);
return 0;
}
@@ -728,24 +740,24 @@
#if defined(QCACHE)
qc_destroy(vm);
#endif /* defined(QCACHE) */
- if (vm->vm_hashlist != NULL) {
- int i;
+ VMEM_LOCK(vm);
- for (i = 0; i < vm->vm_hashsize; i++) {
- bt_t *bt;
+ for (int i = 0; i < vm->vm_hashsize; i++) {
+ bt_t *bt;
- while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) {
- KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC);
- bt_free(vm, bt);
- }
- }
- if (vm->vm_hashlist != &vm->vm_hash0) {
- xfree(vm->vm_hashlist,
- sizeof(struct vmem_hashlist *) * vm->vm_hashsize);
+ while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) {
+ KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC);
+ LIST_REMOVE(bt, bt_hashlist);
+ bt_free(vm, bt);
}
}
+ /* bt_freetrim() drops the lock. */
bt_freetrim(vm, 0);
+ if (vm->vm_hashlist != &vm->vm_hash0) {
+ xfree(vm->vm_hashlist,
+ sizeof(struct vmem_hashlist) * vm->vm_hashsize);
+ }
VMEM_CONDVAR_DESTROY(vm);
VMEM_LOCK_DESTROY(vm);
@@ -758,6 +770,8 @@
vmem_addr_t addr;
int rc;
+ VMEM_ASSERT_LOCKED(vm);
+
if (vm->vm_importfn == NULL) {
return EINVAL;
}
@@ -766,18 +780,23 @@
size *= 16;
}
+ VMEM_UNLOCK(vm);
if (vm->vm_flags & VM_XIMPORT) {
rc = __FPTRCAST(vmem_ximport_t *, vm->vm_importfn)(vm->vm_arg,
size, &size, flags, &addr);
} else {
rc = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
}
+ VMEM_LOCK(vm);
+
if (rc) {
return ENOMEM;
}
if (vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN) != 0) {
+ VMEM_UNLOCK(vm);
(*vm->vm_releasefn)(vm->vm_arg, addr, size);
+ VMEM_LOCK(vm);
return ENOMEM;
}
@@ -795,8 +814,11 @@
KASSERT(newhashsize > 0);
+ /* Round hash size up to a power of 2. */
+ newhashsize = 1 << (ilog2(newhashsize) + 1);
+
newhashlist =
- xmalloc(sizeof(struct vmem_hashlist *) * newhashsize, flags);
+ xmalloc(sizeof(struct vmem_hashlist) * newhashsize, flags);
if (newhashlist == NULL) {
return ENOMEM;
}
@@ -804,15 +826,21 @@
LIST_INIT(&newhashlist[i]);
}
- if (!VMEM_TRYLOCK(vm)) {
- xfree(newhashlist,
- sizeof(struct vmem_hashlist *) * newhashsize);
- return EBUSY;
+ VMEM_LOCK(vm);
+ /* Decay back to a small hash slowly. */
+ if (vm->vm_maxbusytag >= 2) {
+ vm->vm_maxbusytag = vm->vm_maxbusytag / 2 - 1;
+ if (vm->vm_nbusytag > vm->vm_maxbusytag) {
+ vm->vm_maxbusytag = vm->vm_nbusytag;
+ }
+ } else {
+ vm->vm_maxbusytag = vm->vm_nbusytag;
}
oldhashlist = vm->vm_hashlist;
oldhashsize = vm->vm_hashsize;
vm->vm_hashlist = newhashlist;
vm->vm_hashsize = newhashsize;
+ vm->vm_hashmask = newhashsize - 1;
if (oldhashlist == NULL) {
VMEM_UNLOCK(vm);
return 0;
@@ -827,7 +855,7 @@
if (oldhashlist != &vm->vm_hash0) {
xfree(oldhashlist,
- sizeof(struct vmem_hashlist *) * oldhashsize);
+ sizeof(struct vmem_hashlist) * oldhashsize);
}
Home |
Main Index |
Thread Index |
Old Index