[src/trunk]: src/sys/kern Kernel Heap Hardening: use bitmaps on all off-page ...

[maxv <maxv%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/2f89aecd69d4
branches:  trunk
changeset: 453545:2f89aecd69d4
user:      maxv <maxv%NetBSD.org@localhost>
date:      Sat Aug 17 12:37:49 2019 +0000

description:
Kernel Heap Hardening: use bitmaps on all off-page pools. This migrates 29
MI pools on amd64 from linked lists to bitmaps, which have higher security
properties.

Then, change the computation of the size of the PH pools: take into account
the bitmap area available by default in the ph_u2 union, and don't go with
&phpool[>0] if &phpool[0] already has enough space to embed a bitmap.

The pools that are migrated in this change all use bitmaps small enough to
fit in &phpool[0], therefore there is no increase in memory consumption.

diffstat:

 sys/kern/subr_pool.c |  41 +++++++++++++++++++++++------------------
 1 files changed, 23 insertions(+), 18 deletions(-)

diffs (112 lines):

diff -r 41a628963faf -r 2f89aecd69d4 sys/kern/subr_pool.c
--- a/sys/kern/subr_pool.c      Sat Aug 17 09:44:01 2019 +0000
+++ b/sys/kern/subr_pool.c      Sat Aug 17 12:37:49 2019 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: subr_pool.c,v 1.255 2019/08/16 10:41:35 maxv Exp $     */
+/*     $NetBSD: subr_pool.c,v 1.256 2019/08/17 12:37:49 maxv Exp $     */
 
 /*
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015, 2018
@@ -33,7 +33,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.255 2019/08/16 10:41:35 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.256 2019/08/17 12:37:49 maxv Exp $");
 
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
@@ -81,7 +81,7 @@
 #define        PHPOOL_MAX      8
 static struct pool phpool[PHPOOL_MAX];
 #define        PHPOOL_FREELIST_NELEM(idx) \
-       (((idx) == 0) ? 0 : BITMAP_SIZE * (1 << (idx)))
+       (((idx) == 0) ? BITMAP_MIN_SIZE : BITMAP_SIZE * (1 << (idx)))
 
 #if defined(KASAN)
 #define POOL_REDZONE
@@ -162,6 +162,7 @@
 typedef uint32_t pool_item_bitmap_t;
 #define        BITMAP_SIZE     (CHAR_BIT * sizeof(pool_item_bitmap_t))
 #define        BITMAP_MASK     (BITMAP_SIZE - 1)
+#define        BITMAP_MIN_SIZE (CHAR_BIT * sizeof(((struct pool_item_header *)NULL)->ph_u2))
 
 struct pool_item_header {
        /* Page headers */
@@ -201,6 +202,9 @@
 
 #define PHSIZE ALIGN(sizeof(struct pool_item_header))
 
+CTASSERT(offsetof(struct pool_item_header, ph_u2) +
+    BITMAP_MIN_SIZE / CHAR_BIT == sizeof(struct pool_item_header));
+
 #if defined(DIAGNOSTIC) && !defined(KASAN)
 #define POOL_CHECK_MAGIC
 #endif
@@ -588,13 +592,11 @@
                size_t sz;
 
                nelem = PHPOOL_FREELIST_NELEM(idx);
+               KASSERT(nelem != 0);
                snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
                    "phpool-%d", nelem);
-               sz = sizeof(struct pool_item_header);
-               if (nelem) {
-                       sz = offsetof(struct pool_item_header,
-                           ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
-               }
+               sz = offsetof(struct pool_item_header,
+                   ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
                pool_init(&phpool[idx], sz, 0, 0, 0,
                    phpool_names[idx], &pool_allocator_meta, IPL_VM);
        }
@@ -657,12 +659,16 @@
        }
 
        /*
+        * If we're off-page, go with a bitmap.
+        */
+       if (!(pp->pr_roflags & PR_PHINPAGE)) {
+               return true;
+       }
+
+       /*
         * If we're on-page, and the page header can already contain a bitmap
         * big enough to cover all the items of the page, go with a bitmap.
         */
-       if (!(pp->pr_roflags & PR_PHINPAGE)) {
-               return false;
-       }
        bmapsize = roundup(PHSIZE, pp->pr_align) -
            offsetof(struct pool_item_header, ph_bitmap[0]);
        KASSERT(bmapsize % sizeof(pool_item_bitmap_t) == 0);
@@ -801,14 +807,15 @@
        }
 
        /*
-        * If we're off-page and use a bitmap, choose the appropriate pool to
-        * allocate page headers, whose size varies depending on the bitmap. If
-        * we're just off-page, take the first pool, no extra size. If we're
-        * on-page, nothing to do.
+        * If we're off-page, then we're using a bitmap; choose the appropriate
+        * pool to allocate page headers, whose size varies depending on the
+        * bitmap. If we're on-page, nothing to do.
         */
-       if (!(pp->pr_roflags & PR_PHINPAGE) && (pp->pr_roflags & PR_USEBMAP)) {
+       if (!(pp->pr_roflags & PR_PHINPAGE)) {
                int idx;
 
+               KASSERT(pp->pr_roflags & PR_USEBMAP);
+
                for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx);
                    idx++) {
                        /* nothing */
@@ -823,8 +830,6 @@
                            pp->pr_wchan, pp->pr_itemsperpage);
                }
                pp->pr_phpool = &phpool[idx];
-       } else if (!(pp->pr_roflags & PR_PHINPAGE)) {
-               pp->pr_phpool = &phpool[0];
        } else {
                pp->pr_phpool = NULL;
        }