[src/trunk]: src/sys/uvm mmap(2): Avoid arithmetic overflow in search for fre...

[riastradh <riastradh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/46b365b0ce3f
branches:  trunk
changeset: 372415:46b365b0ce3f
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Wed Nov 23 23:53:53 2022 +0000

description:
mmap(2): Avoid arithmetic overflow in search for free space.

PR kern/56900

Reported-by: syzbot+3833ae1d38037a263d05%syzkaller.appspotmail.com@localhost
https://syzkaller.appspot.com/bug?id=e542bcf59b2564cca1cb38c12f076fb08dcac37e

diffstat:

 sys/uvm/uvm_map.c |  19 ++++++++++++++++---
 1 files changed, 16 insertions(+), 3 deletions(-)

diffs (40 lines):

diff -r ac8a729c5b6d -r 46b365b0ce3f sys/uvm/uvm_map.c
--- a/sys/uvm/uvm_map.c Wed Nov 23 23:49:23 2022 +0000
+++ b/sys/uvm/uvm_map.c Wed Nov 23 23:53:53 2022 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: uvm_map.c,v 1.402 2022/06/08 16:55:00 macallan Exp $   */
+/*     $NetBSD: uvm_map.c,v 1.403 2022/11/23 23:53:53 riastradh Exp $  */
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.402 2022/06/08 16:55:00 macallan Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.403 2022/11/23 23:53:53 riastradh Exp $");
 
 #include "opt_ddb.h"
 #include "opt_pax.h"
@@ -1994,7 +1994,20 @@
        /* Try to find the space in the red-black tree */
 
        /* Check slot before any entry */
-       hint = topdown ? entry->next->start - length : entry->end;
+       if (topdown) {
+               KASSERTMSG(entry->next->start >= vm_map_min(map),
+                   "map=%p entry=%p entry->next=%p"
+                   " entry->next->start=0x%"PRIxVADDR" min=0x%"PRIxVADDR,
+                   map, entry, entry->next,
+                   entry->next->start, vm_map_min(map));
+               if (length > entry->next->start - vm_map_min(map))
+                       hint = vm_map_min(map); /* XXX goto wraparound? */
+               else
+                       hint = entry->next->start - length;
+               KASSERT(hint >= vm_map_min(map));
+       } else {
+               hint = entry->end;
+       }
        INVARIANTS();
        avail = uvm_map_space_avail(&hint, length, uoffset, align, flags,
            topdown, entry);