[src/trunk]: src/lib/libc/stdio for fread(3) and fwrite(3) check for (size * ...

[jdolecek <jdolecek%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/eb1d7fe6f633
branches:  trunk
changeset: 959121:eb1d7fe6f633
user:      jdolecek <jdolecek%NetBSD.org@localhost>
date:      Mon Feb 01 17:50:53 2021 +0000

description:
for fread(3) and fwrite(3) check for (size * nmemb) size_t overflow, and
error out with EOVERFLOW if it happens; this is less silly answer
to a silly call than returning some randomly wrapped length

change adapted from OpenBSD

FreeBSD has a similar check, but they return EINVAL instead, feel
free to adjust if SUS or other standard mandates specific value

suggested by Kamil Rytarowski

diffstat:

 lib/libc/stdio/fread.3  |  13 +++++++++++--
 lib/libc/stdio/fread.c  |  17 +++++++++++++++--
 lib/libc/stdio/fwrite.c |  16 ++++++++++++++--
 3 files changed, 40 insertions(+), 6 deletions(-)

diffs (123 lines):

diff -r 34124128d288 -r eb1d7fe6f633 lib/libc/stdio/fread.3
--- a/lib/libc/stdio/fread.3    Mon Feb 01 17:49:29 2021 +0000
+++ b/lib/libc/stdio/fread.3    Mon Feb 01 17:50:53 2021 +0000
@@ -1,4 +1,4 @@
-.\"    $NetBSD: fread.3,v 1.15 2011/09/11 04:55:48 jruoho Exp $
+.\"    $NetBSD: fread.3,v 1.16 2021/02/01 17:50:53 jdolecek Exp $
 .\"
 .\" Copyright (c) 1990, 1991, 1993
 .\"    The Regents of the University of California.  All rights reserved.
@@ -33,7 +33,7 @@
 .\"
 .\"     @(#)fread.3    8.2 (Berkeley) 3/8/94
 .\"
-.Dd September 11, 2011
+.Dd February 1, 2020
 .Dt FREAD 3
 .Os
 .Sh NAME
@@ -96,6 +96,15 @@
 is 0, the functions return 0 and the state of
 .Fa stream
 remains unchanged.
+.Pp
+If the product of
+.Fa size
+and
+.Fa nmemb
+results in size_t overflow, 0 is returned and errno
+is set to
+.Er EOVERFLOW .
+
 If an error occurs, or the end-of-file is reached,
 the return value is a short object count (or zero).
 .Pp
diff -r 34124128d288 -r eb1d7fe6f633 lib/libc/stdio/fread.c
--- a/lib/libc/stdio/fread.c    Mon Feb 01 17:49:29 2021 +0000
+++ b/lib/libc/stdio/fread.c    Mon Feb 01 17:50:53 2021 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: fread.c,v 1.24 2021/01/31 16:18:22 jdolecek Exp $      */
+/*     $NetBSD: fread.c,v 1.25 2021/02/01 17:50:53 jdolecek Exp $      */
 
 /*-
  * Copyright (c) 1990, 1993
@@ -37,7 +37,7 @@
 #if 0
 static char sccsid[] = "@(#)fread.c    8.2 (Berkeley) 12/11/93";
 #else
-__RCSID("$NetBSD: fread.c,v 1.24 2021/01/31 16:18:22 jdolecek Exp $");
+__RCSID("$NetBSD: fread.c,v 1.25 2021/02/01 17:50:53 jdolecek Exp $");
 #endif
 #endif /* LIBC_SCCS and not lint */
 
@@ -48,6 +48,8 @@
 #include "reentrant.h"
 #include "local.h"
 
+#define MUL_NO_OVERFLOW        (1UL << (sizeof(size_t) * 4))
+
 size_t
 fread(void *buf, size_t size, size_t count, FILE *fp)
 {
@@ -57,6 +59,17 @@
        size_t total;
 
        _DIAGASSERT(fp != NULL);
+
+       /*
+        * Extension:  Catch integer overflow
+        */
+       if ((size >= MUL_NO_OVERFLOW || count >= MUL_NO_OVERFLOW) &&
+           size > 0 && count > SIZE_MAX / size) {
+               errno = EOVERFLOW;
+               fp->_flags |= __SERR;
+               return (0);
+       }
+
        /*
         * The ANSI standard requires a return value of 0 for a count
         * or a size of 0.  Whilst ANSI imposes no such requirements on
diff -r 34124128d288 -r eb1d7fe6f633 lib/libc/stdio/fwrite.c
--- a/lib/libc/stdio/fwrite.c   Mon Feb 01 17:49:29 2021 +0000
+++ b/lib/libc/stdio/fwrite.c   Mon Feb 01 17:50:53 2021 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: fwrite.c,v 1.18 2018/02/04 01:13:45 mrg Exp $  */
+/*     $NetBSD: fwrite.c,v 1.19 2021/02/01 17:50:53 jdolecek Exp $     */
 
 /*-
  * Copyright (c) 1990, 1993
@@ -37,7 +37,7 @@
 #if 0
 static char sccsid[] = "@(#)fwrite.c   8.1 (Berkeley) 6/4/93";
 #else
-__RCSID("$NetBSD: fwrite.c,v 1.18 2018/02/04 01:13:45 mrg Exp $");
+__RCSID("$NetBSD: fwrite.c,v 1.19 2021/02/01 17:50:53 jdolecek Exp $");
 #endif
 #endif /* LIBC_SCCS and not lint */
 
@@ -48,6 +48,8 @@
 #include "local.h"
 #include "fvwrite.h"
 
+#define MUL_NO_OVERFLOW        (1UL << (sizeof(size_t) * 4))
+
 /*
  * Write `count' objects (each size `size') from memory to the given file.
  * Return the number of whole objects written.
@@ -60,6 +62,16 @@
        struct __siov iov;
 
        /*
+        * Extension:  Catch integer overflow
+        */
+       if ((size >= MUL_NO_OVERFLOW || count >= MUL_NO_OVERFLOW) &&
+           size > 0 && count > SIZE_MAX / size) {
+               errno = EOVERFLOW;
+               fp->_flags |= __SERR;
+               return (0);
+       }
+
+       /*
         * SUSv2 requires a return value of 0 for a count or a size of 0.
         */
        if ((n = count * size) == 0)