[src/trunk]: src/lib/libc/stdlib Avoid undefined behavior in the rand48(3) im...

[kamil <kamil%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/89fb74d6d091
branches:  trunk
changeset: 1007534:89fb74d6d091
user:      kamil <kamil%NetBSD.org@localhost>
date:      Sat Feb 22 14:07:57 2020 +0000

description:
Avoid undefined behavior in the rand48(3) implementation

Instead of implicid promotion to signed int,
explicitly cast the arguments to unsigned int.

_rand48.c:53:27, signed integer overflow:
58989 * 58970 cannot be represented in type 'int'

_rand48.c:53:38, signed integer overflow:
-2093025904 + -1496809120 cannot be represented in type 'int'

_rand48.c:53:57, signed integer overflow:
57068 * 42787 cannot be represented in type 'int'

New and old code produce the same code as tested with:

#include <stdio.h>
#include <stdlib.h>

#define COUNT 1000 * 1000

int
main(void)
{
        FILE *fp;
        int i;

        fp = fopen("numbers.txt", "w+");
        if (!fp)
                abort();

        for(i = 0; i < COUNT; i++) {
                fprintf(fp, "%f\n", drand48());
                fprintf(fp, "%ld\n", lrand48());
                fprintf(fp, "%ld\n", mrand48());
        }

        fclose(fp);

        return 0;
}

diffstat:

 lib/libc/stdlib/_rand48.c |  8 +++++---
 1 files changed, 5 insertions(+), 3 deletions(-)

diffs (29 lines):

diff -r 3c5a6567d507 -r 89fb74d6d091 lib/libc/stdlib/_rand48.c
--- a/lib/libc/stdlib/_rand48.c Sat Feb 22 14:06:05 2020 +0000
+++ b/lib/libc/stdlib/_rand48.c Sat Feb 22 14:07:57 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: _rand48.c,v 1.8 2020/02/22 11:24:47 kamil Exp $        */
+/*     $NetBSD: _rand48.c,v 1.9 2020/02/22 14:07:57 kamil Exp $        */
 
 /*
  * Copyright (c) 1993 Martin Birgmeier
@@ -15,7 +15,7 @@
 
 #include <sys/cdefs.h>
 #if defined(LIBC_SCCS) && !defined(lint)
-__RCSID("$NetBSD: _rand48.c,v 1.8 2020/02/22 11:24:47 kamil Exp $");
+__RCSID("$NetBSD: _rand48.c,v 1.9 2020/02/22 14:07:57 kamil Exp $");
 #endif /* LIBC_SCCS and not lint */
 
 #include <assert.h>
@@ -50,7 +50,9 @@
        accu += (unsigned long) __rand48_mult[1] * (unsigned long) xseed[0];
        temp[1] = (unsigned short) accu;        /* middle 16 bits */
        accu >>= sizeof(unsigned short) * 8;
-       accu += __rand48_mult[0] * xseed[2] + __rand48_mult[1] * xseed[1] + __rand48_mult[2] * xseed[0];
+       accu += (unsigned int) __rand48_mult[0] * (unsigned int) xseed[2];
+       accu += (unsigned int) __rand48_mult[1] * (unsigned int) xseed[1];
+       accu += (unsigned int) __rand48_mult[2] * (unsigned int) xseed[0];
        xseed[0] = temp[0];
        xseed[1] = temp[1];
        xseed[2] = (unsigned short) accu;