[src/trunk]: src/bin/expr Improve the * operator handling in expr(1)

[kamil <kamil%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/c51c95e676a1
branches:  trunk
changeset: 320240:c51c95e676a1
user:      kamil <kamil%NetBSD.org@localhost>
date:      Wed Jun 27 17:12:49 2018 +0000

description:
Improve the * operator handling in expr(1)

Fixes overflow detection in expressions INT * -UINT.

Detected with libFuzzer & UBSan.

diffstat:

 bin/expr/expr.y |  24 +++++++++++++++++-------
 1 files changed, 17 insertions(+), 7 deletions(-)

diffs (51 lines):

diff -r 727f452acaf3 -r c51c95e676a1 bin/expr/expr.y
--- a/bin/expr/expr.y   Wed Jun 27 16:44:38 2018 +0000
+++ b/bin/expr/expr.y   Wed Jun 27 17:12:49 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: expr.y,v 1.43 2018/06/14 02:46:56 christos Exp $ */
+/* $NetBSD: expr.y,v 1.44 2018/06/27 17:12:49 kamil Exp $ */
 
 /*_
  * Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
 %{
 #include <sys/cdefs.h>
 #ifndef lint
-__RCSID("$NetBSD: expr.y,v 1.43 2018/06/14 02:46:56 christos Exp $");
+__RCSID("$NetBSD: expr.y,v 1.44 2018/06/27 17:12:49 kamil Exp $");
 #endif /* not lint */
 
 #include <sys/types.h>
@@ -351,17 +351,27 @@
                 * Check for over-& underflow.
                 */
 
-               /* Simplify the conditions */
+               /*
+                * Simplify the conditions:
+                *  - remove the case of both negative arguments
+                *    unless the operation will cause an overflow
+                */
                if (l < 0 && r < 0 && l != INT64_MIN && r != INT64_MIN) {
                        l = -l;
                        r = -r;
                }
 
+               /* - remove the case of legative l and positive r */
+               if (l < 0 && r >= 0) {
+                       /* Use res as a temporary variable */
+                       res = l;
+                       l = r;
+                       r = res;
+               }
+
                if ((l < 0 && r < 0) ||
-                   ((l != 0 && r != 0) &&
-                    (((l > 0 && r > 0) && (l > INT64_MAX / r)) ||
-                    ((((l < 0 && r > 0) || (l > 0 && r < 0)) &&
-                     (r != -1 && (l < INT64_MIN / r))))))) {
+                   (r > 0 && l > INT64_MAX / r) ||
+                   (r <= 0 && r < INT64_MIN / l)) {
                        yyerror("integer overflow or underflow occurred for "
                            "operation '%s %s %s'", left, op, right);
                        /* NOTREACHED */