[src/trunk]: src/usr.bin/make/unit-tests tests/make: document the history of ...

[rillig <rillig%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/5cbd3819995c
branches:  trunk
changeset: 361148:5cbd3819995c
user:      rillig <rillig%NetBSD.org@localhost>
date:      Sat Feb 12 13:17:57 2022 +0000

description:
tests/make: document the history of bugs in '-k' mode

Reported in PR#49720 in 2015, fixed independently in compat.c 1.199 from
2020-12-07.

diffstat:

 distrib/sets/lists/tests/mi                         |   4 +-
 usr.bin/make/unit-tests/Makefile                    |   3 +-
 usr.bin/make/unit-tests/opt-keep-going-indirect.exp |  32 +++++++
 usr.bin/make/unit-tests/opt-keep-going-indirect.mk  |  88 +++++++++++++++++++++
 4 files changed, 125 insertions(+), 2 deletions(-)

diffs (163 lines):

diff -r 864f80ba8a3b -r 5cbd3819995c distrib/sets/lists/tests/mi
--- a/distrib/sets/lists/tests/mi       Sat Feb 12 11:14:48 2022 +0000
+++ b/distrib/sets/lists/tests/mi       Sat Feb 12 13:17:57 2022 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.1187 2022/02/12 01:15:17 rillig Exp $
+# $NetBSD: mi,v 1.1188 2022/02/12 13:17:57 rillig Exp $
 #
 # Note: don't delete entries from here - mark them as "obsolete" instead.
 #
@@ -5803,6 +5803,8 @@
 ./usr/tests/usr.bin/make/unit-tests/opt-jobs-no-action.mk                      tests-usr.bin-tests     compattestfile,atf
 ./usr/tests/usr.bin/make/unit-tests/opt-jobs.exp                               tests-usr.bin-tests     compattestfile,atf
 ./usr/tests/usr.bin/make/unit-tests/opt-jobs.mk                                        tests-usr.bin-tests     compattestfile,atf
+./usr/tests/usr.bin/make/unit-tests/opt-keep-going-indirect.exp                        tests-usr.bin-tests     compattestfile,atf
+./usr/tests/usr.bin/make/unit-tests/opt-keep-going-indirect.mk                 tests-usr.bin-tests     compattestfile,atf
 ./usr/tests/usr.bin/make/unit-tests/opt-keep-going-multiple.exp                        tests-usr.bin-tests     compattestfile,atf
 ./usr/tests/usr.bin/make/unit-tests/opt-keep-going-multiple.mk                 tests-usr.bin-tests     compattestfile,atf
 ./usr/tests/usr.bin/make/unit-tests/opt-keep-going.exp                         tests-usr.bin-tests     compattestfile,atf
diff -r 864f80ba8a3b -r 5cbd3819995c usr.bin/make/unit-tests/Makefile
--- a/usr.bin/make/unit-tests/Makefile  Sat Feb 12 11:14:48 2022 +0000
+++ b/usr.bin/make/unit-tests/Makefile  Sat Feb 12 13:17:57 2022 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.308 2022/02/12 01:15:18 rillig Exp $
+# $NetBSD: Makefile,v 1.309 2022/02/12 13:17:57 rillig Exp $
 #
 # Unit tests for make(1)
 #
@@ -260,6 +260,7 @@
 TESTS+=                opt-jobs-internal
 TESTS+=                opt-jobs-no-action
 TESTS+=                opt-keep-going
+TESTS+=                opt-keep-going-indirect
 TESTS+=                opt-keep-going-multiple
 TESTS+=                opt-m-include-dir
 TESTS+=                opt-no-action
diff -r 864f80ba8a3b -r 5cbd3819995c usr.bin/make/unit-tests/opt-keep-going-indirect.exp
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.bin/make/unit-tests/opt-keep-going-indirect.exp       Sat Feb 12 13:17:57 2022 +0000
@@ -0,0 +1,32 @@
+direct compat
+false
+*** Error code 1 (continuing)
+
+Stop.
+make: stopped in unit-tests
+exited 1
+
+direct jobs
+false
+*** [direct] Error code 1
+
+make: stopped in unit-tests
+exited 1
+
+indirect compat
+false
+*** Error code 1 (continuing)
+`indirect' not remade because of errors.
+
+Stop.
+make: stopped in unit-tests
+exited 1
+
+indirect jobs
+false
+*** [direct] Error code 1
+
+make: stopped in unit-tests
+exited 1
+
+exit status 0
diff -r 864f80ba8a3b -r 5cbd3819995c usr.bin/make/unit-tests/opt-keep-going-indirect.mk
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.bin/make/unit-tests/opt-keep-going-indirect.mk        Sat Feb 12 13:17:57 2022 +0000
@@ -0,0 +1,88 @@
+# $NetBSD: opt-keep-going-indirect.mk,v 1.1 2022/02/12 13:17:57 rillig Exp $
+#
+# Tests for the -k command line option, which stops building a target as soon
+# as an error is detected, but continues building the other, independent
+# targets, as far as possible.
+#
+# History:
+#      In 1993, the exit status for the option '-k' was always 0, even if a
+#      a direct or an indirect target failed.
+#
+#      Since 2000.12.30.02.05.21, the word '(continuing)' is missing in jobs
+#      mode, both for direct as well as indirect targets.
+#
+#      Since 2001.10.16.18.50.12, the exit status for a direct failure in
+#      compat mode is 1, while jobs mode and indirect failures still return
+#      exit status 0.  The number of empty lines between the various error
+#      messages differs between the modes, for no reason.
+#
+#      At 2006.11.17.22.07.39, the exit status for direct failures in both
+#      modes and for indirect failures in jobs mode was corrected, leaving
+#      only indirect failures in compat mode wrong.  On the downside, a
+#      failed indirect target in jobs mode was no longer listed as "not
+#      remade because of errors".
+#
+#      At 2016.08.26.23.28.39, the additional empty lines for a direct
+#      failure in compat mode was removed, making it consistent with a direct
+#      failure in jobs mode.  This left only one inconsistency, in that
+#      indirect failures in jobs mode (by far the most common when building
+#      large projects) did not produce any empty line.
+#
+#      Since 2020.12.07.00.53.30, the exit status is consistently 1 for
+#      failures in all 4 modes.
+#
+# Bugs:
+#      The output in case of a failure needlessly differs between compat and
+#      jobs mode.  As of 2022-02-12, compat mode outputs '(continuing)' while
+#      jobs mode doesn't.  In compat mode, the output does not mention which
+#      target failed.
+#
+# See also:
+#      https://gnats.netbsd.org/49720
+
+.PHONY: all direct indirect
+
+# The 'set +e' was necessary in 2003, when the shell was run with '-e' by
+# default.
+# The 'env -i' prevents that the environment variable MAKEFLAGS is passed down
+# to the child processes.
+all:
+       @echo 'direct compat'
+       @set +e; env -i ${MAKE} -r -f ${MAKEFILE} -k direct; echo "exited $$?"
+       @echo
+
+       @echo 'direct jobs'
+       @set +e; env -i ${MAKE} -r -f ${MAKEFILE} -k direct -j1; echo "exited $$?"
+       @echo
+
+       @echo 'indirect compat'
+       @set +e; env -i ${MAKE} -r -f ${MAKEFILE} -k indirect; echo "exited $$?"
+       @echo
+
+       @echo 'indirect jobs'
+       @set +e; env -i ${MAKE} -r -f ${MAKEFILE} -k indirect -j1; echo "exited $$?"
+       @echo
+
+indirect: direct
+direct:
+       false
+
+# TODO: Mention the target that failed, maybe even the chain of targets.
+# expect: direct compat
+# expect: *** Error code 1 (continuing)
+# expect: exited 1
+
+# TODO: Add '(continuing)'.
+# expect: direct jobs
+# expect: *** [direct] Error code 1
+# expect: exited 1
+
+# TODO: Mention the target that failed, maybe even the chain of targets.
+# expect: indirect compat
+# expect: *** Error code 1 (continuing)
+# expect: exited 1
+
+# TODO: Add '(continuing)'.
+# expect: indirect jobs
+# expect: *** [direct] Error code 1
+# expect: exited 1