Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/usr.bin/make/unit-tests make(1): add post-processing for tests
details: https://anonhg.NetBSD.org/src/rev/1aaf82ae4b70
branches: trunk
changeset: 973588:1aaf82ae4b70
user: rillig <rillig%NetBSD.org@localhost>
date: Sat Jul 04 22:01:10 2020 +0000
description:
make(1): add post-processing for tests
By defining SED_CMDS.${test}, a test can do additional post-processing
on its output, for example to normalize paths or error messages.
The Makefile for the tests now follows the common style to have all
relevant definitions at the top of the file, and all the implementation
details at the bottom.
By using the += instead of backslashed lists, it becomes easier to
comment out a single test, either temporarily or permanently (such as
the POSIX tests).
The :C modifier has been replaced with the simpler :S, some :S modifiers
have been replaced with simpler := modifiers.
The postprocessing commands have been extracted from the actual command.
This makes editing and commenting them easier.
diffstat:
usr.bin/make/unit-tests/Makefile | 157 ++++++++++++++++++----------------
usr.bin/make/unit-tests/varshell.exp | 2 +-
2 files changed, 84 insertions(+), 75 deletions(-)
diffs (242 lines):
diff -r 7026c95f4c9c -r 1aaf82ae4b70 usr.bin/make/unit-tests/Makefile
--- a/usr.bin/make/unit-tests/Makefile Sat Jul 04 21:07:29 2020 +0000
+++ b/usr.bin/make/unit-tests/Makefile Sat Jul 04 22:01:10 2020 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.60 2020/07/04 21:04:25 rillig Exp $
+# $NetBSD: Makefile,v 1.61 2020/07/04 22:01:10 rillig Exp $
#
# Unit tests for make(1)
#
@@ -11,83 +11,90 @@
# accept:
# move generated output to expected results
#
+# Settable variables
+#
+# TEST_MAKE
+# The make program to be tested.
+#
#
# Adding a test case
#
# Each feature should get its own set of tests in its own suitably
# named makefile (*.mk), with its own set of expected results (*.exp),
-# and it should be added to the TESTNAMES list.
+# and it should be added to the TESTS list.
#
# Any added files must also be added to src/distrib/sets/lists/tests/mi.
-# Makefiles that are not added to TESTNAMES must be ignored in
+# Makefiles that are not added to TESTS must be ignored in
# src/tests/usr.bin/make/t_make.sh (example: include-sub).
#
-.MAIN: all
-
-UNIT_TESTS:= ${.PARSEDIR}
-.PATH: ${UNIT_TESTS}
-
# Each test is in a sub-makefile.
# Keep the list sorted.
-TESTNAMES= \
- comment \
- cond-late \
- cond-short \
- cond1 \
- cond2 \
- dollar \
- doterror \
- dotwait \
- error \
- export \
- export-all \
- export-env \
- forloop \
- forsubst \
- hash \
- include-main \
- misc \
- moderrs \
- modmatch \
- modmisc \
- modorder \
- modts \
- modword \
- order \
- posix \
- qequals \
- sunshcmd \
- sysv \
- ternary \
- unexport \
- unexport-env \
- varcmd \
- varmisc \
- varmod-edge \
- varquote \
- varshell
+TESTS+= comment
+TESTS+= cond-late
+TESTS+= cond-short
+TESTS+= cond1
+TESTS+= cond2
+TESTS+= dollar
+TESTS+= doterror
+TESTS+= dotwait
+TESTS+= error
+TESTS+= # escape # broken by referting POSIX changes
+TESTS+= export
+TESTS+= export-all
+TESTS+= export-env
+TESTS+= forloop
+TESTS+= forsubst
+TESTS+= hash
+TESTS+= # impsrc # broken by referting POSIX changes
+TESTS+= include-main
+TESTS+= misc
+TESTS+= moderrs
+TESTS+= modmatch
+TESTS+= modmisc
+TESTS+= modorder
+TESTS+= modts
+TESTS+= modword
+TESTS+= order
+TESTS+= # phony-end # broken by referting POSIX changes
+TESTS+= posix
+TESTS+= # posix1 # broken by referting POSIX changes
+TESTS+= qequals
+TESTS+= # suffixes # broken by referting POSIX changes
+TESTS+= sunshcmd
+TESTS+= sysv
+TESTS+= ternary
+TESTS+= unexport
+TESTS+= unexport-env
+TESTS+= varcmd
+TESTS+= varmisc
+TESTS+= varmod-edge
+TESTS+= varquote
+TESTS+= varshell
-# these tests were broken by referting POSIX chanegs
-STRICT_POSIX_TESTS = \
- escape \
- impsrc \
- phony-end \
- posix1 \
- suffixes
+# Override make flags for certain tests; default is -k.
+FLAGS.doterror= # none
+FLAGS.order= -j1
+
+# Some tests need extra post-processing.
+SED_CMDS.varshell+= -e 's,^[a-z]*sh: ,,'
+SED_CMDS.varshell+= -e '/command/s,No such.*,not found,'
-# Override make flags for certain tests
-flags.doterror=
-flags.order=-j1
+# End of the configuration section.
+
+.MAIN: all
-OUTFILES= ${TESTNAMES:S/$/.out/}
+UNIT_TESTS:= ${.PARSEDIR}
+.PATH: ${UNIT_TESTS}
+
+OUTFILES= ${TESTS:=.out}
all: ${OUTFILES}
-CLEANFILES += *.rawout *.out *.status *.tmp *.core *.tmp
-CLEANFILES += obj*.[och] lib*.a # posix1.mk
-CLEANFILES += issue* .[ab]* # suffixes.mk
-CLEANRECURSIVE += dir dummy # posix1.mk
+CLEANFILES+= *.rawout *.out *.status *.tmp *.core *.tmp
+CLEANFILES+= obj*.[och] lib*.a # posix1.mk
+CLEANFILES+= issue* .[ab]* # suffixes.mk
+CLEANRECURSIVE+= dir dummy # posix1.mk
clean:
rm -f ${CLEANFILES}
@@ -95,34 +102,36 @@
rm -rf ${CLEANRECURSIVE}
.endif
-TEST_MAKE?= ${.MAKE}
-TOOL_SED?= sed
+TEST_MAKE?= ${.MAKE}
+TOOL_SED?= sed
# ensure consistent results from sort(1)
-LC_ALL= C
-LANG= C
+LC_ALL= C
+LANG= C
.export LANG LC_ALL
# the tests are actually done with sub-makes.
.SUFFIXES: .mk .rawout .out
.mk.rawout:
- @echo ${TEST_MAKE} ${flags.${.TARGET:R}:U-k} -f ${.IMPSRC}
+ @echo ${TEST_MAKE} ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC}
-@cd ${.OBJDIR} && \
- { ${TEST_MAKE} ${flags.${.TARGET:R}:U-k} -f ${.IMPSRC} \
+ { ${TEST_MAKE} ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC} \
2>&1 ; echo $$? >${.TARGET:R}.status ; } > ${.TARGET}.tmp
@mv ${.TARGET}.tmp ${.TARGET}
+_SED_CMDS+= -e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
+_SED_CMDS+= -e 's,${TEST_MAKE:S,.,\\.,g},make,'
+_SED_CMDS+= -e '/stopped/s, /.*, unit-tests,'
+_SED_CMDS+= -e 's,${.CURDIR:S,.,\\.,g}/,,g'
+_SED_CMDS+= -e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
+
# We always pretend .MAKE was called 'make'
# and strip ${.CURDIR}/ from the output
# and replace anything after 'stopped in' with unit-tests
# so the results can be compared.
.rawout.out:
@echo postprocess ${.TARGET}
- @${TOOL_SED} -e 's,^${TEST_MAKE:T:C/\./\\\./g}[][0-9]*:,make:,' \
- -e 's,${TEST_MAKE:C/\./\\\./g},make,' \
- -e '/stopped/s, /.*, unit-tests,' \
- -e 's,${.CURDIR:C/\./\\\./g}/,,g' \
- -e 's,${UNIT_TESTS:C/\./\\\./g}/,,g' \
+ @${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
< ${.IMPSRC} > ${.TARGET}.tmp
@echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp
@mv ${.TARGET}.tmp ${.TARGET}
@@ -130,7 +139,7 @@
# Compare all output files
test: ${OUTFILES} .PHONY
@failed= ; \
- for test in ${TESTNAMES}; do \
+ for test in ${TESTS}; do \
diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
|| failed="$${failed}$${failed:+ }$${test}" ; \
done ; \
@@ -141,14 +150,14 @@
fi
accept:
- @for test in ${TESTNAMES}; do \
+ @for test in ${TESTS}; do \
cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
|| { echo "Replacing $${test}.exp" ; \
cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
done
.if exists(${TEST_MAKE})
-${TESTNAMES:S/$/.rawout/}: ${TEST_MAKE}
+${TESTS:=.rawout}: ${TEST_MAKE}
.endif
.-include <bsd.obj.mk>
diff -r 7026c95f4c9c -r 1aaf82ae4b70 usr.bin/make/unit-tests/varshell.exp
--- a/usr.bin/make/unit-tests/varshell.exp Sat Jul 04 21:07:29 2020 +0000
+++ b/usr.bin/make/unit-tests/varshell.exp Sat Jul 04 22:01:10 2020 +0000
@@ -1,4 +1,4 @@
-sh: /bin/no/such/command: not found
+/bin/no/such/command: not found
make: "varshell.mk" line 5: warning: "/bin/no/such/command" returned non-zero status
make: "varshell.mk" line 6: warning: "kill -14 $$" exited on a signal
make: "varshell.mk" line 7: warning: "false" returned non-zero status
Home |
Main Index |
Thread Index |
Old Index