RFC: checkflist extra prefixes and post-build hooks

[Christoph Badura <bad%bsd.de@localhost>]

I have the following patches in my local tree that I'd like to commit.

The first one adds a way to tell checkflist to ignore extra or missing
files in the destdir through the mk.conf variable
CHECKFLIST_EXTRAPREFIXES.

It also reports the files that were exempted at the end of the checkflist
run.

The second one runs the executables listed in the POST_BUILD_HOOKS mk.conf
variable after the do-build and do-obsolete targets.

I use that to customise the destdir (e.g. create missing device files,
create additional users, add ssh keys and other config files.

I use that to create images that I can boot in qemu and have a working
test environment that I can ssh into.

I think this is generic enough to be of general interest.

I'd like to commit this soonish.

--chris

3 files changed, 40 insertions(+), 4 deletions(-)
distrib/sets/Makefile    |  4 ++++
distrib/sets/checkflist  | 34 ++++++++++++++++++++++++++++++----
share/man/man5/mk.conf.5 |  6 ++++++

modified   distrib/sets/Makefile
@@ -280,6 +280,10 @@ installsets: .PHONY check_DESTDIR sanitise_METALOG
 
 # Should we ignore errors like extra or missing files in the flists?
 SLOPPY_FLIST?= NO
+.if !empty(CHECKFLIST_EXTRAPREFIXES)
+CFEPARGS!= echo "${CHECKFLIST_EXTRAPREFIXES}" | ${TOOL_SED} -Ee 's/(^|[ ])/\1-E /g'
+CHECKFLIST_FLAGS+=	${CFEPARGS}
+.endif
 .if !empty(SLOPPY_FLIST:M[Yy][Ee][Ss])
 CHECKFLIST_FLAGS+=	-e -m
 REGPKG.sloppy=		-m
modified   distrib/sets/checkflist
@@ -48,10 +48,11 @@ xargs=""
 dargs=""
 metalog=
 allowextra=false
+allowextraprefixes=""
 allowmissing=false
 
 # handle args
-while getopts xbL:M:em ch; do
+while getopts xbL:M:E:em ch; do
 	case ${ch} in
 	x)
 		xargs="-x"
@@ -67,6 +68,9 @@ while getopts xbL:M:em ch; do
 	M)
 		metalog="${OPTARG}"
 		;;
+	E)
+		allowextraprefixes="${allowextraprefixes} ${OPTARG}"
+		;;
 	e)
 		allowextra=true
 		;;
@@ -80,6 +84,7 @@ Usage: ${prog} [-x|-b|-L lists] [-M metalog] [-e] [-m]
 	-b		check netbsd + x11 lists
 	-L base,x,ext	check specified lists
 	-M metalog	metalog file
+	-E pathprefix	prefix for for files not to be considered extraneous (can be used multiple times)
 	-e		extra files are not considered an error
 	-m		missing files are not considered an error
 USAGE
@@ -236,20 +241,41 @@ check_destdir_missing
 check_destdir_extra()
 {
 if [ -s "${SETS_DESTDIR_EXTRA}" ]; then
-	count="$(${AWK} 'END {print NR}' "${SETS_DESTDIR_EXTRA}")"
+	# do not consider paths beginning with any of the alloexraprefixes
+	# to be extra files.
+	aepfxre='^$' # matches empty lines, of which we have none
+	if [ -n "${allowextraprefixes}" ]; then
+		# note that lines start with a dot '.'.
+		# replace every space but the first with '\|'
+		aepfxre="^\.\($(echo "${allowextraprefixes}" | sed -e 's/^ //;s/ /\\|/g')\)"
+	fi
+	count=$(${GREP} -cv -e "${aepfxre}" "${SETS_DESTDIR_EXTRA}")
+	if [ ${count} -gt 0 ]; then
 	echo ""
 	echo "=======  ${count} extra files in DESTDIR  ========="
 	echo "Files in DESTDIR but missing from flist."
 	echo "File is obsolete or flist is out of date ?"
 	if ${allowextra}; then
-		echo "This is non-fatal, due to '-e' option."
+		echo "This is non-fatal, due to the '-e' option."
 	else
 		es=1
 	fi
 	echo "------------------------------------------"
-	cat "${SETS_DESTDIR_EXTRA}"
+	${GREP} -v -e "${aepfxre}" "${SETS_DESTDIR_EXTRA}"
 	echo "=========  end of ${count} extra files  ==========="
 	echo ""
+	fi
+	# list allowed extra files
+	count=$(${GREP} -c -e "${aepfxre}" "${SETS_DESTDIR_EXTRA}")
+	if [ ${count} -eq 0 ]; then
+		return		# no allowed extra files
+	fi
+	echo "=======  ${count} allowed extra files in DESTDIR  ========="
+	echo "Allowed files in DESTDIR but missing from flist."
+	echo "------------------------------------------"
+	${GREP} -e "${aepfxre}" "${SETS_DESTDIR_EXTRA}"
+	echo "=========  end of ${count} allowed extra files  ==========="
+	echo ""
 fi
 }
 
modified   share/man/man5/mk.conf.5
@@ -203,6 +203,12 @@ options, or via a single option whose argument contains multiple
 space-separated paths.
 .DFLTu
 .
+.It Sy CHECKFLIST_EXTRAPREFIXES
+A space-separated list of absolute prefixes that are used to filter
+allowed extraneous files from the sets lists.  Files whose absolute path starts with one of the prefixes are not considered "extra files in DESTDIR".
+They will be reported as "allowed extra files", however.
+.DFLTu
+.
 .It Sy CONFIGOPTS
 Additional options to
 .Xr config 1

diff --git a/Makefile b/Makefile
index 5c9d95ca29b9..0b0dfaa18e99 100644
--- a/Makefile
+++ b/Makefile
@@ -252,6 +252,9 @@ BUILDTARGETS+=	do-x11
 BUILDTARGETS+=	do-build
 BUILDTARGETS+=	do-obsolete
 .endif
+.if defined(POST_BUILD_HOOKS)
+BUILDTARGETS+=	do-post-build-hooks
+.endif
 
 #
 # Enforce proper ordering of some rules.
@@ -495,6 +498,16 @@ do-build: .PHONY .MAKE
 	${MAKEDIRTARGET} . ${targ} BUILD_tools=no BUILD_lib=no
 .endfor
 
+do-post-build-hooks: .PHONY .MAKE
+	${_MKMSG} "post build hooks:" ${POST_BUILD_HOOKS:Q}
+	@for hook in ${POST_BUILD_HOOKS}; do \
+		echo "===> post build hook $$hook";  \
+		$$hook || { \
+			rc=$$?; \
+			echo $$hook failed with exit status $$rc 1>&2; \
+			exit $$rc; } \
+	done
+
 do-x11: .PHONY .MAKE
 .if ${MKX11} != "no"
 	${MAKEDIRTARGET} external/mit/xorg/tools all
diff --git a/share/man/man5/mk.conf.5 b/share/man/man5/mk.conf.5
index f82e691c6ecf..e69a36fa14d3 100644
--- a/share/man/man5/mk.conf.5
+++ b/share/man/man5/mk.conf.5
@@ -2478,6 +2478,19 @@ option
 .Fl j
 instead.
 .
+.It Sy POST_BUILD_HOOKS
+A space separated list of hooks to be executed as the last step of the
+.Xr make 1
+.Dq build
+target.
+Hooks must be executable files, either with absolute paths or found through
+.Sy PATH
+.
+A non-zero exit status from any hook will cause the
+.Sy build
+target to fail.
+.DFLTu
+.
 .It Sy SHAREDSTRINGS
 Obsolete.
 .