--- Begin Message ---
Module Name: pkgsrc
Committed By: jperkin
Date: Tue Jun 7 10:04:25 UTC 2022
Modified Files:
pkgsrc/mk: bsd.pkg.use.mk bsd.prefs.mk
pkgsrc/mk/buildlink3: bsd.buildlink3.mk
pkgsrc/mk/defaults: mk.conf
Log Message:
mk: Add PKGSRC_USE_MKTOOLS support.
When enabled, pull in the pkgtools/mktools package to use C-based tools for
certain parts of the pkgsrc mk infrastructure. Default is off for now while
any portability issues are shaken out, but it has been tested successfully on
at least illumos, macOS, and NetBSD.
The first tool is mk-buildlink-symlinks which creates the buildlink3 symlinks
as part of the "wrapper" phase. This significantly improves performance,
especially with packages that have a lot of buildlink3 dependencies. For
example, the time for "bmake wrapper" in x11/kde-workspace4 goes from:
real 3:20.696394375
user 50.553556463
sys 2:23.883823896
to:
real 19.873100598
user 8.141441461
sys 11.740602820
It's expected that other tools will be added over time to speed up other parts
of the infrastructure.
To generate a diff of this commit:
cvs rdiff -u -r1.71 -r1.72 pkgsrc/mk/bsd.pkg.use.mk
cvs rdiff -u -r1.421 -r1.422 pkgsrc/mk/bsd.prefs.mk
cvs rdiff -u -r1.247 -r1.248 pkgsrc/mk/buildlink3/bsd.buildlink3.mk
cvs rdiff -u -r1.333 -r1.334 pkgsrc/mk/defaults/mk.conf
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: pkgsrc/mk/bsd.pkg.use.mk
diff -u pkgsrc/mk/bsd.pkg.use.mk:1.71 pkgsrc/mk/bsd.pkg.use.mk:1.72
--- pkgsrc/mk/bsd.pkg.use.mk:1.71 Mon May 9 15:35:31 2022
+++ pkgsrc/mk/bsd.pkg.use.mk Tue Jun 7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.pkg.use.mk,v 1.71 2022/05/09 15:35:31 nia Exp $
+# $NetBSD: bsd.pkg.use.mk,v 1.72 2022/06/07 10:04:25 jperkin Exp $
#
# Turn USE_* macros into proper depedency logic. Included near the top of
# bsd.pkg.mk, after bsd.prefs.mk.
@@ -109,3 +109,10 @@ TOOL_DEPENDS+= libtool-base>=${_OPSYS_L
CONFIGURE_ENV+= LIBTOOL="${LIBTOOL} ${LIBTOOL_FLAGS}"
MAKE_ENV+= LIBTOOL="${LIBTOOL} ${LIBTOOL_FLAGS}"
.endif
+
+# PKGSRC_USE_MKTOOLS. Cyclic dependencies prevent us from using mktools
+# when building cwrappers, so use the shell tools in that instance.
+.if ${_PKGSRC_USE_MKTOOLS} == "yes" && empty(PKGPATH:Mpkgtools/cwrappers)
+TOOL_DEPENDS+= mktools-[0-9]*:../../pkgtools/mktools
+PKG_MKSYMLINKS?= ${LOCALBASE}/libexec/mktools/mk-buildlink-symlinks
+.endif
Index: pkgsrc/mk/bsd.prefs.mk
diff -u pkgsrc/mk/bsd.prefs.mk:1.421 pkgsrc/mk/bsd.prefs.mk:1.422
--- pkgsrc/mk/bsd.prefs.mk:1.421 Sun May 1 08:03:40 2022
+++ pkgsrc/mk/bsd.prefs.mk Tue Jun 7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.prefs.mk,v 1.421 2022/05/01 08:03:40 nia Exp $
+# $NetBSD: bsd.prefs.mk,v 1.422 2022/06/07 10:04:25 jperkin Exp $
#
# This file includes the mk.conf file, which contains the user settings.
#
@@ -874,6 +874,14 @@ _USE_CWRAPPERS= yes
_USE_CWRAPPERS= no
.endif
+# Use C-based tools to speed up pkgsrc infrastructure tasks.
+.if empty(PKGPATH:Mpkgtools/mktools) && \
+ ${PKGSRC_USE_MKTOOLS:tl} == "yes"
+_PKGSRC_USE_MKTOOLS= yes
+.else
+_PKGSRC_USE_MKTOOLS= no
+.endif
+
# Wrapper framework definitions
.include "wrapper/wrapper-defs.mk"
Index: pkgsrc/mk/buildlink3/bsd.buildlink3.mk
diff -u pkgsrc/mk/buildlink3/bsd.buildlink3.mk:1.247 pkgsrc/mk/buildlink3/bsd.buildlink3.mk:1.248
--- pkgsrc/mk/buildlink3/bsd.buildlink3.mk:1.247 Tue Jun 8 10:10:02 2021
+++ pkgsrc/mk/buildlink3/bsd.buildlink3.mk Tue Jun 7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.buildlink3.mk,v 1.247 2021/06/08 10:10:02 bouyer Exp $
+# $NetBSD: bsd.buildlink3.mk,v 1.248 2022/06/07 10:04:25 jperkin Exp $
#
# Copyright (c) 2004 The NetBSD Foundation, Inc.
# All rights reserved.
@@ -660,7 +660,7 @@ _BLNK_FILES_CMD.${_pkg_}+= )
_BLNK_FILES_CMD.${_pkg_}+= | ${SORT} -u
${_BLNK_COOKIE.${_pkg_}}:
- ${RUN} \
+ ${RUN} \
case "${BUILDLINK_PREFIX.${_pkg_}}" in \
*not_found) \
${ERROR_MSG} "${BUILDLINK_API_DEPENDS.${_pkg_}} is not installed; can't buildlink files."; \
@@ -671,6 +671,47 @@ ${_BLNK_COOKIE.${_pkg_}}:
${ERROR_MSG} "[bsd.buildlink3.mk] X11BASE is not set correctly."; \
exit 1; \
}
+.if ${_PKGSRC_USE_MKTOOLS} == "yes"
+ ${RUN} \
+ case "${BUILDLINK_PREFIX.${_pkg_}}" in \
+ ${LOCALBASE}) buildlink_dir="${BUILDLINK_DIR}" ;; \
+ ${X11BASE}) buildlink_dir="${BUILDLINK_X11_DIR}" ;; \
+ *) buildlink_dir="${BUILDLINK_DIR}" ;; \
+ esac; \
+ [ -z "${BUILDLINK_PREFIX.${_pkg_}:Q}" ] || \
+ cd ${BUILDLINK_PREFIX.${_pkg_}} && \
+ ${_BLNK_FILES_CMD.${_pkg_}} | \
+ while read file; do \
+ src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
+ [ -f "$$src" ] || continue; \
+ dest="$$buildlink_dir/$$file"; \
+ if [ -n "${BUILDLINK_FNAME_TRANSFORM.${_pkg_}:Q}" ]; then \
+ dest=`${ECHO} $$dest | ${SED} ${BUILDLINK_FNAME_TRANSFORM.${_pkg_}}`; \
+ fi; \
+ case "$$src" in \
+ *.la) \
+ dir="$${dest%/*}"; \
+ if [ ! -d "$$dir" ]; then \
+ ${MKDIR} "$$dir"; \
+ fi; \
+ ${_BLNK_LT_ARCHIVE_FILTER.${_pkg_}} \
+ "$$src" > "$$dest"; \
+ ;; \
+ *) \
+ ${ECHO} "$$dest -> $$src" >>${.TARGET}; \
+ ;; \
+ esac; \
+ done; \
+ if [ -f ${.TARGET} ]; then \
+ ${PKG_MKSYMLINKS} < ${.TARGET} || { \
+ ${ECHO} "Removing ${.TARGET}"; \
+ ${RM} -f ${.TARGET}; \
+ exit 1; \
+ } \
+ else \
+ ${TOUCH} ${TOUCH_FLAGS} ${.TARGET}; \
+ fi
+.else
${RUN} \
case "${BUILDLINK_PREFIX.${_pkg_}}" in \
${LOCALBASE}) buildlink_dir="${BUILDLINK_DIR}" ;; \
@@ -681,8 +722,8 @@ ${_BLNK_COOKIE.${_pkg_}}:
cd ${BUILDLINK_PREFIX.${_pkg_}} && \
${_BLNK_FILES_CMD.${_pkg_}} | \
while read file; do \
- src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
- if [ ! -f "$$src" ]; then \
+ src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
+ if [ ! -f "$$src" ]; then \
msg="$$src: not found"; \
else \
if [ -z "${BUILDLINK_FNAME_TRANSFORM.${_pkg_}:Q}" ]; then \
@@ -713,6 +754,7 @@ ${_BLNK_COOKIE.${_pkg_}}:
fi; \
${ECHO} "$$msg" >> ${.TARGET}; \
done
+.endif
# _BLNK_LT_ARCHIVE_FILTER.${_pkg_} is a command-line filter used in
# the previous target for transforming libtool archives (*.la) to
Index: pkgsrc/mk/defaults/mk.conf
diff -u pkgsrc/mk/defaults/mk.conf:1.333 pkgsrc/mk/defaults/mk.conf:1.334
--- pkgsrc/mk/defaults/mk.conf:1.333 Tue Jun 7 09:35:15 2022
+++ pkgsrc/mk/defaults/mk.conf Tue Jun 7 10:04:25 2022
@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.333 2022/06/07 09:35:15 wiz Exp $
+# $NetBSD: mk.conf,v 1.334 2022/06/07 10:04:25 jperkin Exp $
#
# This file provides default values for variables that may be overridden
@@ -22,6 +22,15 @@ USE_CWRAPPERS?= auto
# Possible: yes, no, auto
# Default: auto (automatic detection, see above)
+PKGSRC_USE_MKTOOLS?= no
+# Use C-based tools for various parts of the infrastructure instead of the
+# slower shell-based constructs. Significantly improves performance, but
+# not enabled by default until portability across all platforms has been
+# confirmed.
+#
+# Possible: yes, no
+# Default: no
+
USE_PKG_ADMIN_DIGEST?= no
# Record file digests instead of extracing the NetBSD RCS ID from files.
#
--- End Message ---