--=-ncOmaNLq+z/dykuTpder
Content-Type: text/plain
Content-Transfer-Encoding: 7bit

Hello

This is an updated patch for my new 'parallel' target in pkgsrc, making
it possible to have dependancies (in the form of other packages) built
in parallel. The update includes; removal of unneeded ${SED}s, changed
xterm -e into sh -c and also fix a bug in the show-depends-dirs-needed
target.


Some numbers from using this on my SMP box (dual iP3-700):

(clean out all existing packages to get a clean environment, then
install devel/cpuflags and security/sudo for a suitable build env.)

# cd /usr/pkgsrc/editors/emacs
# time make parallel | make -k -j 3 -f -
[...]
make parallel  19.51s user 16.78s system 159% cpu 22.746 total
make -k -j 3 -f -  844.20s user 404.90s system 152% cpu 13:41.18 total


The same thing without using make parallel:

# cd /usr/pkgsrc/editors/emacs
# time make install clean
[...]
make install clean  840.60s user 369.32s system 126% cpu 15:59.72 total


(not the best example since most of the time is spent in emacs itself,
but making more tests takes a while on this computer..)


The speedup on more complex packages (i.e. with more dependancies) is
more dramatic. A cheap trick to increase the number of dependancies of a
package is to create a meta-pkg containing a reference to all packages
you want installed and then run make parallel on that meta-pkg. It is
very straigt forward to create such a meta-pkg if you base your Makefile
on the output from /usr/pkgsrc/pkgchk.conf (generated by 'pkg_chk -g')
and then just do some emacs keyboard macro magic. Rebuild of a "complete
pkgsrc tree" (including the packages to user actually wants / had
installed when pkg_chk -g was run) can then be done at maximum warp
almost all the time.


Comments?


Best regards
	Lars Nordlund


--=-ncOmaNLq+z/dykuTpder
Content-Disposition: attachment; filename=parallel-pkgsrc.diff
Content-Type: text/x-patch; name=parallel-pkgsrc.diff; charset=iso8859-1
Content-Transfer-Encoding: 7bit

Index: bsd.pkg.mk
===================================================================
RCS file: /cvsroot/pkgsrc/mk/bsd.pkg.mk,v
retrieving revision 1.1616
diff -u -r1.1616 bsd.pkg.mk
--- bsd.pkg.mk	22 Apr 2005 02:20:22 -0000	1.1616
+++ bsd.pkg.mk	24 Apr 2005 11:53:00 -0000
@@ -1500,8 +1500,26 @@
 	${ECHO} "$$dlist"
 .endif
 
+# show both build and run depends directories (non-recursively),
+# skipping already installed packages
+.PHONY: show-depends-dirs-needed
+.if !target(show-depends-dirs-needed)
+show-depends-dirs-needed:
+	${_PKG_SILENT}${_PKG_DEBUG}					\
+	dlist="";							\
+	for i in `${MAKE} show-depends-dirs`; do			\
+		cd ${PKGSRCDIR}/$$i;					\
+		wild=`${MAKE} show-var VARNAME=PKGWILDCARD`;		\
+		have=`${PKG_BEST_EXISTS} "$$wild" || ${TRUE}`;		\
+		if [ -z "$$have" ]; then dlist="$$dlist $$i"; fi;	\
+	done;								\
+	${ECHO} "$$dlist"
+.endif
+
 # Show all build and run depends, reverse-breadth first, with options.
-.if make(show-all-depends-dirs) || make(show-all-depends-dirs-excl) || make (show-root-dirs)
+.if make(show-all-depends-dirs) || make(show-all-depends-dirs-excl) ||	\
+  make(show-all-depends-dirs-needed) || make (show-all-depends-dirs-needed-excl) || \
+  make (parallel) || make (show-root-dirs)
 
 # "awk" macro to recurse over the dependencies efficiently, never running in
 # the same same directory twice. You may set the following options via "-v":
@@ -1510,10 +1528,14 @@
 #	RootsOnly = 1	to print only root directories (i.e. directories
 #			of packages with no dependencies), including possibly
 #			own directory
+#	OnlyNeeded = 1  to skip recursing into already installed branches
 #
 _RECURSE_DEPENDS_DIRS=							\
 	function append_dirs(dir) {					\
-		command = "cd ../../" dir " && ${MAKE} show-depends-dirs"; \
+		if (OnlyNeeded)						\
+			command = "cd ../../" dir " && ${MAKE} show-depends-dirs-needed"; \
+		else							\
+			command = "cd ../../" dir " && ${MAKE} show-depends-dirs"; \
 		command | getline tmp_dirs;				\
 		close(command);						\
 		if (tmp_dirs ~ /^$$/)					\
@@ -1558,13 +1580,40 @@
 	@${AWK} -v NonSelf=1 '${_RECURSE_DEPENDS_DIRS}'
 .endif
 
+.PHONY: show-all-depends-dirs-needed
+.if make(show-all-depends-dirs-needed)
+show-all-depends-dirs-needed:
+	@${AWK} -v OnlyNeeded=1 '${_RECURSE_DEPENDS_DIRS}'
+.endif
+
+.PHONY: show-all-depends-dirs-needed-excl
+.if make(show-all-depends-dirs-needed-excl)
+show-all-depends-dirs-needed-excl:
+	@${AWK} -v OnlyNeeded=1 -v NonSelf=1 '${_RECURSE_DEPENDS_DIRS}'
+.endif
+
+.PHONY: parallel
+.if make(parallel)
+parallel:
+	@WD=`${PWD_CMD}`;						\
+	d=`${DIRNAME} $$WD`;						\
+	absdir=`${BASENAME} $$d`/`${BASENAME} $$WD`;			\
+	${ECHO} "all: $${absdir}"
+	@for j in `${MAKE} show-all-depends-dirs-needed`; do		\
+		target=$${j};						\
+		deps=`cd ../../$${j}; ${MAKE} show-depends-dirs-needed`;\
+		${ECHO} "$${target}: $${deps}"; \
+		${ECHO} "	sh -c \"cd /usr/pkgsrc/$${j};env MAKEFLAGS= ${MAKE} -X install clean\""; \
+	done
+.endif
+
 .PHONY: show-root-dirs
 .if make(show-root-dirs)
 show-root-dirs:
 	${_PKG_SILENT}${_PKG_DEBUG}${AWK} -v RootsOnly=1 '${_RECURSE_DEPENDS_DIRS}'
 .endif
 
-.endif # make(show-{all-depends-dirs{,-excl},root-dirs})
+.endif # make(parallel,show-{all-depends-dirs{,-excl,-needed,-needed-excl},root-dirs})
 
 .PHONY: show-distfiles
 .if !target(show-distfiles)
@@ -2904,7 +2953,7 @@
 	         ${WRKDIR}/.files.diff ${WRKDIR}/.files.expected;	\
 	${TEST} $$errors -eq 0
 .endif
-	
+
 .PHONY: acquire-extract-lock acquire-patch-lock acquire-tools-lock
 .PHONY: acquire-wrapper-lock acquire-configure-lock acquire-build-lock
 .PHONY: acquire-install-lock acquire-package-lock

--=-ncOmaNLq+z/dykuTpder--