--=-eIHFWJ/QjpKj2DhuMQJ0
Content-Type: text/plain
Content-Transfer-Encoding: 7bit

Hello

I have been working on a way of taking advantage of multiple CPUs (both
SMP and networked computers) when building packages in pkgsrc. It has
resulted in a new build target, parallel, which will create a temporary
makefile on stdout describing dependencies between needed packages for
that given moment.

Simple example:

I have most of yplot's dependencies installed already, so the generated
makefile only contains two packages.

# cd /usr/pkgsrc/graphics/yplot
# make parallel
all: graphics_yplot
graphics_plplot: 
        xterm -e "cd /usr/pkgsrc/graphics/plplot;env MAKEFLAGS="" /usr/bin/make -X install"
graphics_yplot:  graphics_plplot
        xterm -e "cd /usr/pkgsrc/graphics/yplot;env MAKEFLAGS="" /usr/bin/make -X install"
#

This can be fed to a make process which will take care of job scheduling
and dependencies.

Example:

# make parallel | make -j 3 -k -f -


Note that the above example is not distributing any work to a distant
network, it is just building separate packages in individual xterms (and
thereby keeping the parent shell nice and tidy). This is of course not
what bulk builders want, but it is nice to look at.

Furthermore, the -k does not actually make any difference. This, because
xterm hides the return status of the make process and just returns 0 to
the top level make. Perhaps this should be fixed in xterm?


Known limitations:

* The 'parallel' target only works on individual packages. To actually
do a bulk build, one has to create a meta-package containing
dependencies towards all other packages.

* It can take some time to calculate the dependency graph for
complicated packages. On my Athlon XP 2100+, 'make parallel' in
meta-pkgs/kde3 (with a fair amount of dependencies already installed)
takes 3 minutes to complete. I believe the time can be halfed, but not
easily.


Best regards
	Lars Nordlund


--=-eIHFWJ/QjpKj2DhuMQJ0
Content-Disposition: attachment; filename=diff.pkg
Content-Type: text/x-patch; name=diff.pkg; charset=iso8859-1
Content-Transfer-Encoding: 7bit

Index: bsd.pkg.mk
===================================================================
RCS file: /cvsroot/pkgsrc/mk/bsd.pkg.mk,v
retrieving revision 1.1613
diff -u -r1.1613 bsd.pkg.mk
--- bsd.pkg.mk	16 Apr 2005 09:26:22 -0000	1.1613
+++ bsd.pkg.mk	18 Apr 2005 22:38:30 -0000
@@ -1498,8 +1498,28 @@
 	${ECHO} "$$dlist"
 .endif
 
+# show both build and run depends directories (non-recursively),
+# skipping already installed packages
+.PHONY: show-depends-dirs-needed
+.if !target(show-depends-dirs-needed)
+show-depends-dirs-needed:
+.  if !empty(DEPENDS)
+	${_PKG_SILENT}${_PKG_DEBUG}					\
+	dlist="";							\
+	for i in `${MAKE} show-depends-dirs`; do			\
+		cd ${PKGSRCDIR}/$$i;					\
+		wild=`${MAKE} show-var VARNAME=PKGWILDCARD`;		\
+		have=`${PKG_BEST_EXISTS} "$$wild" || ${TRUE}`;		\
+		if [ -z "$$have" ]; then dlist="$$dlist $$i"; fi;	\
+	done;								\
+	${ECHO} "$$dlist"
+.  endif
+.endif
+
 # Show all build and run depends, reverse-breadth first, with options.
-.if make(show-all-depends-dirs) || make(show-all-depends-dirs-excl) || make (show-root-dirs)
+.if make(show-all-depends-dirs) || make(show-all-depends-dirs-excl) ||	\
+  make(show-all-depends-dirs-needed) || make (show-all-depends-dirs-needed-excl) || \
+  make (parallel) || make (show-root-dirs)
 
 # "awk" macro to recurse over the dependencies efficiently, never running in
 # the same same directory twice. You may set the following options via "-v":
@@ -1508,9 +1528,13 @@
 #	RootsOnly = 1	to print only root directories (i.e. directories
 #			of packages with no dependencies), including possibly
 #			own directory
+#	OnlyNeeded = 1  to skip recursing into already installed branches
 #
 _RECURSE_DEPENDS_DIRS=							\
 	function append_dirs(dir) {					\
+		if (OnlyNeeded)						\
+			command = "cd ../../" dir " && ${MAKE} show-depends-dirs-needed"; \
+		else							\
 		command = "cd ../../" dir " && ${MAKE} show-depends-dirs"; \
 		command | getline tmp_dirs;				\
 		close(command);						\
@@ -1556,13 +1580,40 @@
 	@${AWK} -v NonSelf=1 '${_RECURSE_DEPENDS_DIRS}'
 .endif
 
+.PHONY: show-all-depends-dirs-needed
+.if make(show-all-depends-dirs-needed)
+show-all-depends-dirs-needed:
+	@${AWK} -v OnlyNeeded=1 '${_RECURSE_DEPENDS_DIRS}'
+.endif
+
+.PHONY: show-all-depends-dirs-needed-excl
+.if make(show-all-depends-dirs-needed-excl)
+show-all-depends-dirs-needed-excl:
+	@${AWK} -v OnlyNeeded=1 -v NonSelf=1 '${_RECURSE_DEPENDS_DIRS}'
+.endif
+
+.PHONY: parallel
+.if make(parallel)
+parallel:
+	@WD=`${PWD_CMD}`;						\
+	d=`${DIRNAME} $$WD`;						\
+	absdir=`${BASENAME} $$d`/`${BASENAME} $$WD`;			\
+	${ECHO} "all: `${ECHO} $${absdir} | ${SED} 's:/:_:'`"
+	@for j in `${MAKE} show-all-depends-dirs-needed`; do		\
+		target=`${ECHO} $${j} | ${SED} 's:/:_:'`;		\
+		deps=`cd ../../$${j}; ${MAKE} show-depends-dirs-needed | ${SED} 's:/:_:g'`; \
+		${ECHO} "$${target}: $${deps}"; \
+		${ECHO} "	xterm -e \"cd /usr/pkgsrc/$${j};env MAKEFLAGS=\"\" ${MAKE} -X install clean\""; \
+	done
+.endif
+
 .PHONY: show-root-dirs
 .if make(show-root-dirs)
 show-root-dirs:
 	${_PKG_SILENT}${_PKG_DEBUG}${AWK} -v RootsOnly=1 '${_RECURSE_DEPENDS_DIRS}'
 .endif
 
-.endif # make(show-{all-depends-dirs{,-excl},root-dirs})
+.endif # make(parallel,show-{all-depends-dirs{,-excl,-needed,-needed-excl},root-dirs})
 
 .PHONY: show-distfiles
 .if !target(show-distfiles)
@@ -2902,7 +2953,7 @@
 	         ${WRKDIR}/.files.diff ${WRKDIR}/.files.expected;	\
 	${TEST} $$errors -eq 0
 .endif
-	
+
 .PHONY: acquire-extract-lock acquire-patch-lock acquire-tools-lock
 .PHONY: acquire-wrapper-lock acquire-configure-lock acquire-build-lock
 .PHONY: acquire-install-lock acquire-package-lock

--=-eIHFWJ/QjpKj2DhuMQJ0--