pkgsrc-Bugs archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
pkg/33456: pkg_chk -u: perfomance speedup
>Number: 33456
>Category: pkg
>Synopsis: pkgtools/pkg_chk: pkg_chk -u: perfomance speedup
>Confidential: no
>Severity: non-critical
>Priority: low
>Responsible: pkg-manager
>State: open
>Class: change-request
>Submitter-Id: net
>Arrival-Date: Wed May 10 13:30:00 +0000 2006
>Originator: Auster
>Release: NetBSD 3.99.18
>Organization:
>Environment:
System: NetBSD lrou.x.ua 3.99.18 NetBSD 3.99.18 (lrou-1.747) #0: Thu Apr 27
14:13:15 EEST 2006 yx%lrou.x.ua@localhost:/usr/src/sys/arch/i386/compile/lrou
i386
Architecture: i386
Machine: i386
>Description:
pkg_chk -u: extract-make-vars (very slow) is not enough in pass2.
>How-To-Repeat:
my test for example:
installed packages - 561
mismatched packages - 2
(machine - Celeron-2.4G, 256M)
# /tmp/pkg_chk.test -u
[ PASS1: check 561 installed packages at 14:33:59 ]
vim-7.0.0: version mismatch - vim-6.4.8
vim-share-7.0.0: version mismatch - vim-share-6.4.8nb1
[ PASS1: done at 14:39:01 ]
[ duration: 302 seconds ]
14:39 /usr/bin/env /usr/sbin/pkg_delete -K /var/db/pkg -r vim-6.4.8
14:39 /usr/bin/env /usr/sbin/pkg_delete -K /var/db/pkg -r vim-share-6.4.8nb1
[ Rechecking packages after deletions ]
[ PASS2: recheck 561 installed packages at 14:39:02 ]
[ using cached vars ]
vim-7.0.0: missing
vim-share-7.0.0: missing
[ PASS2: done at 14:44:03 ]
[ duration: 301 seconds ]
^^^^^^^^^^^^^^^^^^^^^
[ restore previous versions, and test again ]
# /tmp/pkg_chk.test -I
[ PASS1: check 561 installed packages at 14:50:14 ]
vim-7.0.0: version mismatch - vim-6.4.8
vim-share-7.0.0: version mismatch - vim-share-6.4.8nb1
[ PASS1: done at 14:55:18 ]
[ duration: 304 seconds ]
14:55 /usr/bin/env /usr/sbin/pkg_delete -K /var/db/pkg -r vim-6.4.8
14:55 /usr/bin/env /usr/sbin/pkg_delete -K /var/db/pkg -r vim-share-6.4.8nb1
[ Rechecking packages after deletions ]
[ PASS2: recheck 561 installed packages at 14:55:19 ]
[ using cached vars ]
vim-share-7.0.0: missing
vim-share-7.0.0: missing
[ PASS2: done at 14:55:24 ]
[ duration: 5 seconds ]
^^^^^^^^^^^^^^^^^^^
>Fix:
eliminate extract-make-vars in pass2 (look diff).
[ good solution to further improve perfomance,
the idea of ports/pkgsrc index file from FreeBSD
($PKGSRCDIR/INDEX[.db] updated every few hours on (cvs)server side) ].
% diff -u /usr/pkg/sbin/pkg_chk /tmp/pkg_chk.test
--- /usr/pkg/sbin/pkg_chk
+++ /tmp/pkg_chk.test
@@ -139,10 +139,37 @@
msg "WARNING: No $pkgdir/Makefile - package moved or obsolete?"
return
fi
- cd $PKGSRCDIR/$PKGDIR
- extract_make_vars Makefile "$@"
- if [ -z "$PKGNAME" ]; then
- fatal "Unable to extract PKGNAME for $pkgdir"
+
+ if [ -n "$USE_CACHED_VARS" ]; then # using cached vars
+ pkg_vars=$(${GREP} "^$PKGDIR" "$INDEX_VARS" 2>/dev/null)
+ if [ -n "$pkg_vars" ]; then
+ args=$@
+ set --
+ for var in $args; do
+ eval $(echo $pkg_vars | ${SED} -En \
+ "s,.*\|($var=[^\|]*).*,\1,p")
+ if [ -z "$var" ]; then # hmm.. var not found
+ set -- $@ $var # return var in args
+ fi
+ done
+ fi
+ fi
+
+ if [ $# -gt 0 ]; then # very slow
+ cd $PKGSRCDIR/$PKGDIR
+ extract_make_vars Makefile "$@"
+ if [ -z "$PKGNAME" ]; then
+ fatal "Unable to extract PKGNAME for $pkgdir"
+ fi
+ if [ -n "$opt_I" ]; then # cache vars
+ vars=""
+ while [ $# -gt 0 ]; do
+ var="$1"
+ shift
+ vars="$vars|$var=$(eval echo \$$var)"
+ done
+ echo "${PKGDIR}$vars" >>"$INDEX_VARS"
+ fi
fi
fi
}
@@ -184,6 +211,12 @@
if [ -z "$PKGCHK_UPDATE_CONF" ];then
PKGCHK_UPDATE_CONF=$PKGSRCDIR/pkgchk_update-$(hostname).conf
fi
+
+ if [ -n "$opt_I" ] ; then
+ INDEX_VARS="$PKGSRCDIR/INDEX.vars"
+ rm -f $INDEX_VARS
+ USE_CACHED_VARS=
+ fi
}
fatal()
@@ -556,6 +589,7 @@
-g Generate an initial pkgchk.conf file
-h This help
-i Check versions of installed packages (not using pkgchk.conf)
+ -I Update all mismatched packages (pass2: using cached vars)
-k Continue with further packages if errors are encountered
-L file Redirect output from commands run into file (should be fullpath)
-l List binary packages including dependencies (implies -c)
@@ -587,7 +621,7 @@
fi
}
-args=$(getopt BC:D:L:P:U:abcfghiklNnqrsSuv $*)
+args=$(getopt BC:D:L:P:U:abcfghiIklNnqrsSuv $*)
if [ $? != 0 ]; then
opt_h=1
fi
@@ -604,6 +638,7 @@
-g ) opt_g=1 ;;
-h ) opt_h=1 ;;
-i ) opt_u=1 ; opt_q=1 ; echo "-i is deprecated - use -u -q" ;;
+ -I ) opt_u=1 ; opt_I=1 ;;
-k ) opt_k=1 ;;
-L ) opt_L="$2" ; shift ;;
-l ) opt_l=1 ;;
@@ -747,7 +782,18 @@
if [ -n "$opt_l" ] ; then
list_packages $PKGDIRLIST
else
+ ### TEMP: profiling
+ _ST_TIME=$(date +%s)
+ _INST_PKGS=$(echo $(${PKG_INFO} -a | wc -l))
+ msg_progress "PASS1: check ${_INST_PKGS} installed packages at $(date
+%H:%M:%S)"
+ ### TEMP: end
+
check_packages_installed $PKGDIRLIST
+
+ ### TEMP: profiling
+ msg_progress "PASS1: done at $(date +%H:%M:%S)"
+ msg_progress " duration: $(( $(date +%s) - ${_ST_TIME} )) seconds"
+ ### TEMP: end
fi
if [ -n "$MISMATCH_TODO" ]; then
@@ -780,7 +826,22 @@
PKGDIRLIST="$(pkgdirs_from_conf $PKGCHK_UPDATE_CONF $PKGDIRLIST)"
fi
if [ -n "$opt_a" -o -n "$opt_u" ]; then
+ if [ -n "$opt_I" -a -f "$INDEX_VARS" ]; then
+ USE_CACHED_VARS=yes
+ fi
+ ### TEMP: profiling
+ _ST_TIME=$(date +%s)
+ _INST_PKGS=$(echo $(wc -l <$PKGCHK_UPDATE_CONF))
+ msg_progress "PASS2: recheck ${_INST_PKGS} installed packages
at $(date +%H:%M:%S)"
+ msg_progress " using cached vars"
+ ### TEMP: end
+
check_packages_installed $PKGDIRLIST # May need to add more
+
+ ### TEMP: profiling
+ msg_progress "PASS2: done at $(date +%H:%M:%S)"
+ msg_progress " duration: $(( $(date +%s) - ${_ST_TIME} ))
seconds"
+ ### TEMP: end
fi
fi
fi
@@ -798,6 +859,9 @@
if [ -n "$opt_u" -a -z "$FAIL_DONE" -a -f $PKGCHK_UPDATE_CONF ] ; then
run_cmd "rm -f $PKGCHK_UPDATE_CONF"
fi
+if [ -n "$opt_I" -a -f $INDEX_VARS ] ; then
+ run_cmd "rm -f $INDEX_VARS"
+fi
[ -n "$MISS_DONE" ] && msg "Missing:$MISS_DONE"
[ -n "$INSTALL_DONE" ] && msg "Installed:$INSTALL_DONE"
Home |
Main Index |
Thread Index |
Old Index