CVS commit: pkgsrc/mk/fetch

["Jonathan Perkin" <jperkin%netbsd.org@localhost>]

Module Name:    pkgsrc
Committed By:   jperkin
Date:           Fri Oct 11 11:52:23 UTC 2024

Modified Files:
        pkgsrc/mk/fetch: fetch.mk

Log Message:
mk: Add mktool support to fetch.

mktool is an optional external tool, available from pkgtools/mktool or
"cargo install mktool", that provides significant performance improvements
and an improved progress bar compared to other fetch commands.

After installing, set both:

  FETCH_USING=                  mktool
  TOOLS_PLATFORM.mktool=        /path/to/mktool

to enable it.  This is so that an alternate fetch command can still be used
if required, while still using mktool elsewhere in the infrastructure, as
not all network access methods (e.g. proxies) have been fully tested.

When testing the performance of "bmake fetch" inside wip/grafana on an M1
MacBook Pro running macOS, fetching distfiles from http://localhost/ backed
by www/caddy, FETCH_USING=curl took 47m 58s while FETCH_USING=mktool took
just 4 seconds, well over 500x faster!

This can be seen in realtime along with an example of the streamlined
progress bar at <https://asciinema.org/a/S4MWXHLSJmL4GKYAhOBIIHE31>.


To generate a diff of this commit:
cvs rdiff -u -r1.76 -r1.77 pkgsrc/mk/fetch/fetch.mk

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: pkgsrc/mk/fetch/fetch.mk
diff -u pkgsrc/mk/fetch/fetch.mk:1.76 pkgsrc/mk/fetch/fetch.mk:1.77
--- pkgsrc/mk/fetch/fetch.mk:1.76       Wed Feb 21 10:53:28 2024
+++ pkgsrc/mk/fetch/fetch.mk    Fri Oct 11 11:52:23 2024
@@ -1,4 +1,4 @@
-# $NetBSD: fetch.mk,v 1.76 2024/02/21 10:53:28 jperkin Exp $
+# $NetBSD: fetch.mk,v 1.77 2024/10/11 11:52:23 jperkin Exp $
 
 .if empty(INTERACTIVE_STAGE:Mfetch) && empty(FETCH_MESSAGE:U)
 _MASTER_SITE_BACKUP=   ${MASTER_SITE_BACKUP:=${DIST_SUBDIR}${DIST_SUBDIR:D/}}
@@ -94,7 +94,15 @@ fetch: ${_FETCH_TARGETS}
 .if !target(do-fetch)
 .  if !empty(_ALLFILES)
 do-fetch: ${_ALLFILES:S/^/${DISTDIR}\//}
+.    if ${FETCH_USING} == "mktool" && !empty(TOOLS_PLATFORM.mktool)
+       @{ ${_ALLFILES:@file@                                           \
+               unsorted_sites="${SITES.${file:T}}";                    \
+               sites="${_ORDERED_SITES} ${_MASTER_SITE_BACKUP}";       \
+               echo ${file} ${DISTDIR} $$sites;                        \
+       @} } | ${TOOLS_PLATFORM.mktool} fetch -I - ${_MKTOOL_FETCH_ARGS}
+.    else
        @${DO_NADA}
+.    endif
 .  else
 do-fetch:
        @${DO_NADA}
@@ -120,7 +128,11 @@ ${DISTDIR}/${_file_}:
 .    if empty(IGNORE_INTERACTIVE_FETCH:Uno:M[yY][eE][sS])
 ${DISTDIR}/${_file_}: fetch-check-interactive
 .    endif
+.    if ${FETCH_USING} == "mktool" && !empty(TOOLS_PLATFORM.mktool)
+${DISTDIR}/${_file_}: error-check
+.    else
 ${DISTDIR}/${_file_}: do-fetch-file error-check
+.    endif
 .  endif
 .endfor
 
@@ -299,10 +311,13 @@ _FETCH_CMD=       ${PKGSRC_SETENV} CHECKSUM=${
                WC=${TOOLS_WC:Q}                                \
                ${SH} ${PKGSRCDIR}/mk/fetch/fetch
 
+_MKTOOL_FETCH_ARGS=    -d ${DISTDIR}
+
 _FETCH_ARGS+=  ${PKG_VERBOSE:D-v}
 .if exists(${DISTINFO_FILE}) && !make(distinfo) && !make(makesum) \
     && !make(makedistinfo) && !make(mdi)
 _FETCH_ARGS+=  ${FAILOVER_FETCH:D-c} -f ${DISTINFO_FILE:tA:Q}
+_MKTOOL_FETCH_ARGS+=   -f ${DISTINFO_FILE:tA:Q}
 .endif
 .if !empty(PKG_RESUME_TRANSFERS:M[yY][eE][sS])
 _FETCH_ARGS+=  -r