[pkgsrc/trunk]: pkgsrc/mk Add initial support for building packages reproducibly

[khorben <khorben%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/96aef44659b0
branches:  trunk
changeset: 371435:96aef44659b0
user:      khorben <khorben%pkgsrc.org@localhost>
date:      Sun Nov 12 13:34:14 2017 +0000

description:
Add initial support for building packages reproducibly

It currently tackles two problems:
- gcc(1) hard-coding full paths in debugging information (with one
  caveat at the moment)
- ar(1) hard-coding user IDs in archive headers

This allows packages built from the same tree and options to produce
identical results bit by bit. This option should be combined with ASLR
and PKGSRC_MKPIE to avoid predictable address offsets for attackers
attempting to exploit security vulnerabilities.

This is still disabled by default, and only supports NetBSD so far.

As discussed on tech-pkg@

diffstat:

 mk/bsd.pkg.mk         |   6 +++++-
 mk/bsd.prefs.mk       |   8 +++++++-
 mk/compiler/gcc.mk    |  13 ++++++++++++-
 mk/defaults/mk.conf   |  13 ++++++++++++-
 mk/platform/NetBSD.mk |   5 ++++-
 mk/repro/ar           |  11 +++++++++++
 mk/repro/repro.mk     |  15 +++++++++++++++
 7 files changed, 66 insertions(+), 5 deletions(-)

diffs (149 lines):

diff -r e449ecb8fd09 -r 96aef44659b0 mk/bsd.pkg.mk
--- a/mk/bsd.pkg.mk     Sun Nov 12 10:53:18 2017 +0000
+++ b/mk/bsd.pkg.mk     Sun Nov 12 13:34:14 2017 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: bsd.pkg.mk,v 1.2027 2017/09/16 09:34:25 wiz Exp $
+#      $NetBSD: bsd.pkg.mk,v 1.2028 2017/11/12 13:34:14 khorben Exp $
 #
 # This file is in the public domain.
 #
@@ -315,6 +315,10 @@
 .endif
 .endif
 
+# Handle Reproducible Builds
+#
+.include "repro/repro.mk"
+
 # Define SMART_MESSAGES in /etc/mk.conf for messages giving the tree
 # of dependencies for building, and the current target.
 _PKGSRC_IN?=           ===${SMART_MESSAGES:D> ${.TARGET} [${PKGNAME}${_PKGSRC_DEPS}] ===}
diff -r e449ecb8fd09 -r 96aef44659b0 mk/bsd.prefs.mk
--- a/mk/bsd.prefs.mk   Sun Nov 12 10:53:18 2017 +0000
+++ b/mk/bsd.prefs.mk   Sun Nov 12 13:34:14 2017 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.prefs.mk,v 1.394 2017/11/03 18:07:40 bsiegert Exp $
+# $NetBSD: bsd.prefs.mk,v 1.395 2017/11/12 13:34:14 khorben Exp $
 #
 # This file includes the mk.conf file, which contains the user settings.
 #
@@ -705,6 +705,12 @@
 _PKGSRC_MKPIE= yes
 .endif
 
+_PKGSRC_MKREPRO=       no
+.if (${PKGSRC_MKREPRO:tl} == "yes") && \
+    (${_OPSYS_SUPPORTS_MKREPRO:Uno} == "yes")
+_PKGSRC_MKREPRO=       yes
+.endif
+
 _PKGSRC_USE_FORTIFY=   no
 .if (${PKGSRC_USE_FORTIFY:tl} != "no") && \
     (${_OPSYS_SUPPORTS_FORTIFY:Uno} == "yes")
diff -r e449ecb8fd09 -r 96aef44659b0 mk/compiler/gcc.mk
--- a/mk/compiler/gcc.mk        Sun Nov 12 10:53:18 2017 +0000
+++ b/mk/compiler/gcc.mk        Sun Nov 12 13:34:14 2017 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: gcc.mk,v 1.186 2017/11/07 16:57:58 khorben Exp $
+# $NetBSD: gcc.mk,v 1.187 2017/11/12 13:34:14 khorben Exp $
 #
 # This is the compiler definition for the GNU Compiler Collection.
 #
@@ -347,6 +347,17 @@
 # CWRAPPERS_APPEND.ld+=        ${_MKPIE_LDFLAGS.gcc}
 .endif
 
+.if ${_PKGSRC_MKREPRO} == "yes"
+.export WRKDIR
+# XXX the dollar sign should not be expanded by the shell
+_GCC_CFLAGS+=          -fdebug-prefix-map=$$$$WRKDIR/=
+.endif
+
+.if ${_PKGSRC_MKREPRO} == "yes"
+_GCC_CFLAGS+=          ${_MKREPRO_CFLAGS.gcc}
+CWRAPPERS_APPEND.cc+=  ${_MKREPRO_CFLAGS.gcc}
+.endif
+
 # The user can choose the level of FORTIFY.
 .if ${PKGSRC_USE_FORTIFY} == "weak"
 _FORTIFY_CFLAGS=       -D_FORTIFY_SOURCE=1
diff -r e449ecb8fd09 -r 96aef44659b0 mk/defaults/mk.conf
--- a/mk/defaults/mk.conf       Sun Nov 12 10:53:18 2017 +0000
+++ b/mk/defaults/mk.conf       Sun Nov 12 13:34:14 2017 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.285 2017/10/28 15:56:48 schmonz Exp $
+# $NetBSD: mk.conf,v 1.286 2017/11/12 13:34:14 khorben Exp $
 #
 
 # This file provides default values for variables that may be overridden
@@ -231,6 +231,17 @@
 # Possible: yes, no
 # Default: no
 
+PKGSRC_MKREPRO?= no
+# If no, do not alter the build process. Otherwise, try to build reproducibly.
+# This allows packages built from the same tree and options to produce identical
+# results bit by bit.
+# This option should be combined with ASLR and PKGSRC_MKPIE to avoid predictable
+# address offsets for attackers attempting to exploit security vulnerabilities.
+# Possible: yes, no
+# Default: no
+#
+# Keywords: reproducible
+
 PKGSRC_USE_FORTIFY?= strong
 # Turns on substitute wrappers for commonly used functions that do not bounds
 # checking regularly, but could in some cases. This is effectively in use only
diff -r e449ecb8fd09 -r 96aef44659b0 mk/platform/NetBSD.mk
--- a/mk/platform/NetBSD.mk     Sun Nov 12 10:53:18 2017 +0000
+++ b/mk/platform/NetBSD.mk     Sun Nov 12 13:34:14 2017 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: NetBSD.mk,v 1.56 2017/10/03 13:18:00 jperkin Exp $
+# $NetBSD: NetBSD.mk,v 1.57 2017/11/12 13:34:14 khorben Exp $
 #
 # Variable definitions for the NetBSD operating system.
 
@@ -142,6 +142,9 @@
 _OPSYS_SUPPORTS_RELRO= yes
 .endif
 
+# Register support for REPRO (with GCC)
+_OPSYS_SUPPORTS_MKREPRO=       yes
+
 # Register support for SSP on most architectures (with GCC)
 .if (${MACHINE_ARCH} != "alpha") && \
     (${MACHINE_ARCH} != "hppa") && \
diff -r e449ecb8fd09 -r 96aef44659b0 mk/repro/ar
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/mk/repro/ar       Sun Nov 12 13:34:14 2017 +0000
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ $# -ge 2 ]; then
+       args="$1"
+       mod="$2"
+       shift 2
+
+       exec /usr/bin/ar "$mod$args" "$@"
+else
+       exec /usr/bin/ar "$@"
+fi
diff -r e449ecb8fd09 -r 96aef44659b0 mk/repro/repro.mk
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/mk/repro/repro.mk Sun Nov 12 13:34:14 2017 +0000
@@ -0,0 +1,15 @@
+# $NetBSD: repro.mk,v 1.1 2017/11/12 13:34:14 khorben Exp $
+#
+# Infrastructure support for PKGSRC_MKREPRO.
+#
+# Keywords: reproducible
+#
+
+.if ${_PKGSRC_MKREPRO} == "yes"
+
+# force ar(1) to be deterministic
+TOOLS_CREATE+= ar
+TOOLS_PATH.ar?=        ${PKGSRCDIR}/mk/repro/ar
+TOOLS_ARGS.ar?=        D
+
+.endif