dlopen interception on libasan

[christos%zoulas.com@localhost (Christos Zoulas)]

Well, the problem in PR/51670 is that with -fsanitizer=address the
dlopen interception is broken. libasan.so provides its own dlopen()
and then tries to find the real dlopen using dlsym(RTLD_NEXT,
"dlopen"). That does not work for NetBSD because dlopen in libc is
just a stub (that perhaps should abort), and the real dlopen is in
ld.elf_so itself. Unfortunately we can't get to it anymore easily
because RTLD_DEFAULT, RTLD_SELF, RTLD_NEXT, NULL all return the
wrong dlopen(). A simple fix is to provide RTLD_DL that returns
symbols defined by the dynamic linker. Here's a patch that does
this. Unless I hear otherwise I am planning to commit this soon.

christos

Index: include/dlfcn.h
===================================================================
RCS file: /cvsroot/src/include/dlfcn.h,v
retrieving revision 1.24
diff -u -u -r1.24 dlfcn.h
--- include/dlfcn.h	16 Feb 2012 23:00:39 -0000	1.24
+++ include/dlfcn.h	30 Nov 2016 01:28:01 -0000
@@ -80,6 +80,7 @@
 #define	RTLD_NEXT	((void *) -1)	/* Search subsequent objects. */
 #define	RTLD_DEFAULT	((void *) -2)	/* Use default search algorithm. */
 #define	RTLD_SELF	((void *) -3)	/* Search the caller itself. */
+#define	RTLD_DL		((void *) -4)	/* Search only the dynamic linker */
 
 /*
  * dlctl() commands
Index: external/gpl3/gcc/dist/libsanitizer/interception/interception_linux.cc
===================================================================
RCS file: /cvsroot/src/external/gpl3/gcc/dist/libsanitizer/interception/interception_linux.cc,v
retrieving revision 1.4
diff -u -u -r1.4 interception_linux.cc
--- external/gpl3/gcc/dist/libsanitizer/interception/interception_linux.cc	24 Jan 2016 09:43:38 -0000	1.4
+++ external/gpl3/gcc/dist/libsanitizer/interception/interception_linux.cc	30 Nov 2016 01:28:01 -0000
@@ -28,12 +28,16 @@
 namespace __interception {
 bool GetRealFunctionAddress(const char *func_name, uptr *func_addr,
     uptr real, uptr wrapper) {
+  void *w = RTLD_NEXT;
 #ifdef __NetBSD__
   // XXX: Until I come up with something better to deal with renames.
   if (mystrcmp(func_name, "sigaction") == 0)
     func_name = "__sigaction14";
+  if (mystrcmp(func_name, "dlopen") == 0 || mystrcmp(func_name, "dlclose") == 0)
+    w = RTLD_DL;
+
 #endif
-  *func_addr = (uptr)dlsym(RTLD_NEXT, func_name);
+  *func_addr = (uptr)dlsym(w, func_name);
   return real == wrapper;
 }
 
Index: libexec/ld.elf_so/rtld.c
===================================================================
RCS file: /cvsroot/src/libexec/ld.elf_so/rtld.c,v
retrieving revision 1.178
diff -u -u -r1.178 rtld.c
--- libexec/ld.elf_so/rtld.c	24 May 2016 20:32:33 -0000	1.178
+++ libexec/ld.elf_so/rtld.c	30 Nov 2016 01:28:02 -0000
@@ -1106,6 +1106,9 @@
 	defobj = NULL;
 
 	switch ((intptr_t)handle) {
+	case (intptr_t)RTLD_DL:
+		obj = &_rtld_objself;
+		goto gotobj;
 	case (intptr_t)NULL:
 	case (intptr_t)RTLD_NEXT:
 	case (intptr_t)RTLD_DEFAULT:
@@ -1115,8 +1118,9 @@
 			lookup_mutex_exit();
 			return NULL;
 		}
-
+gotobj:
 		switch ((intptr_t)handle) {
+		case (intptr_t)RTLD_DL:
 		case (intptr_t)NULL:	 /* Just the caller's shared object. */
 			def = _rtld_symlook_obj(name, hash, obj, flags, ventry);
 			defobj = obj;