Re: port-mips/59064 (jemalloc switch to 5.3 broke userland)

[Taylor R Campbell <riastradh%NetBSD.org@localhost>]

[+cc another mips wizard]

At this point I am well and truly baffled.

Running the attached test program (same name as the other program but
rather inaptly named now) under gdb, it works fine every time, with:

(gdb) set environment LD_LIBRARY_PATH = /var/tmp/20250413/lib

(/var/tmp/20250413/lib/libc.so.12 is built with jemalloc using
initial-exec tls, while /lib/libc.so.12 is not -- this way programs
generally work, including gdb, but I can test the broken libc with a
target program.)

If I set a breakpoint on malloc_default, and single-step through, I
get to:

(gdb) disas
Dump of assembler code for function malloc_default:
   0x7853be08 <+0>:     addiu   sp,sp,-144
   0x7853be0c <+4>:     sd      gp,120(sp)
   0x7853be10 <+8>:     lui     gp,0x16
   0x7853be14 <+12>:    addu    gp,gp,t9
   0x7853be18 <+16>:    addiu   gp,gp,-31640
=> 0x7853be1c <+20>:    lw      v0,-23368(gp)
   0x7853be20 <+24>:    .word   0x7c03e83b		/* rdhwr v1,$29 */
...
(gdb) print (void *)$gp
$4 = (void *) 0x78694270 <path+752>
(gdb) x/xw $gp - 23368
0x7868e728:     0xffff9008

That looks fine (0xffff9008 = -28664, a plausible tls offset from
$v1), and $v0 and $v1 hold reasonable values when I single-step
through to

   0x7853be50 <+72>:    lbu     v1,600(a2)

and it all works:

(gdb) print (void *)$v0
$8 = (void *) 0xffff9008
(gdb) print (void *)$v1
$9 = (void *) 0x786b9008

But for some runs of

$ LD_LIBRARY_PATH=/var/tmp/20250413/lib ./rdhwr

it crashes at the lbu instruction, and gdb on the core dump shows:

(gdb) disas
Dump of assembler code for function malloc_default:
   0x7853be08 <+0>:     addiu   sp,sp,-144
   0x7853be0c <+4>:     sd      gp,120(sp)
   0x7853be10 <+8>:     lui     gp,0x16
   0x7853be14 <+12>:    addu    gp,gp,t9
   0x7853be18 <+16>:    addiu   gp,gp,-31640
   0x7853be1c <+20>:    lw      v0,-23368(gp)
   0x7853be20 <+24>:    .word   0x7c03e83b		/* rdhwr v1,$29 */
...
=> 0x7853be50 <+72>:    lbu     v1,600(a2)
...
(gdb) print (void *)$gp                   
$4 = (void *) 0x78694270 <path+752>                               
(gdb) print (void *)$v0
$14 = (void *) 0x7853be20 <malloc_default+24>                                  
(gdb) x/xw $gp - 23368
0x7868e728:     0xffff9008

But:

(gdb) print (void *)$v0
$15 = (void *) 0x7853be20 <malloc_default+24>
(gdb) print (void *)$v1
$16 = (void *) 0x786b9008

$v1 is right, but $v0 is completely bonkers -- and doesn't match what
is at -23368(gp), which surely  lw v0,-23368(gp)  should have loaded!

How does $v0 wind up with the address of the _rdhwr instruction_ when
I run this _not_ under gdb???  Are we dumping core dumps wrong, hiding
some underlying problem?

#include <machine/lwp_private.h>

#include <link_elf.h>
#include <lwp.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

__thread int foo = 123;

struct tls_tcb tcb_storage[0x8000];

__noinline __used
static struct tls_tcb *
gettcb_rdhwr(void)
{
	struct tls_tcb *tcb;

	asm(".set push\n\t"
	    ".set mips32r2\n\t"
	    "rdhwr $3,$29\n\t"
	    "move %[tcb],$3\n\t"
	    ".set pop"
	    : [tcb]"=r"(tcb)
	    :
	    : "$3");

	return tcb - (TLS_TP_OFFSET/sizeof(*tcb) + 1);
}

static int
dlitercb(struct dl_phdr_info *dlpi, size_t size, void *cookie)
{
	char buf[1024];

	snprintf(buf, sizeof(buf),
	    "dlpi_addr=0x%lx\n"
	    "dlpi_name=%s\n"
	    "dlpi_phdr=%p\n"
	    "dlpi_phnum=%d\n"
	    "dlpi_adds=%lld\n"
	    "dlpi_subs=%lld\n"
	    "dlpi_tls_modid=0x%zx\n"
	    "dlpi_tls_data=%p\n"
	    "\n",
	    (long)dlpi->dlpi_addr,
	    dlpi->dlpi_name,
	    dlpi->dlpi_phdr,
	    dlpi->dlpi_phnum,
	    dlpi->dlpi_adds,
	    dlpi->dlpi_subs,
	    dlpi->dlpi_tls_modid,
	    dlpi->dlpi_tls_data);
	(void)write(STDOUT_FILENO, buf, strlen(buf));

	return 0;
}

int
main(void)
{
	char buf[1024];
	struct tls_tcb *tcb_rdhwr, *tcb_syscall;

	(void)dl_iterate_phdr(&dlitercb, NULL);

	tcb_syscall = __lwp_gettcb_fast();
	snprintf_ss(buf, sizeof(buf), "tcb_syscall %p\n", tcb_syscall);
	(void)write(STDOUT_FILENO, buf, strlen(buf));

	tcb_rdhwr = gettcb_rdhwr();
	snprintf_ss(buf, sizeof(buf), "tcb_rdhwr %p\n", tcb_rdhwr);
	(void)write(STDOUT_FILENO, buf, strlen(buf));

	printf("%p\n", malloc(1));
	fflush(stdout);

#if 1
	__USE(tcb_storage);
	__USE(tcb_syscall);
	__USE(tcb_rdhwr);
	__USE(buf);
#else
	__lwp_settcb(tcb_storage);

	tcb_syscall = __lwp_gettcb_fast();
	snprintf_ss(buf, sizeof(buf), "tcb_syscall %p\n", tcb_syscall);
	(void)write(STDOUT_FILENO, buf, strlen(buf));

	tcb_rdhwr = gettcb_rdhwr();
	snprintf_ss(buf, sizeof(buf), "tcb_rdhwr %p\n", tcb_rdhwr);
	(void)write(STDOUT_FILENO, buf, strlen(buf));
#endif

	return 0;
}