Fixed-address mmap call segfault when invoked from main but works inside custom malloc override in C++

Question

I have 3 files. 1.cpp has a function map that uses mmap to do some fixed memory mapping. 2.cpp is a malloc interceptor and 3.cpp has the main function.

The problem is ... depending on from where map is called it either works or segfaults without even executing anything from main.

1.cpp

#include <cstdint>
#include <sys/mman.h>

#define end 0x10007fff7fff
#define start 0x02008fff7000
#define MAPFLAGS (MAP_PRIVATE | MAP_FIXED | MAP_ANON | MAP_NORESERVE)

extern "C" {
void map()
{

    void *ret;
    uintptr_t size = end - start + 1;
    ret = mmap((void*) start, size, PROT_READ | PROT_WRITE, MAPFLAGS, -1, 0);
}
}

2.cpp

#include <cstdio>

extern "C" void map(void);

extern "C" void* malloc(size_t usize)
{
    // map(); // uncommenting this and commenting out in 1.cpp works
    void* ptr = (void*) 0x8003fffb000; // test address which falls b/n
    *(char*)ptr = 0xab;
    return ptr;
}

3.cpp

#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <unistd.h>

extern "C" void map(void);

int main()
{
    puts("Inside main");
    map();
    void* ptr = NULL;
    ptr = (void*) malloc(6); // returns 0x8003fffb000 address which should be mapped eariler
    *(char*) ptr = 0xef;
    printf("%x\n", *(char*)ptr);
    return 0;
}

To build:

clang++ -c 1.cpp 2.cpp 3.cpp -g
clang++ 1.o 2.o 3.o -g

If I comment out the map call from 3.cpp and run from 2.cpp it works as expected. But the other way it gives segfault.

When I use clang/gcc instead of clang++/g++ it works in both the cases. I have no idea what is going on.

P.S. I am trying to implement a toy version of address sanitizer hence intercepting malloc calls to set up setup shadow memory in the runtime.

Answer 1

Yep malloc being called before main was indeed the problem. Thanks @IgorTandetnik and @Mike for the detailed clarification.

To fix the issue I'm currently doing this. Its dirty but seems to work. I'll have to clean this later.

1.h

#define end 0x10007fff7fff
#define start 0x02008fff7000
#define MAPFLAGS (MAP_PRIVATE | MAP_FIXED | MAP_ANON | MAP_NORESERVE)

extern bool ismapped;
void internal_map(void);

1.cpp

#include <cstdint>
#include <sys/mman.h>
#include "1.h"

void internal_map() {
    if (!ismapped) {
        void *ret;
        uintptr_t size = end - start + 1;
        ret = mmap((void*) start, size, PROT_READ | PROT_WRITE, MAPFLAGS, -1, 0);
        (MAP_FAILED == ret? ismapped = false: ismapped = true);
    }
    else
        ismapped = true;
}

extern "C" {
void map()
{
    if (ismapped)
        return;
    internal_map();
}
}

2.cpp

#include <cstdio>
#include <dlfcn.h>
#include "1.h"

extern "C" void map(void);
bool ismapped = false;
void* (*libc_malloc)(size_t size) = NULL;


extern "C" void* malloc(size_t usize)
{
    void *ret;
    if (!libc_malloc)
        libc_malloc = (void*(*)(size_t)) dlsym(RTLD_NEXT, "malloc");

    ret = libc_malloc(usize);

    if (!ismapped) {
        internal_map();
    }
    else {
        void* ptr = (void*) 0x8003fffb000;
        *(char*)ptr = 0xab;
    }
    return ret;
}

Basically, checking if the mapping is successful or not each time malloc is called.

Answer 2

As @IgorTandetnik says, malloc is called before main, and is therefore called before map when you are calling map from main.

He's also right in suggesting that the segfaulting call to malloc is coming from libstdc++.so initialisation, although this is far from the first malloc call in the program initialisation. (It's actually the 38th, though I won't bore you with the demonstration). malloc calls commence in scope of the C-runtime startup routine _start() (defined in /usr/lib/x86_64-linux-gnu/Scrt1.o, linked by default) before it calls main.

You commented:

but how do I verify that?

There's more than one way but the most vivid is just debugging with gdb. Here's a version of your code that we can conditionally compile to call map in malloc or to call map in main:

$ tail -n +1 *.cpp
==> 1.cpp <==
#include <cstdint>
#include <sys/mman.h>

#define end 0x10007fff7fff
#define start 0x02008fff7000
#define MAPFLAGS (MAP_PRIVATE | MAP_FIXED | MAP_ANON | MAP_NORESERVE)

extern "C" {
void map()
{
    void *ret;
    uintptr_t size = end - start + 1;
    ret = mmap((void*) start, size, PROT_READ | PROT_WRITE, MAPFLAGS, -1, 0);
}
}

==> 2.cpp <==
#include <cstdio>

extern "C" void map(void);

extern "C" void* malloc(size_t usize)
{
#ifdef MAP_IN_MALLOC
    map(); // uncommenting this and commenting out in 1.cpp works
#endif
    void* ptr = (void*) 0x8003fffb000; // test address which falls b/n
    *(char*)ptr = 0xab;
    return ptr;
}

==> 3.cpp <==
#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <unistd.h>

extern "C" void map(void);

int main()
{
    puts("Inside main");
#ifndef MAP_IN_MALLOC
    map();
#endif 
    void* ptr = NULL;
    ptr = (void*) malloc(6); // returns 0x8003fffb000 address which should be mapped eariler
    *(char*) ptr = 0xef;
    printf("%x\n", *(char*)ptr);
    return 0;
}

Let's first see that it does what you expect both ways:

$ clang++ 1.cpp 2.cpp 3.cpp -g && ./a.out
Segmentation fault (core dumped)

$ clang++ -DMAP_IN_MALLOC 1.cpp 2.cpp 3.cpp -g && ./a.out
Inside main
ffffffef

Then debug into that last one:

$ gdb a.out
...[cut]...
Reading symbols from a.out...

Set breakpoints on both malloc and main and see which one we get to first:

(gdb) b main
Breakpoint 1 at 0x11ef: file 3.cpp, line 10.
(gdb) b malloc
Breakpoint 2 at 0x11bc: file 2.cpp, line 8.
(gdb) r
Starting program: /home/imk/develop/so/scrap/a.out 

Breakpoint 2.22, malloc (size=1497) at ../include/rtld-malloc.h:56
warning: 56 ../include/rtld-malloc.h: No such file or directory

So it's malloc before main. Let's see the backtrace:

(gdb) bt
malloc (size=1497) at ../include/rtld-malloc.h:56
#1  __minimal_calloc (nmemb=<optimised out>, size=1) at ./elf/dl-minimal-malloc.c:91
#2  0x00007ffff7fd172d in calloc (b=1, a=<optimised out>) at ../include/rtld-malloc.h:44
#3  _dl_new_object (realname=realname@entry=0x7ffff7ff3fa7 "", libname=libname@entry=0x7ffff7ff3fa7 "", type=type@entry=0, loader=loader@entry=0x0, mode=mode@entry=536870912, 
    nsid=nsid@entry=0) at ./elf/dl-object.c:92
#4  0x00007ffff7fe602c in dl_main (phdr=0x555555554040, phnum=<optimised out>, user_entry=<optimised out>, auxv=<optimised out>) at ./elf/rtld.c:1637
#5  0x00007ffff7fe3f46 in _dl_sysdep_start (start_argptr=start_argptr@entry=0x7fffffffdda0, dl_main=dl_main@entry=0x7ffff7fe5af0 <dl_main>)
    at ../sysdeps/unix/sysv/linux/dl-sysdep.c:140
#6  0x00007ffff7fe575e in _dl_start_final (arg=0x7fffffffdda0) at ./elf/rtld.c:494
#7  _dl_start (arg=0x7fffffffdda0) at ./elf/rtld.c:581
#8  0x00007ffff7fe4548 in _start () from /lib64/ld-linux-x86-64.so.2
#9  0x0000000000000001 in ?? ()
#10 0x00007fffffffe0f5 in ?? ()
#11 0x0000000000000000 in ?? ()

We're still executing _start pre-main. Let's see what's been loaded:

(gdb) info proc mappings
process 19016
Mapped address spaces:

          Start Addr           End Addr       Size     Offset  Perms  objfile
      0x555555554000     0x555555555000     0x1000        0x0  r--p   /home/imk/develop/so/scrap/a.out
      0x555555555000     0x555555556000     0x1000     0x1000  r-xp   /home/imk/develop/so/scrap/a.out
      0x555555556000     0x555555557000     0x1000     0x2000  r--p   /home/imk/develop/so/scrap/a.out
      0x555555557000     0x555555559000     0x2000     0x2000  rw-p   /home/imk/develop/so/scrap/a.out
      0x7ffff7fbf000     0x7ffff7fc3000     0x4000        0x0  r--p   [vvar]
      0x7ffff7fc3000     0x7ffff7fc5000     0x2000        0x0  r-xp   [vdso]
      0x7ffff7fc5000     0x7ffff7fc6000     0x1000        0x0  r--p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7fc6000     0x7ffff7ff1000    0x2b000     0x1000  r-xp   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7ff1000     0x7ffff7ffb000     0xa000    0x2c000  r--p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7ffb000     0x7ffff7fff000     0x4000    0x36000  rw-p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffffffde000     0x7ffffffff000    0x21000        0x0  rw-p   [stack]
  0xffffffffff600000 0xffffffffff601000     0x1000        0x0  --xp   [vsyscall]
  

The only files mapped so far are a.out and the dynamic linker itself. Not yet libstdc++.so or even libc.so.

Let's carry on to main:

(gdb) del 2
(gdb) c
Continuing.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Breakpoint 1, main () at 3.cpp:10
10      puts("Inside main");

And now the program is fully loaded:

(gdb) info proc mappings
process 19416
Mapped address spaces:

          Start Addr           End Addr       Size     Offset  Perms  objfile
       0x2008fff7000     0x10007fff8000 0xdfff0001000        0x0  rw-p   
      0x555555554000     0x555555555000     0x1000        0x0  r--p   /home/imk/develop/so/scrap/a.out
      0x555555555000     0x555555556000     0x1000     0x1000  r-xp   /home/imk/develop/so/scrap/a.out
      0x555555556000     0x555555557000     0x1000     0x2000  r--p   /home/imk/develop/so/scrap/a.out
      0x555555557000     0x555555558000     0x1000     0x2000  r--p   /home/imk/develop/so/scrap/a.out
      0x555555558000     0x555555559000     0x1000     0x3000  rw-p   /home/imk/develop/so/scrap/a.out
      0x7ffff7800000     0x7ffff7828000    0x28000        0x0  r--p   /usr/lib/x86_64-linux-gnu/libc.so.6
      0x7ffff7828000     0x7ffff79b0000   0x188000    0x28000  r-xp   /usr/lib/x86_64-linux-gnu/libc.so.6
      0x7ffff79b0000     0x7ffff79ff000    0x4f000   0x1b0000  r--p   /usr/lib/x86_64-linux-gnu/libc.so.6
      0x7ffff79ff000     0x7ffff7a03000     0x4000   0x1fe000  r--p   /usr/lib/x86_64-linux-gnu/libc.so.6
      0x7ffff7a03000     0x7ffff7a05000     0x2000   0x202000  rw-p   /usr/lib/x86_64-linux-gnu/libc.so.6
      0x7ffff7a05000     0x7ffff7a12000     0xd000        0x0  rw-p   
      0x7ffff7c00000     0x7ffff7c9d000    0x9d000        0x0  r--p   /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33
      0x7ffff7c9d000     0x7ffff7de5000   0x148000    0x9d000  r-xp   /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33
      0x7ffff7de5000     0x7ffff7e6c000    0x87000   0x1e5000  r--p   /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33
      0x7ffff7e6c000     0x7ffff7e77000     0xb000   0x26b000  r--p   /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33
      0x7ffff7e77000     0x7ffff7e7a000     0x3000   0x276000  rw-p   /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.33
      0x7ffff7e7a000     0x7ffff7e7e000     0x4000        0x0  rw-p   
      0x7ffff7e8e000     0x7ffff7e92000     0x4000        0x0  rw-p   
      0x7ffff7e92000     0x7ffff7e96000     0x4000        0x0  r--p   /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
      0x7ffff7e96000     0x7ffff7eba000    0x24000     0x4000  r-xp   /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
      0x7ffff7eba000     0x7ffff7ebe000     0x4000    0x28000  r--p   /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
      0x7ffff7ebe000     0x7ffff7ebf000     0x1000    0x2b000  r--p   /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
      0x7ffff7ebf000     0x7ffff7ec0000     0x1000    0x2c000  rw-p   /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
      0x7ffff7ec0000     0x7ffff7ed0000    0x10000        0x0  r--p   /usr/lib/x86_64-linux-gnu/libm.so.6
      0x7ffff7ed0000     0x7ffff7f4f000    0x7f000    0x10000  r-xp   /usr/lib/x86_64-linux-gnu/libm.so.6
      0x7ffff7f4f000     0x7ffff7fa7000    0x58000    0x8f000  r--p   /usr/lib/x86_64-linux-gnu/libm.so.6
      0x7ffff7fa7000     0x7ffff7fa8000     0x1000    0xe7000  r--p   /usr/lib/x86_64-linux-gnu/libm.so.6
      0x7ffff7fa8000     0x7ffff7fa9000     0x1000    0xe8000  rw-p   /usr/lib/x86_64-linux-gnu/libm.so.6
      0x7ffff7fbd000     0x7ffff7fbf000     0x2000        0x0  rw-p   
      0x7ffff7fbf000     0x7ffff7fc3000     0x4000        0x0  r--p   [vvar]
      0x7ffff7fc3000     0x7ffff7fc5000     0x2000        0x0  r-xp   [vdso]
      0x7ffff7fc5000     0x7ffff7fc6000     0x1000        0x0  r--p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7fc6000     0x7ffff7ff1000    0x2b000     0x1000  r-xp   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7ff1000     0x7ffff7ffb000     0xa000    0x2c000  r--p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7ffb000     0x7ffff7ffd000     0x2000    0x36000  r--p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffff7ffd000     0x7ffff7fff000     0x2000    0x38000  rw-p   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
      0x7ffffffde000     0x7ffffffff000    0x21000        0x0  rw-p   [stack]
  0xffffffffff600000 0xffffffffff601000     0x1000        0x0  --xp   [vsyscall]
  

If we debug the segfaulting version with a break on main:

$ clang++ 1.cpp 2.cpp 3.cpp -g
$ gdb a.out
...[cut]...
Reading symbols from a.out...
(gdb) b main
Breakpoint 1 at 0x11ef: file 3.cpp, line 10.

we segfault before getting there:

(gdb) r
Starting program: /home/imk/develop/so/scrap/a.out 
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Program received signal SIGSEGV, Segmentation fault.
0x00005555555551ca in malloc (usize=73728) at 2.cpp:11
11      *(char*)ptr = 0xab;
(gdb) bt
#0  0x00005555555551ca in malloc (usize=73728) at 2.cpp:11
#1  0x00007ffff7cb738f in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
#2  0x00007ffff7fca71f in call_init (l=<optimised out>, argc=argc@entry=1, argv=argv@entry=0x7fffffffdda8, env=env@entry=0x7fffffffddb8) at ./elf/dl-init.c:74
#3  0x00007ffff7fca824 in call_init (env=<optimised out>, argv=<optimised out>, argc=<optimised out>, l=<optimised out>) at ./elf/dl-init.c:120
#4  _dl_init (main_map=0x7ffff7ffe2e0, argc=1, argv=0x7fffffffdda8, env=0x7fffffffddb8) at ./elf/dl-init.c:121
#5  0x00007ffff7fe45a0 in _dl_start_user () from /lib64/ld-linux-x86-64.so.2
#6  0x0000000000000001 in ?? ()
#7  0x00007fffffffe0f5 in ?? ()
#8  0x0000000000000000 in ?? ()

and the segfaulting malloc (the 38th) is called somewhere in /lib/x86_64-linux-gnu/libstdc++.so.6