exec fails to find files after root changed using pivot_root

Question

I'm trying to experiment with namespaces and chroot emulation, and have succeeded in creating an environment that only has the new root in it (confirmed by a directory traversal), but for some reason, I can't seem to execute anything in it.

Here is a minimally reproducible example:

mkdir /jail
mkdir /jail/bin
mkdir /jail/usr
mkdir /jail/lib
cp /bin/bash /jail/bin

/* isoroot.c */
#define _GNU_SOURCE

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <signal.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/resource.h>
#include <sched.h> /* use clone */
#include <syscall.h>
#include <sys/mount.h>
#include <sched.h>

#ifndef pivot_root
#define pivot_root(new, old) syscall(SYS_pivot_root, new, old)
#endif

int main(int argc, char *argv[])
{
    int res = 0;
    const char *new_root = "/jail";
    const char *old_root = "/oldroot";

    (void) argc;

    if (unshare(CLONE_NEWNS)) { /* Must be executed with privileges. Don't forget this, or we basically blow up the system since we'll remove the real mount */
        fprintf(stderr, "unshare: %s\n", strerror(errno));
        _exit(errno);
    }
    if (mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL)) { /* Prevent shared propagation */
        fprintf(stderr, "mount: %s\n", strerror(errno));
        _exit(errno);
    }
    if (mount(new_root, new_root, NULL, MS_BIND, NULL)) { /* Ensure new_root is a mount point */ 
        fprintf(stderr, "mount: %s\n", strerror(errno));
        _exit(errno);
    }
    if (mkdir("/jail/oldroot", 0777)) { /* should be new_root/old_root */
        fprintf(stderr, "mkdir: %s\n", strerror(errno));
        _exit(errno);
    }

    /* Mount some things we need before we dismount the old root */
    /* /usr has /bin, /sbin, /lib, and /lib64 */
    res = mount("/usr", "/jail/usr", NULL, MS_BIND | MS_REC | MS_RDONLY, NULL);
    if (res) {
        fprintf(stderr, "mount usr failed: %s\n", strerror(errno));
    }

    if (pivot_root(new_root, "/jail/oldroot")) { /* Actually make it the new root */
        fprintf(stderr, "pivot_root: %s\n", strerror(errno));
        _exit(errno);
    }
    if (chdir("/")) { /* Switch to new root */
        fprintf(stderr, "chdir failed: %s\n", strerror(errno));
        _exit(errno);
    }

    if (umount2(old_root, MNT_DETACH)) { /* Unmount old root */
        fprintf(stderr, "umount2 failed: %s\n", strerror(errno));
        _exit(errno);
    }
    if (rmdir(old_root)) { /* Remove old mount point */
        fprintf(stderr, "rmdir failed: %s\n", strerror(errno));
        _exit(errno);
    }
    if (access("/bin/bash", X_OK)) {
        fprintf(stderr, "bash4 not found: %s\n", strerror(errno));
        _exit(errno);
    }
    printf("Got all the way here\n");
    execvp("/bin/bash", argv);
    fprintf(stderr, "%s\n", strerror(errno));
    exit(errno);
}

Compile with:

gcc -Wall -Werror -Wunused -Wextra -Wmaybe-uninitialized -Wstrict-prototypes -Wmissing-prototypes -Wdeclaration-after-statement -Wmissing-declarations -Wmissing-format-attribute -Wnull-dereference -Wformat=2 -Wshadow -Wsizeof-pointer-memaccess -std=gnu99 -pthread -O0 -g -Wstack-protector -fno-omit-frame-pointer -fwrapv -D_FORTIFY_SOURCE=2 -c isoroot.c
gcc -Wall -Werror -Wunused -Wextra -Wmaybe-uninitialized -Wstrict-prototypes -Wmissing-prototypes -Wdeclaration-after-statement -Wmissing-declarations -Wmissing-format-attribute -Wnull-dereference -Wformat=2 -Wshadow -Wsizeof-pointer-memaccess -std=gnu99 -pthread -O0 -g -Wstack-protector -fno-omit-frame-pointer -fwrapv -D_FORTIFY_SOURCE=2 -o isoroot *.o

What seems particularly strange is that the last access succeeds, indicating that access can find /bin/bash. However, execvp always fails with No such file or directory.

I'm assuming since that access is using the new namespace at this point, exec would too, but for some reason, that doesn't seem to hold here. Is there some trick to exec being able to find stuff after the root has been changed?

As a comparison, the example here does work in the sense that exec succeeds, but it doesn't really seem to pivot the root in that everything from the real root is still visible: https://lkml.iu.edu/hypermail/linux/kernel/0803.0/1805.html

chdir("/jail"); 
unshare(CLONE_NEWNS);
mount("/jail", "/jail", NULL, MS_BIND, NULL);
pivot_root("/jail", "/jail/old_root");
chdir("/");
mount("/old_root/bin", "bin", NULL, MS_BIND, NULL);
mount("/old_root/usr", "usr", NULL, MS_BIND, NULL);
mount("/old_root/lib", "lib", NULL, MS_BIND, NULL);
umount2("/old_root", MNT_DETACH);
exec("/busybox");

Using ldd, I've confirmed I should have everything needed to run something like /bin/sh as well. Both /bin and /lib, as well as /lib64 are simply symlinks to folders in

# ldd /bin/bash
        linux-vdso.so.1 (0x00007ffc99116000)
        libtinfo.so.6 => /lib/x86_64-linux-gnu/libtinfo.so.6 (0x00007fd041cb9000)
        libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007fd041cb3000)
        libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fd041ade000)
        /lib64/ld-linux-x86-64.so.2 (0x00007fd041e3a000)

I tried manually adding the symlinks too, just in case, but I just get "Error: File already exists" when I do this at the end, just before exec:

    if (symlink("/bin", "/usr/bin")) {
        fprintf(stderr, "symlink failed: %s\n", strerror(errno));
    }
    if (symlink("/lib", "/usr/lib")) {
        fprintf(stderr, "symlink failed: %s\n", strerror(errno));
    }
    if (symlink("/lib64", "/usr/lib64")) {
        fprintf(stderr, "symlink failed: %s\n", strerror(errno));
    }

Trying to mount /proc and /lib64 fail although /bin and /lib succeed (even though all of these except /proc are symlinks to /usr/<dir>