How to use PTRACE to get a consistent view of multiple threads?

Question

While I was working on this question, I've come across a possible idea that uses ptrace, but I'm unable to get a proper understanding of how ptrace interacts with threads.

Suppose I have a given, multithreaded main process, and I want to attach to a specific thread in it (perhaps from a forked child).

1. Can I attach to a specific thread? (The manuals diverge on this question.)

2. If so, does that mean that single-stepping only steps through that one thread's instructions? Does it stop all the process's threads?

3. If so, do all the other threads remain stopped while I call PTRACE_SYSCALL or PTRACE_SINGLESTEP, or do all threads continue? Is there a way to step forward only in one single thread but guarantee that the other threads remain stopped?

Basically, I want to synchronise the original program by forcing all threads to stop, and then only execute a small set of single-threaded instructions by single-stepping the one traced thread.

My personal attempts so far look a bit like this:

pid_t target = syscall(SYS_gettid);   // get the calling thread's ID
pid_t pid = fork();

if (pid > 0)
{
    waitpid(pid, NULL, 0);            // synchronise main process

    important_instruction();
}
else if (pid == 0)
{
    ptrace(target, PTRACE_ATTACH, NULL, NULL);    // does this work?

    // cancel parent's "waitpid" call, e.g. with a signal

    // single-step to execute "important_instruction()" above

   ptrace(target, PTRACE_DETACH, NULL, NULL);     // parent's threads resume?

   _Exit(0);
}

However, I'm not sure, and can't find suitable references, that this is concurrently-correct and that important_instruction() is guaranteed to be executed only when all other threads are stopped. I also understand that there may be race conditions when the parent receives signals from elsewhere, and I heard that I should use PTRACE_SEIZE instead, but that doesn't seem to exist everywhere.

Any clarification or references would be greatly appreciated!

Answer 1

I wrote a second test case. I had to add a separate answer, since it was too long to fit into the first one with example output included.

First, here is tracer.c:

#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/ptrace.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <sys/user.h>
#include <dirent.h>
#include <string.h>
#include <signal.h>
#include <errno.h>
#include <stdio.h>
#ifndef   SINGLESTEPS
#define   SINGLESTEPS 10
#endif

/* Similar to getline(), except gets process pid task IDs.
 * Returns positive (number of TIDs in list) if success,
 * otherwise 0 with errno set. */
size_t get_tids(pid_t **const listptr, size_t *const sizeptr, const pid_t pid)
{
    char     dirname[64];
    DIR     *dir;
    pid_t   *list;
    size_t   size, used = 0;

    if (!listptr || !sizeptr || pid < (pid_t)1) {
        errno = EINVAL;
        return (size_t)0;
    }

    if (*sizeptr > 0) {
        list = *listptr;
        size = *sizeptr;
    } else {
        list = *listptr = NULL;
        size = *sizeptr = 0;
    }

    if (snprintf(dirname, sizeof dirname, "/proc/%d/task/", (int)pid) >= (int)sizeof dirname) {
        errno = ENOTSUP;
        return (size_t)0;
    }

    dir = opendir(dirname);
    if (!dir) {
        errno = ESRCH;
        return (size_t)0;
    }

    while (1) {
        struct dirent *ent;
        int            value;
        char           dummy;

        errno = 0;
        ent = readdir(dir);
        if (!ent)
            break;

        /* Parse TIDs. Ignore non-numeric entries. */
        if (sscanf(ent->d_name, "%d%c", &value, &dummy) != 1)
            continue;

        /* Ignore obviously invalid entries. */
        if (value < 1)
            continue;

        /* Make sure there is room for another TID. */
        if (used >= size) {
            size = (used | 127) + 128;
            list = realloc(list, size * sizeof list[0]);
            if (!list) {
                closedir(dir);
                errno = ENOMEM;
                return (size_t)0;
            }
            *listptr = list;
            *sizeptr = size;
        }

        /* Add to list. */
        list[used++] = (pid_t)value;
    }
    if (errno) {
        const int saved_errno = errno;
        closedir(dir);
        errno = saved_errno;
        return (size_t)0;
    }
    if (closedir(dir)) {
        errno = EIO;
        return (size_t)0;
    }

    /* None? */
    if (used < 1) {
        errno = ESRCH;
        return (size_t)0;
    }

    /* Make sure there is room for a terminating (pid_t)0. */
    if (used >= size) {
        size = used + 1;
        list = realloc(list, size * sizeof list[0]);
        if (!list) {
            errno = ENOMEM;
            return (size_t)0;
        }
        *listptr = list;
        *sizeptr = size;
    }

    /* Terminate list; done. */
    list[used] = (pid_t)0;
    errno = 0;
    return used;
}


static int wait_process(const pid_t pid, int *const statusptr)
{
    int   status;
    pid_t p;

    do {
        status = 0;
        p = waitpid(pid, &status, WUNTRACED | WCONTINUED);
    } while (p == (pid_t)-1 && errno == EINTR);
    if (p != pid)
        return errno = ESRCH;

    if (statusptr)
        *statusptr = status;

    return errno = 0;
}

static int continue_process(const pid_t pid, int *const statusptr)
{
    int   status;
    pid_t p;

    do {

        if (kill(pid, SIGCONT) == -1)
            return errno = ESRCH;

        do {
            status = 0;
            p = waitpid(pid, &status, WUNTRACED | WCONTINUED);
        } while (p == (pid_t)-1 && errno == EINTR);

        if (p != pid)
            return errno = ESRCH;

    } while (WIFSTOPPED(status));

    if (statusptr)
        *statusptr = status;

    return errno = 0;
}

void show_registers(FILE *const out, pid_t tid, const char *const note)
{
    struct user_regs_struct regs;
    long                    r;

    do {
        r = ptrace(PTRACE_GETREGS, tid, &regs, &regs);
    } while (r == -1L && errno == ESRCH);
    if (r == -1L)
        return;

#if (defined(__x86_64__) || defined(__i386__)) && __WORDSIZE == 64
    if (note && *note)
        fprintf(out, "Task %d: RIP=0x%016lx, RSP=0x%016lx. %s\n", (int)tid, regs.rip, regs.rsp, note);
    else
        fprintf(out, "Task %d: RIP=0x%016lx, RSP=0x%016lx.\n", (int)tid, regs.rip, regs.rsp);
#elif (defined(__x86_64__) || defined(__i386__)) && __WORDSIZE == 32
    if (note && *note)
        fprintf(out, "Task %d: EIP=0x%08lx, ESP=0x%08lx. %s\n", (int)tid, regs.eip, regs.esp, note);
    else
        fprintf(out, "Task %d: EIP=0x%08lx, ESP=0x%08lx.\n", (int)tid, regs.eip, regs.esp);
#endif
}


int main(int argc, char *argv[])
{
    pid_t *tid = 0;
    size_t tids = 0;
    size_t tids_max = 0;
    size_t t, s;
    long   r;

    pid_t child;
    int   status;

    if (argc < 2 || !strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")) {
        fprintf(stderr, "\n");
        fprintf(stderr, "Usage: %s [ -h | --help ]\n", argv[0]);
        fprintf(stderr, "       %s COMMAND [ ARGS ... ]\n", argv[0]);
        fprintf(stderr, "\n");
        fprintf(stderr, "This program executes COMMAND in a child process,\n");
        fprintf(stderr, "and waits for it to stop (via a SIGSTOP signal).\n");
        fprintf(stderr, "When that occurs, the register state of each thread\n");
        fprintf(stderr, "is dumped to standard output, then the child process\n");
        fprintf(stderr, "is sent a SIGCONT signal.\n");
        fprintf(stderr, "\n");
        return 1;
    }

    child = fork();
    if (child == (pid_t)-1) {
        fprintf(stderr, "fork() failed: %s.\n", strerror(errno));
        return 1;
    }

    if (!child) {
        prctl(PR_SET_DUMPABLE, (long)1);
        prctl(PR_SET_PTRACER, (long)getppid());
        fflush(stdout);
        fflush(stderr);
        execvp(argv[1], argv + 1);
        fprintf(stderr, "%s: %s.\n", argv[1], strerror(errno));
        return 127;
    }

    fprintf(stderr, "Tracer: Waiting for child (pid %d) events.\n\n", (int)child);
    fflush(stderr);

    while (1) {

        /* Wait for a child event. */
        if (wait_process(child, &status))
            break;

        /* Exited? */
        if (WIFEXITED(status) || WIFSIGNALED(status)) {
            errno = 0;
            break;
        }

        /* At this point, only stopped events are interesting. */
        if (!WIFSTOPPED(status))
            continue;

        /* Obtain task IDs. */
        tids = get_tids(&tid, &tids_max, child);
        if (!tids)
            break;

        printf("Process %d has %d tasks,", (int)child, (int)tids);
        fflush(stdout);

        /* Attach to all tasks. */
        for (t = 0; t < tids; t++) {
            do {
                r = ptrace(PTRACE_ATTACH, tid[t], (void *)0, (void *)0);
            } while (r == -1L && (errno == EBUSY || errno == EFAULT || errno == ESRCH));
            if (r == -1L) {
                const int saved_errno = errno;
                while (t-->0)
                    do {
                        r = ptrace(PTRACE_DETACH, tid[t], (void *)0, (void *)0);
                    } while (r == -1L && (errno == EBUSY || errno == EFAULT || errno == ESRCH));
                tids = 0;
                errno = saved_errno;
                break;
            }
        }
        if (!tids) {
            const int saved_errno = errno;
            if (continue_process(child, &status))
                break;
            printf(" failed to attach (%s).\n", strerror(saved_errno));
            fflush(stdout);
            if (WIFCONTINUED(status))
                continue;
            errno = 0;
            break;
        }

        printf(" attached to all.\n\n");
        fflush(stdout);

        /* Dump the registers of each task. */
        for (t = 0; t < tids; t++)
            show_registers(stdout, tid[t], "");
        printf("\n");
        fflush(stdout);

        for (s = 0; s < SINGLESTEPS; s++) {
            do {
                r = ptrace(PTRACE_SINGLESTEP, tid[tids-1], (void *)0, (void *)0);
            } while (r == -1L && errno == ESRCH);
            if (!r) {
                for (t = 0; t < tids - 1; t++)
                    show_registers(stdout, tid[t], "");
                show_registers(stdout, tid[tids-1], "Advanced by one step.");
                printf("\n");
                fflush(stdout);
            } else {
                fprintf(stderr, "Single-step failed: %s.\n", strerror(errno));
                fflush(stderr);
            }
        }

        /* Detach from all tasks. */
        for (t = 0; t < tids; t++)
            do {
                r = ptrace(PTRACE_DETACH, tid[t], (void *)0, (void *)0);
            } while (r == -1 && (errno == EBUSY || errno == EFAULT || errno == ESRCH));
        tids = 0;
        if (continue_process(child, &status))
            break;
        if (WIFCONTINUED(status)) {
            printf("Detached. Waiting for new stop events.\n\n");
            fflush(stdout);
            continue;
        }
        errno = 0;
        break;
    }
    if (errno)
        fprintf(stderr, "Tracer: Child lost (%s)\n", strerror(errno));
    else
    if (WIFEXITED(status))
        fprintf(stderr, "Tracer: Child exited (%d)\n", WEXITSTATUS(status));
    else
    if (WIFSIGNALED(status))
        fprintf(stderr, "Tracer: Child died from signal %d\n", WTERMSIG(status));
    else
        fprintf(stderr, "Tracer: Child vanished\n");
    fflush(stderr);

    return status;
}

tracer.c executes the specified command, waiting for the command to receive a SIGSTOP signal. (tracer.c does not send it itself; you can either have the tracee stop itself, or send the signal externally.)

When the command has stopped, tracer.c attaches a ptrace to every thread, and single-steps one of the threads a fixed number of steps (SINGLESTEPS compile-time constant), showing the pertinent register state for each thread.

After that, it detaches from the command, and sends it a SIGCONT signal to let it continue its operation normally.

Here is a simple test program, worker.c, I used for testing:

#include <pthread.h>
#include <signal.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>

#ifndef   THREADS
#define   THREADS  2
#endif

volatile sig_atomic_t   done = 0;

void catch_done(int signum)
{
    done = signum;
}

int install_done(const int signum)
{
    struct sigaction act;

    sigemptyset(&act.sa_mask);
    act.sa_handler = catch_done;
    act.sa_flags = 0;
    if (sigaction(signum, &act, NULL))
        return errno;
    else
        return 0;
}

void *worker(void *data)
{
    volatile unsigned long *const counter = data;

    while (!done)
        __sync_add_and_fetch(counter, 1UL);

    return (void *)(unsigned long)__sync_or_and_fetch(counter, 0UL);
}

int main(void)
{
    unsigned long   counter = 0UL;
    pthread_t       thread[THREADS];
    pthread_attr_t  attrs;
    size_t          i;

    if (install_done(SIGHUP) ||
        install_done(SIGTERM) ||
        install_done(SIGUSR1)) {
        fprintf(stderr, "Worker: Cannot install signal handlers: %s.\n", strerror(errno));
        return 1;
    }

    pthread_attr_init(&attrs);
    pthread_attr_setstacksize(&attrs, 65536);
    for (i = 0; i < THREADS; i++)
        if (pthread_create(&thread[i], &attrs, worker, &counter)) {
            done = 1;
            fprintf(stderr, "Worker: Cannot create thread: %s.\n", strerror(errno));
            return 1;
        }
    pthread_attr_destroy(&attrs);

    /* Let the original thread also do the worker dance. */
    worker(&counter);

    for (i = 0; i < THREADS; i++)
        pthread_join(thread[i], NULL);

    return 0;
}

Compile both using e.g.

gcc -W -Wall -O3 -fomit-frame-pointer worker.c -pthread -o worker
gcc -W -Wall -O3 -fomit-frame-pointer tracer.c -o tracer

and run either in a separate terminal, or on the background, using e.g.

./tracer ./worker &

The tracer shows the PID of the worker:

Tracer: Waiting for child (pid 24275) events.

At this point, the child is running normally. The action starts when you send a SIGSTOP to the child. The tracer detects it, does the desired tracing, then detaches and lets the child continue normally:

kill -STOP 24275

Process 24275 has 3 tasks, attached to all.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a5d, RSP=0x00007f399cfa6ee8.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a5d, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a63, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a65, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a58, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a5d, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a63, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a65, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a58, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a5d, RSP=0x00007f399cfa6ee8. Advanced by one step.

Task 24275: RIP=0x0000000000400a5d, RSP=0x00007fff6895c428.
Task 24276: RIP=0x0000000000400a5d, RSP=0x00007f399cfb7ee8.
Task 24277: RIP=0x0000000000400a63, RSP=0x00007f399cfa6ee8. Advanced by one step.

Detached. Waiting for new stop events.

You can repeat the above as many times as you wish. Note that I picked the SIGSTOP signal as the trigger, because this way tracer.c is also useful as a basis for generating complex multithreaded core dumps per request (as the multithreaded process can simply trigger it by sending itself a SIGSTOP).

The disassembly of the worker() function the threads are all spinning in the above example:

0x400a50: eb 0b                 jmp          0x400a5d
0x400a52: 66 0f 1f 44 00 00     nopw         0x0(%rax,%rax,1)
0x400a58: f0 48 83 07 01        lock addq    $0x1,(%rdi)          = fourth step
0x400a5d: 8b 05 00 00 00 00     mov          0x0(%rip),%eax       = first step
0x400a63: 85 c0                 test         %eax,%eax            = second step
0x400a65: 74 f1                 je           0x400a58             = third step
0x400a67: 48 8b 07              mov          (%rdi),%rax
0x400a6a: 48 89 c2              mov          %rax,%rdx
0x400a6d: f0 48 0f b1 07        lock cmpxchg %rax,(%rdi)
0x400a72: 75 f6                 jne          0x400a6a
0x400a74: 48 89 d0              mov          %rdx,%rax
0x400a77: c3                    retq

Now, this test program does only show how to stop a process, attach to all of its threads, single-step one of the threads a desired number of instructions, then letting all the threads continue normally; it does not yet prove that the same applies for letting specific threads continue normally (via PTRACE_CONT). However, the detail I describe below indicates, to me, that the same approach should work fine for PTRACE_CONT.

The main problem or surprise I encountered while writing the above test programs was the necessity of the

long r;

do {
    r = ptrace(PTRACE_cmd, tid, ...);
} while (r == -1L && (errno == EBUSY || errno == EFAULT || errno == ESRCH));

loop, especially for the ESRCH case (the others I only added due to the ptrace man page description).

You see, most ptrace commands are only allowed when the task is stopped. However, the task is not stopped when it is still completing e.g. a single-step command. Thus, using the above loop -- perhaps adding a millisecond nanosleep or similar to avoid wasting CPU -- makes sure the previous ptrace command has completed (and thus the task stopped) before we try to supply the new one.

Kerrek SB, I do believe at least some of the troubles you've had with your test programs are due to this issue? To me, personally, it was a kind of a D'oh! moment to realize that of course this is necessary, as ptracing is inherently asynchronous, not synchronous.

(This asynchronicity is also the cause for the SIGCONT-PTRACE_CONT interaction I mentioned above. I do believe with proper handling using the loop shown above, that interaction is no longer a problem -- and is actually quite understandable.)

---

Adding to the comments to this answer:

The Linux kernel uses a set of task state flags in the task_struct structure (see include/linux/sched.h for definition) to keep track of the state of each task. The userspace-facing side of ptrace() is defined in kernel/ptrace.c.

When PTRACE_SINGLESTEP or PTRACE_CONT is called, kernel/ptrace.c:ptrace_continue() handles most of the details. It finishes by calling wake_up_state(child, __TASK_TRACED) (kernel/sched/core.c::try_to_wake_up(child, __TASK_TRACED, 0)).

When a process is stopped via SIGSTOP signal, all tasks will be stopped, and end up in the "stopped, not traced" state.

Attaching to every task (via PTRACE_ATTACH or PTRACE_SEIZE, see kernel/ptrace.c:ptrace_attach()) modifies the task state. However, ptrace state bits (see include/linux/ptrace.h:PT_ constants) are separate from the task runnable state bits (see include/linux/sched.h:TASK_ constants).

After attaching to the tasks, and sending the process a SIGCONT signal, the stopped state is not immediately modified (I believe), since the task is also being traced. Doing PTRACE_SINGLESTEP or PTRACE_CONT ends up in kernel/sched/core.c::try_to_wake_up(child, __TASK_TRACED, 0), which updates the task state, and moves the task to the run queue.

Now, the complicated part that I haven't yet found the code path, is how the task state gets updated in the kernel when the task is next scheduled. My tests indicate that with single-stepping (which is yet another task state flag), only the task state gets updated, with the single-step flag cleared. It seems that PTRACE_CONT is not as reliable; I believe it is because the single-step flag "forces" that task state change. Perhaps there is a "race condition" wrt. the continue signal delivery and state change?

(Further edit: the kernel developers definitely expect wait() to be called, see for example this thread.)

In other words, after noticing that the process has stopped (note that you can use /proc/PID/stat or /proc/PID/status if the process is not a child, and not yet attached to), I believe the following procedure is the most robust one:

pid_t  pid, p; /* Process owning the tasks */
tid_t *tid;    /* Task ID array */
size_t tids;   /* Tasks */
long   result;
int    status;
size_t i;

for (i = 0; i < tids; i++) {
    while (1) {
        result = ptrace(PTRACE_ATTACH, tid[i], (void *)0, (void *)0);
        if (result == -1L && (errno == ESRCH || errno == EBUSY || errno == EFAULT || errno == EIO)) {
            /* To avoid burning up CPU for nothing: */
            sched_yield(); /* or nanosleep(), or usleep() */
            continue;
        }
        break;
    }       
    if (result == -1L) {
        /*
         * Fatal error. First detach from tid[0..i-1], then exit.
        */
    }
}

/* Send SIGCONT to the process. */
if (kill(pid, SIGCONT)) {
    /*
     * Fatal error, see errno. Exit.
    */
}

/* Since we are attached to the process,
 * we can wait() on it. */
while (1) {
    errno = 0;
    status = 0;
    p = waitpid(pid, &status, WCONTINUED);
    if (p == (pid_t)-1) {
        if (errno == EINTR)
            continue;
        else
            break;
    } else
    if (p != pid) {
        errno = ESRCH;
        break;
    } else
    if (WIFCONTINUED(status)) {
        errno = 0;
        break;
    }
}
if (errno) {
    /*
     * Fatal error. First detach from tid[0..tids-1], then exit.
    */
}

/* Single-step each task to update the task states. */
for (i = 0; i < tids; i++) {
    while (1) {
        result = ptrace(PTRACE_SINGLESTEP, tid[i], (void *)0, (void *)0);
        if (result == -1L && errno == ESRCH) {
            /* To avoid burning up CPU for nothing: */
            sched_yield(); /* or nanosleep(), or usleep() */
            continue;
        }
        break;
    }       
    if (result == -1L) {
        /*
         * Fatal error. First detach from tid[0..i-1], then exit.
        */
    }
}

/* Obtain task register structures, to make sure the single-steps
 * have completed and their states have stabilized. */
for (i = 0; i < tids; i++) {
    struct user_regs_struct regs;

    while (1) {
        result = ptrace(PTRACE_GETREGS, tid[i], &regs, &regs);
        if (result == -1L && (errno == ESRCH || errno == EBUSY || errno == EFAULT || errno == EIO)) {
            /* To avoid burning up CPU for nothing: */
            sched_yield(); /* or nanosleep(), or usleep() */
            continue;
        }
        break;
    }       
    if (result == -1L) {
        /*
         * Fatal error. First detach from tid[0..i-1], then exit.
        */
    }
}

After the above, all tasks should be attached and in the expected state, so that e.g. PTRACE_CONT works without further tricks.

If the behaviour changes in future kernels -- I do believe the interaction between the STOP/CONT signals and ptracing is something that might change; at least a question to the LKML developers about this behaviour would be warranted! --, the above procedure will still work robustly. (Erring on the side of caution, by using a loop to PTRACE_SINGLESTEP a few times, might also be a good idea.)

The difference to PTRACE_CONT is that if the behaviour changes in the future, the initial PTRACE_CONT might actually continue the process, causing the ptrace() that follow it to fail. With PTRACE_SINGLESTEP, the process will stop, allowing further ptrace() calls to succeed.

Questions?

Answer 2

Can I attach to a specific thread?

Yes, at least on current kernels.

Does that mean that single-stepping only steps through that one thread's instructions? Does it stop all the process's threads?

Yes. It does not stop the other threads, only the attached one.

Is there a way to step forward only in one single thread but guarantee that the other threads remain stopped?

Yes. Send SIGSTOP to the process (use waitpid(PID,,WUNTRACED) to wait for the process to be stopped), then PTRACE_ATTACH to every thread in the process. Send SIGCONT (using waitpid(PID,,WCONTINUED) to wait for the process to continue).

Since all threads were stopped when you attached, and attaching stops the thread, all threads stay stopped after the SIGCONT signal is delivered. You can single-step the threads in any order you prefer.

---

I found this interesting enough to whip up a test case. (Okay, actually I suspect nobody will take my word for it anyway, so I decided it's better to show proof you can duplicate on your own instead.)

My system seems to follow the man 2 ptrace as described in the Linux man-pages project, and Kerrisk seems to be pretty good at maintaining them in sync with kernel behaviour. In general, I much prefer kernel.org sources wrt. the Linux kernel to other sources.

Summary:

• Attaching to the process itself (TID==PID) stops only the original thread, not all threads.

• Attaching to a specific thread (using TIDs from /proc/PID/task/) does stop that thread. (In other words, the thread with TID == PID is not special.)

• Sending a SIGSTOP to the process will stop all threads, but ptrace() still works absolutely fine.

• If you sent a SIGSTOP to the process, do not call ptrace(PTRACE_CONT, TID) before detaching. PTRACE_CONT seems to interfere with the SIGCONT signal.

  You can first send a SIGSTOP, then PTRACE_ATTACH, then send SIGCONT, without any issues; the thread will stay stopped (due to the ptrace). In other words, PTRACE_ATTACH and PTRACE_DETACH mix well with SIGSTOP and SIGCONT, without any side effects I could see.

• SIGSTOP and SIGCONT affect the entire process, even if you try using tgkill() (or pthread_kill()) to send the signal to a specific thread.

• To stop and continue a specific thread, PTHREAD_ATTACH it; to stop and continue all threads of a process, send SIGSTOP and SIGCONT signals to the process, respectively.

Personally, I believe this validates the approach I suggested in that another question.

Here is the ugly test code you can compile and run to test it for yourself, traces.c:

#define  GNU_SOURCE
#include <stdlib.h>
#include <unistd.h>
#include <sys/wait.h>
#include <sys/ptrace.h>
#include <sys/syscall.h>
#include <dirent.h>
#include <pthread.h>
#include <signal.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>

#ifndef   THREADS
#define   THREADS  3
#endif

static int tgkill(int tgid, int tid, int sig)
{
    int retval;

    retval = syscall(SYS_tgkill, tgid, tid, sig);
    if (retval < 0) {
        errno = -retval;
        return -1;
    }

    return 0;
}

volatile unsigned long counter[THREADS + 1] = { 0UL };

volatile sig_atomic_t run = 0;
volatile sig_atomic_t done = 0;

void handle_done(int signum)
{
    done = signum;
}

int install_done(int signum)
{
    struct sigaction act;
    sigemptyset(&act.sa_mask);
    act.sa_handler = handle_done;
    act.sa_flags = 0;
    if (sigaction(signum, &act, NULL))
        return errno;
    return 0;
}

void *worker(void *data)
{
    volatile unsigned long *const counter = data;

    while (!run)
        ;

    while (!done)
        (*counter)++;

    return (void *)(*counter);
}

pid_t *gettids(const pid_t pid, size_t *const countptr)
{
    char           dirbuf[128];
    DIR           *dir;
    struct dirent *ent;

    pid_t         *data = NULL, *temp;
    size_t         size = 0;
    size_t         used = 0;

    int            tid;
    char           dummy;

    if ((int)pid < 2) {
        errno = EINVAL;
        return NULL;
    }

    if (snprintf(dirbuf, sizeof dirbuf, "/proc/%d/task/", (int)pid) >= (int)sizeof dirbuf) {
        errno = ENAMETOOLONG;
        return NULL;
    }

    dir = opendir(dirbuf);
    if (!dir)
        return NULL;

    while (1) {
        errno = 0;
        ent = readdir(dir);
        if (!ent)
            break;

        if (sscanf(ent->d_name, "%d%c", &tid, &dummy) != 1)
            continue;

        if (tid < 2)
            continue;

        if (used >= size) {
            size = (used | 127) + 129;
            temp = realloc(data, size * sizeof data[0]);
            if (!temp) {
                free(data);
                closedir(dir);
                errno = ENOMEM;
                return NULL;
            }
            data = temp;
        }

        data[used++] = (pid_t)tid;
    }
    if (errno) {
        free(data);
        closedir(dir);
        errno = EIO;
        return NULL;
    }
    if (closedir(dir)) {
        free(data);
        errno = EIO;
        return NULL;
    }

    if (used < 1) {
        free(data);
        errno = ENOENT;
        return NULL;
    }

    size = used + 1;
    temp = realloc(data, size * sizeof data[0]);
    if (!temp) {
        free(data);
        errno = ENOMEM;
        return NULL;
    }
    data = temp;

    data[used] = (pid_t)0;

    if (countptr)
        *countptr = used;

    errno = 0;
    return data;
}

int child_main(void)
{
    pthread_t   id[THREADS];
    int         i;

    if (install_done(SIGUSR1)) {
        fprintf(stderr, "Cannot set SIGUSR1 signal handler.\n");
        return 1;
    }

    for (i = 0; i < THREADS; i++)
        if (pthread_create(&id[i], NULL, worker, (void *)&counter[i])) {
            fprintf(stderr, "Cannot create thread %d of %d: %s.\n", i + 1, THREADS, strerror(errno));
            return 1;
        }

    run = 1;

    kill(getppid(), SIGUSR1);

    while (!done)
        counter[THREADS]++;

    for (i = 0; i < THREADS; i++)
        pthread_join(id[i], NULL);

    printf("Final counters:\n");
    for (i = 0; i < THREADS; i++)
        printf("\tThread %d: %lu\n", i + 1, counter[i]);
    printf("\tMain thread: %lu\n", counter[THREADS]);

    return 0;
}

int main(void)
{
    pid_t   *tid = NULL;
    size_t   tids = 0;
    int      i, k;
    pid_t    child, p;

    if (install_done(SIGUSR1)) {
        fprintf(stderr, "Cannot set SIGUSR1 signal handler.\n");
        return 1;
    }

    child = fork();
    if (!child)
        return child_main();

    if (child == (pid_t)-1) {
        fprintf(stderr, "Cannot fork.\n");
        return 1;
    }

    while (!done)
        usleep(1000);

    tid = gettids(child, &tids);
    if (!tid) {
        fprintf(stderr, "gettids(): %s.\n", strerror(errno));
        kill(child, SIGUSR1);
        return 1;
    }

    fprintf(stderr, "Child process %d has %d tasks.\n", (int)child, (int)tids);
    fflush(stderr);

    for (k = 0; k < (int)tids; k++) {
        const pid_t t = tid[k];

        if (ptrace(PTRACE_ATTACH, t, (void *)0L, (void *)0L)) {
            fprintf(stderr, "Cannot attach to TID %d: %s.\n", (int)t, strerror(errno));
            kill(child, SIGUSR1);
            return 1;
        }

        fprintf(stderr, "Attached to TID %d.\n\n", (int)t);

        fprintf(stderr, "Peeking the counters in the child process:\n");
        for (i = 0; i <= THREADS; i++) {
            long v;
            do {
                errno = 0;
                v = ptrace(PTRACE_PEEKDATA, t, &counter[i], NULL);
            } while (v == -1L && (errno == EIO || errno == EFAULT || errno == ESRCH));
            fprintf(stderr, "\tcounter[%d] = %lu\n", i, (unsigned long)v);
        }
        fprintf(stderr, "Waiting a short moment ... ");
        fflush(stderr);

        usleep(250000);

        fprintf(stderr, "and another peek:\n");
        for (i = 0; i <= THREADS; i++) {
            long v;
            do {
                errno = 0;
                v = ptrace(PTRACE_PEEKDATA, t, &counter[i], NULL);
            } while (v == -1L && (errno == EIO || errno == EFAULT || errno == ESRCH));
            fprintf(stderr, "\tcounter[%d] = %lu\n", i, (unsigned long)v);
        }
        fprintf(stderr, "\n");
        fflush(stderr);

        usleep(250000);

        ptrace(PTRACE_DETACH, t, (void *)0L, (void *)0L);
    }

    for (k = 0; k < 4; k++) {
        const pid_t t = tid[tids / 2];

        if (k == 0) {
            fprintf(stderr, "Sending SIGSTOP to child process ... ");
            fflush(stderr);
            kill(child, SIGSTOP);
        } else
        if (k == 1) {
            fprintf(stderr, "Sending SIGCONT to child process ... ");
            fflush(stderr);
            kill(child, SIGCONT);
        } else
        if (k == 2) {
            fprintf(stderr, "Sending SIGSTOP to TID %d ... ", (int)tid[0]);
            fflush(stderr);
            tgkill(child, tid[0], SIGSTOP);
        } else
        if (k == 3) {
            fprintf(stderr, "Sending SIGCONT to TID %d ... ", (int)tid[0]);
            fflush(stderr);
            tgkill(child, tid[0], SIGCONT);
        }
        usleep(250000);
        fprintf(stderr, "done.\n");
        fflush(stderr);

        if (ptrace(PTRACE_ATTACH, t, (void *)0L, (void *)0L)) {
            fprintf(stderr, "Cannot attach to TID %d: %s.\n", (int)t, strerror(errno));
            kill(child, SIGUSR1);
            return 1;
        }

        fprintf(stderr, "Attached to TID %d.\n\n", (int)t);

        fprintf(stderr, "Peeking the counters in the child process:\n");
        for (i = 0; i <= THREADS; i++) {
            long v;
            do {
                errno = 0;
                v = ptrace(PTRACE_PEEKDATA, t, &counter[i], NULL);
            } while (v == -1L && (errno == EIO || errno == EFAULT || errno == ESRCH));
            fprintf(stderr, "\tcounter[%d] = %lu\n", i, (unsigned long)v);
        }
        fprintf(stderr, "Waiting a short moment ... ");
        fflush(stderr);

        usleep(250000);

        fprintf(stderr, "and another peek:\n");
        for (i = 0; i <= THREADS; i++) {
            long v;
            do {
                errno = 0;
                v = ptrace(PTRACE_PEEKDATA, t, &counter[i], NULL);
            } while (v == -1L && (errno == EIO || errno == EFAULT || errno == ESRCH));
            fprintf(stderr, "\tcounter[%d] = %lu\n", i, (unsigned long)v);
        }
        fprintf(stderr, "\n");
        fflush(stderr);

        usleep(250000);

        ptrace(PTRACE_DETACH, t, (void *)0L, (void *)0L);
    }

    kill(child, SIGUSR1);

    do {
        p = waitpid(child, NULL, 0);
        if (p == -1 && errno != EINTR)
            break;
    } while (p != child);

    return 0;
}

Compile and run using e.g.

gcc -DTHREADS=3 -W -Wall -O3 traces.c -pthread -o traces
./traces

The output is a dump of the child process counters (each one incremented in a separate thread, including the original thread which uses the final counter). Compare the counters across the short wait. For example:

Child process 18514 has 4 tasks.
Attached to TID 18514.

Peeking the counters in the child process:
    counter[0] = 0
    counter[1] = 0
    counter[2] = 0
    counter[3] = 0
Waiting a short moment ... and another peek:
    counter[0] = 18771865
    counter[1] = 6435067
    counter[2] = 54247679
    counter[3] = 0

As you can see above, only the initial thread (whose TID == PID), which uses the final counter, is stopped. The same happens for the other three threads, too, which use the first three counters in order:

Attached to TID 18515.

Peeking the counters in the child process:
    counter[0] = 25385151
    counter[1] = 13459822
    counter[2] = 103763861
    counter[3] = 560872
Waiting a short moment ... and another peek:
    counter[0] = 25385151
    counter[1] = 69116275
    counter[2] = 120500164
    counter[3] = 9027691

Attached to TID 18516.

Peeking the counters in the child process:
    counter[0] = 25397582
    counter[1] = 105905400
    counter[2] = 155895025
    counter[3] = 17306682
Waiting a short moment ... and another peek:
    counter[0] = 32358651
    counter[1] = 105905400
    counter[2] = 199601078
    counter[3] = 25023231

Attached to TID 18517.

Peeking the counters in the child process:
    counter[0] = 40600813
    counter[1] = 111675002
    counter[2] = 235428637
    counter[3] = 32298929
Waiting a short moment ... and another peek:
    counter[0] = 48727731
    counter[1] = 143870702
    counter[2] = 235428637
    counter[3] = 39966259

The next two cases examine the SIGCONT/SIGSTOP wrt. the entire process:

Sending SIGSTOP to child process ... done.
Attached to TID 18516.

Peeking the counters in the child process:
    counter[0] = 56887263
    counter[1] = 170646440
    counter[2] = 235452621
    counter[3] = 48077803
Waiting a short moment ... and another peek:
    counter[0] = 56887263
    counter[1] = 170646440
    counter[2] = 235452621
counter[3] = 48077803

Sending SIGCONT to child process ... done.
Attached to TID 18516.

Peeking the counters in the child process:
    counter[0] = 64536344
    counter[1] = 182359343
    counter[2] = 253660731
    counter[3] = 56422231
Waiting a short moment ... and another peek:
    counter[0] = 72029244
    counter[1] = 182359343
    counter[2] = 288014365
    counter[3] = 63797618

As you can see, sending SIGSTOP will stop all threads, but not hinder with ptrace(). Similarly, after SIGCONT, the threads continue running as normal.

The final two cases examine the effects of using tgkill() to send the SIGSTOP/SIGCONT to a specific thread (the one that corresponds to the first counter), while attaching to another thread:

Sending SIGSTOP to TID 18514 ... done.
Attached to TID 18516.

Peeking the counters in the child process:
    counter[0] = 77012930
    counter[1] = 183059526
    counter[2] = 344043770
    counter[3] = 71120227
Waiting a short moment ... and another peek:
    counter[0] = 77012930
    counter[1] = 183059526
    counter[2] = 344043770
    counter[3] = 71120227

Sending SIGCONT to TID 18514 ... done.
Attached to TID 18516.

Peeking the counters in the child process:
    counter[0] = 88082419
    counter[1] = 194059048
    counter[2] = 359342314
    counter[3] = 84887463
Waiting a short moment ... and another peek:
    counter[0] = 100420161
    counter[1] = 194059048
    counter[2] = 392540525
    counter[3] = 111770366

Unfortunately, but as expected, the disposition (stopped/running) is process-wide, not thread-specific, as you can see above. This means that to stop a specific threads and let the other threads run normally, you need to separately PTHREAD_ATTACH to the threads you wish to stop.

To prove all my statements above, you may have to add test cases; I ended up having quite a few copies of the code, all slightly edited, to test it all, and I'm not sure I picked the most complete set. I'd be happy to expand the test program, if you find omissions.

Questions?

Answer 3

Each thread in the process is traced individually (and each can be potentially traced by a different tracing process, or be untraced). When you call ptrace attach, you are always attaching to just a single thread. Only that thread will be stopped - the other threads will continue running as they were.

Recent versions of the ptrace() man page make this very clear:

Attachment and subsequent commands are per thread: in a multithreaded process, every thread can be individually attached to a (potentially different) tracer, or left not attached and thus not debugged. Therefore, "tracee" always means "(one) thread", never "a (possibly multithreaded) process". Ptrace commands are always sent to a specific tracee using a call of the form

ptrace(PTRACE_foo, pid, ...)

where pid is the thread ID of the corresponding Linux thread.

(Note that in this page, a "multithreaded process" means a thread group consisting of threads created using the clone(2) CLONE_THREAD flag.)

Single-stepping affects only the thread that you direct it at. If the other threads are running they continue running, and if they are in tracing stop they stay in tracing stop. (This means that if the thread you are single-stepping tries to acquire a mutex or similar synchronisation resource that is held by another non-running thread, it will not be able to acquire that mutex).

If you want to stop all the threads of the process while you single-step one thread, you will need to attach to all of the threads. There is the added complication that if the process is running while you're trying to attach to it, new threads could be created while you're enumerating them.

Answer 4

Does it stop all the process's threads?

Yes It traces the process, all threads of this process are stop. Imagine it it wasn't how could you see the dirfferent thread in your IDE.

from the manual:

The ptrace() system call provides a means by which one process (the "tracer") may observe and control the execution of another process (the "tracee")

Example code to attach:

printf("Attaching to process %d\n",Tpid);
if ((ptrace(PTRACE_ATTACH, Tpid, 0, 0)) != 0) {;
    printf("Attach result %d\n",res);
}

So yes you are atached to a thread and yes it stops all the threads of the process.

if ((res = ptrace(PTRACE_SINGLESTEP, Tpid, 0, signo)) < 0) {
perror("Ptrace singlestep error");
exit(1);
}
res = wait(&stat);

maybe see here : http://www.secretmango.com/jimb/Whitepapers/ptrace/ptrace.html