How to remote-control GDB on Linux

Question

I have to implement something like this:

1. start a program (e.g a.out) under GDB
2. set some break points
3. periodically send CTRL-C signal to the GDB to pause the execution of a.out.
4. do some commands like "bt, info threads" at the stop point or break points
5. continue the execution of a.out
6. until the end of the a.out

Those steps can be executed interactively under shell, but I need to automate that in a program. I'm considering using fork to create a subprocess for GDB and use popen to execute the initial GDB launch command, but how can I periodically send those GDB subcommands (bt, continue) to that subprocess and let it execute them?

I'm stuck at this point and any thoughts would be greatly appreciated. Thanks ahead.

Answer 1

This is a very simplistic implementation. It forks the target process with no pipes, we just need to learn it's pid. Then it forks gdb with the -p <PID> option to attach to our target. The fork for GDB sets up pipes for stdin/stdout/stderr before exec'ing, so that we can remote control GDB.

A few interesting notes:

1. When GDB is running a debug-target, it doesn't respond to SIGINT. You have to send the SIGINT to the debug-target. This is why I fork twice rather than launching gdb --args <target>. I need the PID of the process it's debugging so I can send SIGINT.
2. When you attach pipes to a process' stdout and stderr, you must read them or the target process will eventually block (when they fill the pipe's buffer). My implementation is stupid here, because I didn't want to take the time to use threads or do proper select calls.
3. You have to be somewhat careful about when the APIs will block. Note that I'm using read/write instead of fread,fwrite due to their behavior when they can't read the amount I have requested.

The 'tracer' program is:

#include <stdio.h>
#include <string.h>

#include <unistd.h>
#include <signal.h>
#include <fcntl.h>
#include <sys/select.h>

char gdb_pid_buf[20];

char *gdb_argv[] =
{
  "gdb",
  "-p",
  gdb_pid_buf,
  NULL
};

char *child_argv[] =
{
  "./looper",
  NULL
};

const char GDB_PROMPT[] = "(gdb)";

int wait_for_prompt(const char *prefix, int fd)
{
  char readbuf[4096];
  size_t used = 0;
  while(1)
  {
    ssize_t amt;
    char *prompt;
    char *end;

    amt = read(fd, readbuf+used, sizeof(readbuf)-used-1);
    if(amt == -1)
    {
      return 1;
    }
    else if(amt == 0)
    {  }
    else
    {
      used += amt;

      readbuf[used] = '\0';
      for(end = strstr(readbuf, "\n"); end; end= strstr(readbuf, "\n"))
      {
        size_t consumed;
        size_t remaining;

        *end = '\0';
        printf("%s: %s\n", prefix, readbuf);

        consumed = (end-readbuf) + strlen("\n");
        remaining = used - consumed;
        memmove(readbuf, readbuf+consumed, remaining);
        used -= consumed;
      }

      prompt = strstr(readbuf, GDB_PROMPT);
      if(prompt)
      {
        *prompt = '\0';
        printf("%s: %s", prefix, readbuf);
        printf("[PROMPT]\n");
        fflush(stdout);
        break;
      }
    }
  }
  return 0;
}

int main(int argc, char *argv)
{
  int i;

  int stdin_pipe[2];
  int stdout_pipe[2];
  int stderr_pipe[2];

  pipe(stdin_pipe);
  pipe(stdout_pipe);
  pipe(stderr_pipe);

  int gdb_pid;
  int child_pid;

  //Launch child
  child_pid = fork();
  if(child_pid == 0)
  {
    close(stdin_pipe[0]);
    close(stdout_pipe[0]);
    close(stderr_pipe[0]);
    close(stdin_pipe[1]);
    close(stdout_pipe[1]);
    close(stderr_pipe[1]);

    execvp(child_argv[0], child_argv);
    return 0;
  }

  sprintf(gdb_pid_buf, "%d", child_pid);

  //Launch gdb with command-line args to attach to child.
  gdb_pid = fork();
  if(gdb_pid == 0)
  {
    close(stdin_pipe[1]);
    close(stdout_pipe[0]);
    close(stderr_pipe[0]);

    dup2(stdin_pipe[0],0);
    dup2(stdout_pipe[1],1);
    dup2(stderr_pipe[1],2);

    execvp(gdb_argv[0], gdb_argv);
    return 0;
  }

  close(stdin_pipe[0]);
  close(stdout_pipe[1]);
  close(stderr_pipe[1]);

  //Wait for GDB to reach its prompt
  if(wait_for_prompt("GDB", stdout_pipe[0]))
    {fprintf(stderr,"child died\n");return 1;}

  printf("[SENDING \"continue\\n\"]\n");
  fflush(stdout);
  write(stdin_pipe[1], "continue\n", strlen("continue\n"));

  sleep(4);

  printf("[SENDING \"CTRL+C\"]\n");
  fflush(stdout);
  kill(child_pid, SIGINT);

  //Then read through all the output until we reach a prompt.
  if(wait_for_prompt("POST SIGINT", stdout_pipe[0]))
    {fprintf(stderr,"child died\n");return 1;}

  //Ask for the stack trace
  printf("[SENDING \"where\\n\"]\n");
  fflush(stdout);
  write(stdin_pipe[1], "where\n", strlen("where\n"));

  //read through the stack trace output until the next prompt
  if(wait_for_prompt("TRACE", stdout_pipe[0]))
    {fprintf(stderr,"child died\n");return 1;}

  kill(child_pid, SIGKILL);
  kill(gdb_pid, SIGKILL);
}

The target program, looper is just:

#include <stdio.h>
#include <unistd.h>

int main(int argc, char *argv[])
{
  while(1)
  {
    printf(".");
    fflush(stdout);
    sleep(1);
  }
}

Example output is:

$ ./a.out
.GDB: GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-80.el7
GDB: Copyright (C) 2013 Free Software Foundation, Inc.
GDB: License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
GDB: This is free software: you are free to change and redistribute it.
GDB: There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
GDB: and "show warranty" for details.
GDB: This GDB was configured as "x86_64-redhat-linux-gnu".
GDB: For bug reporting instructions, please see:
GDB: <http://www.gnu.org/software/gdb/bugs/>.
GDB: Attaching to process 8057
GDB: Reading symbols from /home/<nope>/temp/remotecontrol/looper...(no debugging symbols found)...done.
GDB: Reading symbols from /lib64/libc.so.6...(no debugging symbols     found)...done.
GDB: Loaded symbols for /lib64/libc.so.6
GDB: Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
GDB: Loaded symbols for /lib64/ld-linux-x86-64.so.2
GDB: 0x00007f681b4f9480 in __nanosleep_nocancel () from /lib64/libc.so.6
GDB: Missing separate debuginfos, use: debuginfo-install glibc-2.17-    106.el7_2.4.x86_64
GDB: [PROMPT]
[SENDING "continue\n"]
....[SENDING "CTRL+C"]
POST SIGINT: Continuing.
POST SIGINT:
POST SIGINT: Program received signal SIGINT, Interrupt.
POST SIGINT: 0x00007f681b4f9480 in __nanosleep_nocancel () from /lib64/libc.so.6
POST SIGINT: [PROMPT]
[SENDING "where\n"]
TRACE: #0  0x00007f681b4f9480 in __nanosleep_nocancel () from /lib64/libc.so.6
TRACE: #1  0x00007f681b4f9334 in sleep () from /lib64/libc.so.6
TRACE: #2  0x0000000000400642 in main ()
TRACE: [PROMPT]

You can see from the .... that the target did continue running, even though GDB's output of "Continuing." doesn't show up until later when I read it's stdout pipe.