Confusion related to sigwait in multiprocess system

Question

I am having difficulty in understanding IPC in multiprocess system. I have this system where there are three child processes that send two types of signals to their process group. There are four types of signal handling processes responsible for a particular type of signal.

There is this monitoring process which waits for both the signals and then processes accordingly. When I run this program for a while, the monitoring process doesn't seem to pick up the signal as well as the signal handling process. I could see in the log that the signal is only being generated but not handled at all.

My code is given below

#include <cstdlib>
#include <iostream>
#include <iomanip>
#include <unistd.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <signal.h>
#include <unistd.h>
#include <fcntl.h>
#include <cstdio>
#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>

using namespace std;

double timestamp() {
  struct timeval tp;
  gettimeofday(&tp, NULL);
  return (double)tp.tv_sec + tp.tv_usec / 1000000.;
}

double getinterval() { 
  srand(time(NULL));
  int r = rand()%10 + 1;
  double s = (double)r/100;
}

int count;
int count_1;
int count_2;
double time_1[10];
double time_2[10];


pid_t senders[1];
pid_t handlers[4];
pid_t reporter;

void catcher(int sig) {
  printf("Signal catcher called for %d",sig);
}

int main(int argc, char *argv[]) {

  void signal_catcher_int(int);

  pid_t pid,w;
  int status;

  if(signal(SIGUSR1, SIG_IGN) == SIG_ERR) {
    perror("1");
    return 1;
  }

  if(signal(SIGUSR2 ,SIG_IGN) == SIG_ERR) {
    perror("2");
    return 2;
  }

  if(signal(SIGINT,signal_catcher_int) == SIG_ERR) {
    perror("3");
    return 2;
  }

  //Registering the signal handler
  for(int i=0; i<4; i++) {
    if((pid = fork()) == 0) {
      cout << i << endl;
      //struct sigaction sigact;
      sigset_t sigset;
      int sig;
      int result = 0;

      sigemptyset(&sigset);

      if(i%2 == 0) {

        if(signal(SIGUSR2, SIG_IGN) == SIG_ERR) {
          perror("2");
          return 2;
        }

        sigaddset(&sigset, SIGUSR1);
        sigprocmask(SIG_BLOCK, &sigset, NULL);
      } else {
            if(signal(SIGUSR1, SIG_IGN) == SIG_ERR) {
          perror("2");
          return 2;
        }

        sigaddset(&sigset, SIGUSR2);
        sigprocmask(SIG_BLOCK, &sigset, NULL);
      }

      while(true) {
        int result = sigwait(&sigset, &sig);
        if(result == 0) {
          cout << "The caught signal is " << sig << endl;
        }
      }
      exit(0);

    } else {
      cout << "Registerd the handler " << pid << endl;
      handlers[i] = pid;
    }
  }

  //Registering the monitoring process
  if((pid = fork()) == 0) {
    sigset_t sigset;
    int sig;
    int result = 0;

    sigemptyset(&sigset);
    sigaddset(&sigset, SIGUSR1);
    sigaddset(&sigset, SIGUSR2);

    sigprocmask(SIG_BLOCK, &sigset, NULL);

    while(true) {
      int result = sigwait(&sigset, &sig);
      if(result == 0) {
        cout << "The monitored signal is " << sig << endl;
      } else {
        cout << "error" << endl;
      } 
    }

  } else {
    reporter = pid;
  }


  sleep(3);
  //Registering the signal generator
  for(int i=0; i<1; i++) {
    if((pid = fork()) == 0) {

      if(signal(SIGUSR1, SIG_IGN) == SIG_ERR) {
        perror("1");
        return 1;
      }

      if(signal(SIGUSR2, SIG_IGN) == SIG_ERR) {
        perror("2");
        return 2;
      }
      srand(time(0));
      while(true) {
        volatile int signal_id = rand()%2 + 1;
        cout << "Generating the signal " << signal_id << endl;
        if(signal_id == 1) {
          killpg(getpgid(getpid()), SIGUSR1);
        } else {
              killpg(getpgid(getpid()), SIGUSR2);
        }     
        int r = rand()%10 + 1;
        double s = (double)r/100;  
        sleep(s);
      }

      exit(0); 
    } else {
      cout << "Registered the sender " << pid << endl;
      senders[i] = pid;

    } 

  }


  while(w = wait(&status)) {
    cout << "Wait on PID " << w << endl;
  }

}



void signal_catcher_int(int the_sig) {
  //cout << "Handling the Ctrl C signal " << endl;
  for(int i=0; i<1; i++) {
    kill(senders[i],SIGKILL);
  }

  for(int i=0; i<4; i++) {
    kill(handlers[i],SIGKILL);
  }

  kill(reporter,SIGKILL);

  exit(3);
}

Any suggestions?

Here is a sample of the output as well

In the beginning

Registerd the handler 9544
Registerd the handler 9545
1
Registerd the handler 9546
Registerd the handler 9547
2
3
0
Registered the sender 9550
Generating the signal 1
The caught signal is 10
The monitored signal is 10
The caught signal is 10
Generating the signal 1
The caught signal is 10
The monitored signal is 10
The caught signal is 10
Generating the signal 1
The caught signal is 10
The monitored signal is 10
The caught signal is 10
Generating the signal 1
The caught signal is 10
The monitored signal is 10
The caught signal is 10
Generating the signal 2
The caught signal is 12
The caught signal is 12
The monitored signal is 12
Generating the signal 2
Generating the signal 2
The caught signal is 12
The caught signal is 12
Generating the signal 1
The caught signal is 12
The monitored signal is 10
The monitored signal is 12
Generating the signal 1
Generating the signal 2
The caught signal is 12
Generating the signal 1
Generating the signal 2
10
The monitored signal is 10
The caught signal is 12
Generating the signal 1
The caught signal is 12
The monitored signal is GenThe caught signal is TheThe caught signal is 10
Generating the signal 2

Later on

The monitored signal is GenThe monitored signal is 10
Generating the signal 1
Generating the signal 2
The caught signal is 10
The caught signal is 10
The caught signal is 10
The caught signal is 12
Generating the signal 1
Generating the signal 2
Generating the signal 1
Generating the signal 1
Generating the signal 2
Generating the signal 2
Generating the signal 2
Generating the signal 2
Generating the signal 2
Generating the signal 1
The caught signal is 12
The caught signal is 10
The caught signal is 10
Generating the signal 2
Generating the signal 1
Generating the signal 1
Generating the signal 2
Generating the signal 1
Generating the signal 2
Generating the signal 2
Generating the signal 2
Generating the signal 1
Generating the signal 2
Generating the signal 1
Generating the signal 2
Generating the signal 2
The caught signal is 10
Generating the signal 2
Generating the signal 1
Generating the signal 1

As you can see initially, the signal was generated and handled both by my signal handlers and monitoring processes. But later on the signal was generated a lot, but it was not quite processes in the same magnitude as before. Further I could see very less signal processing by the monitoring process

Can anyone please provide some insights. What's going on?

Answer 1

If multiple signals of the same type are pending, Linux by default delivers only one such signal. This is inline with the sigwait documentation:

If prior to the call to sigwait() there are multiple pending instances of a single signal number, it is implementation-dependent whether upon successful return there are any remaining pending signals for that signal number.

So the output of your program depends on the scheduler, if kill is called multiple times and scheduler does not awakes monitoring processes in a mean time, signals of the same types are collapsed into one.

Linux allows to change the default behavior.