C forking stops after 5 children

Question

I am working on a C program which makes a connection to a database, grabs a list of devices, then forks the requests and creates an SSH connection to that device. The issue I am having is that, the query, which has 700 results, is always starting from the beginning after it hits 5 forks.

Essentially, I've looked into pthread and glibc to handle threading, but some of the examples I found did not work as desired, or added too much complexity.

The problem I am having is, it will stop at 5 children, and stop, instead of finishing the rest of the 700 devices.

Example code:

#include <libssh2.h>

#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/select.h>
#include <unistd.h>

#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif

#include <sys/time.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <stdlib.h>
#include <fcntl.h>
#include <errno.h>
#include <stdio.h>
#include <ctype.h>
#include <limits.h>

#include <mysql.h>


/********************************
 *
 * gcc -o confmgr cfgmgr.c -Wall -lpthread -lz -lm -lrt -ldl -lssh2 $(mysql_config --cflags) $(mysql_config --libs) -std=gnu99
 *
 ********************************/
static int waitsocket(int socket_fd, LIBSSH2_SESSION *session)
{
    struct timeval timeout;
    int rc;
    fd_set fd;
    fd_set *writefd = NULL;
    fd_set *readfd = NULL;
    int dir;

    timeout.tv_sec = 10;
    timeout.tv_usec = 0;

    FD_ZERO(&fd);

    FD_SET(socket_fd, &fd);

    /* now make sure we wait in the correct direction */ 
    dir = libssh2_session_block_directions(session);


    if(dir & LIBSSH2_SESSION_BLOCK_INBOUND)
        readfd = &fd;

    if(dir & LIBSSH2_SESSION_BLOCK_OUTBOUND)
        writefd = &fd;

    rc = select(socket_fd + 1, readfd, writefd, NULL, &timeout);

    return rc;
}

int *connect_to_device(MYSQL_RES** args){

    printf("%s", args[2]);

    const char *hostname = "1.1.1.1";
    const char *commandline = "command_to_run ";
    const char *username    = "static_user";
    const char *password    = "static_pass";

    unsigned long hostaddr;

    int sock;

    struct sockaddr_in sin;

    const char *fingerprint;
    LIBSSH2_SESSION *session;
    LIBSSH2_CHANNEL *channel;
    int rc;
    int exitcode;
    char *exitsignal=(char *)"none";
    int bytecount = 0;
    size_t len;
    LIBSSH2_KNOWNHOSTS *nh;
    int type;



    rc = libssh2_init (0);

    if (rc != 0) {
        fprintf (stderr, "libssh2 initialization failed (%d)\n", rc);
        return 1;
    }

    hostaddr = inet_addr(hostname);

    /* Ultra basic "connect to port 22 on localhost"
     * Your code is responsible for creating the socket establishing the
     * connection
     */ 
    sock = socket(AF_INET, SOCK_STREAM, 0);

    sin.sin_family = AF_INET;
    sin.sin_port = htons(22);
    sin.sin_addr.s_addr = hostaddr;
    if (connect(sock, (struct sockaddr*)(&sin),
                sizeof(struct sockaddr_in)) != 0) {
        fprintf(stderr, "failed to connect!\n");
        return -1;
    }

    /* Create a session instance */ 
    session = libssh2_session_init();

    if (!session)
        return -1;

    /* tell libssh2 we want it all done non-blocking */ 
    libssh2_session_set_blocking(session, 0);


    /* ... start it up. This will trade welcome banners, exchange keys,
     * and setup crypto, compression, and MAC layers
     */ 
    while ((rc = libssh2_session_handshake(session, sock)) ==

           LIBSSH2_ERROR_EAGAIN);
    if (rc) {
        fprintf(stderr, "Failure establishing SSH session: %d\n", rc);
        return -1;
    }




     /* We could authenticate via password */ 
     while ((rc = libssh2_userauth_password(session, username, password)) == LIBSSH2_ERROR_EAGAIN);
     if (rc) {

            fprintf(stderr, "Authentication by password failed.\n");

            goto shutdown;
     }


    libssh2_trace(session, LIBSSH2_TRACE_TRANS | LIBSSH2_TRACE_KEX | LIBSSH2_TRACE_AUTH | LIBSSH2_TRACE_CONN | LIBSSH2_TRACE_SCP | LIBSSH2_TRACE_SFTP | LIBSSH2_TRACE_ERROR | LIBSSH2_TRACE_PUBLICKEY );

    /* Exec non-blocking on the remove host */
    while( (channel = libssh2_channel_open_session(session)) == NULL &&
           libssh2_session_last_error(session,NULL,NULL,0) == LIBSSH2_ERROR_EAGAIN )
    {
        waitsocket(sock, session);
    }
    if( channel == NULL )
    {
        fprintf(stderr,"Error\n");
        exit( 1 );
    }
    while( (rc = libssh2_channel_exec(channel, commandline)) == LIBSSH2_ERROR_EAGAIN )
    {
        waitsocket(sock, session);
    }

    if( rc != 0 )
    {
        fprintf(stderr,"Error\n");
        exit( 1 );
    }

    for( ;; )
    {
        // loop until we block
        int rc;
        do
        {
            char buffer[0x4000];

            /* strange thing */
            sleep( 1 );

            rc = libssh2_channel_read( channel, buffer, sizeof(buffer) );
            if( rc > 0 )
            {
                int i;
                for( i=0; i < rc; ++i )
                    putchar( buffer[i] );
            }
        }
        while( rc > 0 );
        // this is due to blocking that would occur otherwise so we loop on this condition
        if( rc == LIBSSH2_ERROR_EAGAIN )
        {
            waitsocket(sock, session);
        }
        else if( rc == 0 )
            break;
    }


    while( (rc = libssh2_channel_close(channel)) == LIBSSH2_ERROR_EAGAIN )
        ;
    if( rc == 0 )
    {
        //does-not-work if( libssh2_channel_wait_closed(channel) == 0 )
        exitcode = libssh2_channel_get_exit_status( channel );
    }
    printf("\n%d\n", 221 );

    libssh2_channel_free(channel);
    channel = NULL;


    /***********************/

    shutdown:

        libssh2_session_disconnect(session, "Normal Shutdown, Thank you for playing");
        libssh2_session_free(session);


    close(sock);

    fprintf(stderr, "\n----------------------\nScript Finished\n\n");

    libssh2_exit();

    return 7;
}

/********************************
 *
 *
 *
 *
 ********************************/
int main(int argc, char *argv[]){

   pid_t childPID;
   int children = 0;

   MYSQL *conn;
   MYSQL_RES *res;
   MYSQL_ROW row;

   char *mySQLserver = "localhost";
   char *mySQLuser = "root";
   char *mySQLpassword = ""; /* set me first */
   char *mySQLdatabase = "Devices";


   conn = mysql_init(NULL);

   /* Connect to database */
   if (!mysql_real_connect(conn, mySQLserver,
         mySQLuser, mySQLpassword, mySQLdatabase, 0, NULL, 0)) {
      fprintf(stderr, "%s\n", mysql_error(conn));
      exit(1);
   }
   /* send SQL query */
   if (mysql_query(conn, "SELECT Hostname,Descr,IP,Username,Password FROM All_Active_Devices")) {
      fprintf(stderr, "%s\n", mysql_error(conn));
      exit(1);
   }

   res = mysql_use_result(conn);
   /* output table name */
   printf("MySQL Tables in mysql database:\n");
   while ((row = mysql_fetch_row(res)) != NULL){


      printf("%s \n", row[0]);

      children++;  // Last fork() was successful


        while (children >= 5)
        {
            int status;
            // Wait for one child to exit
            if (wait(&status) == 7)
            {
                children--;
            }
        }

      childPID = fork ();
      if (childPID < 0) {
                    printf("Fork Error \n");

      } else if (childPID == 0) {

        printf("\tCreating Fork for %s: pid %d \n", row[0], childPID);
        connect_to_device ( &row );

      }
      else{

        printf("\tDid not create Fork for %s \n", row[0]);

      }


   }

   /* close connection */
   mysql_free_result(res);
   mysql_close(conn);



    return 0;
}

Answer 1

What happens after connect_to_device returns? It looks like that thread will also start in on the while loop as I don't see that child process exiting, sitting you in the while children >=5 loop. The 5 in your condition and 5 threads isn't a coincidence.

Some output from a run would be helpful, also paring down the code.

Try to get your scaffolding working, without the ssh code. Just getting the processes to stop and start shouldn't need ssh. Then add in the application logic once you're sure the underlying support works.

Answer 2

Your child processes are exitting - in particular not with exit status 7 - you have a return in your connect_to_device function, but that is ignored, and each child starts going round the loop, creating more children.

You probably want: return connect_to_device ( &row ); instead.

wait() returns the child PID that died, not its status - which is in WEXITSTATUS(status).

Answer 3

Perhaps ServerAliveCountMax is getting in the way:

EDIT FROM NETCODER BELOW

ServerAliveCountMax
Sets the number of server alive messages (see below) which may be sent without ssh(1) receiving any messages back from the server. If this threshold is reached while server alive messages are being sent, ssh will disconnect from the server, terminating the session. It is important to note that the use of server alive messages is very different from TCPKeepAlive (below). The server alive messages are sent through the encrypted channel and therefore will not be spoofable. The TCP keepalive option enabled by TCPKeepAlive is spoofable. The server alive mechanism is valuable when the client or server depend on knowing when a connection has become inactive.

The default value is 3. If, for example, ServerAliveInterval (see below) is set to 15 and ServerAliveCountMax is left at the default, if the server becomes unresponsive, ssh will disconnect after approximately 45 seconds. This option applies to protocol version 2 only.

See the man page for ssh_config (man 5 ssh_config) for details.