federikowsky
federikowsky

Reputation: 608

client C for API calls with curl_multi_*

I would like to create a C client that makes asynchronous API calls with lib curl and saves the responses, the calls are about a hundred at the same time. I have been looking for internet tutorials and examples for curl_multi_ * and curl_multi_socket with epoll for 4 days (I use linux) but they seem not to exist, and those few examples are not understandable to someone who is a beginner like me. Apparently I'm the only one interested in doing such a thing in C.

I also looked at the official documentation examples, but it uses a maximum of 2 connections at the same time and to do this declares two variables and calls curl_easy_init(), but the problem is that the requests made by the program are not a precise number so I cannot declare a number of variables a priori (even though it's not possible to declare 100 variables). I found out this example of curl_multi_socket with epoll is difficult to understand and replicate for my case without an explanation of how it works.

Is there anyone who can give me a code example on how to use curl_multi_ * for multiple simultaneous connections to start with? it would be much appreciated.


EDIT:

after hours of research, I finally found an example that might be fit, the problem is that it crashes often and for various reasons


#define NUM_URLS 64

typedef struct data {                   // 24 / 24 Bytes
    struct curl_slist * header;         
    char ** sub_match_json;             
    int nbr_sub_match;              
    int response_counter;               
} data_t;   

// list of the same URL repeated multiple times
// assume there are 64 url for example
static char *urls[] = {} 

void make_header(data_t * data) {
  //many curl_slist_append();
}

void init_data(data_t *data) {
  data->sub_match_json = (char **)malloc(sizeof(char *) * NUM_URLS);
  data->response_counter = 0;
  data->nbr_sub_match = NUM_URLS;
  make_header(data);
}

static size_t write_cb(void *response, size_t size, size_t nmemb, void *userp)
{
    size_t realsize = size * nmemb;
    data_t * data = (data_t *) userp;

    data->sub_match_json[data->response_counter] = malloc(realsize + 1);
    if(data->sub_match_json[data->response_counter] == NULL)
    {
        fprintf(stderr, "Memory allocation failed: %s\n", strerror(errno));
        return 0;  /* out of memory! */
    }
    memcpy(data->sub_match_json[data->response_counter], response, realsize);
    data->sub_match_json[data->response_counter][realsize] = 0;
    data->response_counter++;
    return realsize;
}
 
static void add_transfer(CURLM *cm, int i, data_t *data)
{
    CURL *curl = curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 1<<23);
    // curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
    // curl_easy_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L);
    // curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)data);
    curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, data->header);
    curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
    curl_easy_setopt(curl, CURLOPT_URL, urls[i]);
    curl_easy_setopt(curl, CURLOPT_PRIVATE, urls[i]);
    curl_multi_add_handle(cm, curl);
}
 
int main(void)
{
    CURLM *cm;
    CURLMsg *msg;
    data_t global_data;
    unsigned int transfers = 0;
    int msgs_left = -1;
    int still_alive = 1;
    
    curl_global_init(CURL_GLOBAL_ALL);
    cm = curl_multi_init();
    
    init_data(NULL, &global_data);  // my function

    /* Limit the amount of simultaneous connections curl should allow: */
    curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)MAX_PARALLEL);
    
    for(transfers = 0; transfers < MAX_PARALLEL; transfers++)
        add_transfer(cm, transfers, &global_data);
    
    do {
        curl_multi_perform(cm, &still_alive);
    
        while((msg = curl_multi_info_read(cm, &msgs_left))) {
        if(msg->msg == CURLMSG_DONE) {
            char *url;
            CURL *e = msg->easy_handle;
            curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &url);
            fprintf(stderr, "R: %d - %s <%s>\n",
                    msg->data.result, curl_easy_strerror(msg->data.result), url);
            curl_multi_remove_handle(cm, e);
            curl_easy_cleanup(e);
        }
        else {
            fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg);
        }
        if(transfers < global_data.nbr_sub_match)
            add_transfer(cm, transfers++, &global_data);
        }
        if(still_alive)
        curl_multi_wait(cm, NULL, 0, 1000, NULL);
    
    } while(still_alive || (transfers < NUM_URLS));
    
    curl_multi_cleanup(cm);
    curl_global_cleanup();
    

    while (global_data.response_counter-- >= 0) {
        printf("%s\n", global_data.sub_match_json[global_data.response_counter]);
    }
    return EXIT_SUCCESS;
}

Error:

api_calls(75984,0x100088580) malloc: Incorrect checksum for freed object 0x100604c30: probably modified after being freed.
Corrupt value: 0x600002931f10
api_calls(75984,0x100088580) malloc: *** set a breakpoint in malloc_error_break to debug

this is on curl_easy_cleanup(e);

Exception has occurred.
EXC_BAD_ACCESS (code=1, address=0x0)

otherwise, when no error occurs, in sub_match_json there are bytes and no char. Why this ?

Upvotes: 0

Views: 234

Answers (0)

Related Questions