passing array as return from function in multi threaded app

Question

i try to make code which is multithreaded and at certain step i have to retrieve array made on heap.
this is the code:
the following function will call another function called read_bi5_to_bin and it will pass to it unsigned char* initialized as nullptr data_bin_buffer

 int HTTPRequest::read_bi5_main(boost::filesystem::path p, ptime epoch)
{
    boost::unique_lock<boost::mutex> read_bi5_to_bin_lock(m_read_bi5_to_binMutex,boost::defer_lock);

    unsigned char *buffer;
    size_t buffer_size;

    int counter;

    size_t raw_size = 0;

    std::string filename_string = p.generic_string();
    path p2 = p;
    p2.replace_extension(".bin");
    std::string filename_string_2_bin =p2.generic_string() ;

    path p3 = p;
    p3.replace_extension(".csv");
    std::string filename_string_2_csv = p3.generic_string();

    const char *filename = filename_string.c_str();
    const char *filename_2_bin = filename_string_2_bin.c_str();

    const char *filename_2_csv = filename_string_2_csv.c_str();

    if (fs::exists(p) && fs::is_regular(p)) 
    {
        buffer_size = fs::file_size(p);
        buffer = new unsigned char[buffer_size];
    }
    else {
        //??5-17-2020 isolate multithreaded error
        read_bi5_to_bin_lock.lock();
        BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Error: couldn't access the data file. |"
            << filename << "|" << std::endl;
        read_bi5_to_bin_lock.unlock();

        return 2;
    }

    std::ifstream fin(filename, std::ifstream::binary);
    //fin.open(filename, std::ifstream::binary);
    fin.read(reinterpret_cast<char*>(buffer), buffer_size);
    fin.close();

    //5-11-2020 the next line will be commented and put in HTTPCLIent constructor
    //mHTTPRequest_Symbol_str= mHTTPRequest_HTTPClient_shared_pointer->Get_mHttpClient_HttpSymbolPrepareGet_shared_pointer()->mSymbol_strGet() ;
    std::size_t pos = mHTTPRequest_Symbol_str.find("JPY");// position of "h_ticks.bi5" in str
    double PV;
    if (pos != std::string::npos)
    {
        PV = PV_YEN_PAIR;
    }
    else
    {
        PV = PV_DOLLAR_PAIR;
    }
    //??5-17-2020 isolate multithreaded error
    read_bi5_to_bin_lock.lock();
    //5-20-2020
    //boost::shared_ptr<unsigned char> data_bin_buffer = boost::make_shared<unsigned char>() ;
    //n47::tick_data *data = n47::read_bi5_to_bin(
    //  buffer, buffer_size, epoch, PV, &raw_size, data_bin_buffer.get());
    unsigned char* data_bin_buffer = nullptr;
    n47::tick_data *data = n47::read_bi5_to_bin(
            buffer, buffer_size, epoch, PV, &raw_size, data_bin_buffer);
    //5-11-2020 here i will save binary file
    //boost::unique_lock<boost::mutex> read_bi5_to_bin_lock(m_read_bi5_to_binMutex);
    std::string file_name_path_string=output_compressed_file_2(data_bin_buffer, raw_size, filename_2_bin);
    read_bi5_to_bin_lock.unlock();

    path file_name_path_2{ file_name_path_string }; 
    buffer_size = 0;    
    if (fs::exists(file_name_path_2) && fs::is_regular(file_name_path_2)) 
    {
        //??5-17-2020 isolate multithreaded error
        read_bi5_to_bin_lock.lock();

        BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << boost::this_thread::get_id() <<"\t we can access the data .bin file. |"
            << filename_2_bin << "| with size ="<< fs::file_size(file_name_path_2) << std::endl;
        read_bi5_to_bin_lock.unlock();

    }
    else {
        //??5-17-2020 isolate multithreaded error
        read_bi5_to_bin_lock.lock();

        BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Error: couldn't access the data .bin file. |"
            << filename_2_bin << "|" << std::endl;
        read_bi5_to_bin_lock.unlock();

        return 2;
    }

    n47::tick_data_iterator iter;

    //5-11-2020 here i will save file.csv from data which is pointer to vector to pointers to ticks

    if (data == 0) {
        //??5-17-2020 isolate multithreaded error
        read_bi5_to_bin_lock.lock();

        BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Failure: Failed to load the data!" << std::endl;
        read_bi5_to_bin_lock.unlock();

        //5-14-2020 file is empty
        //return 0;
    }
    //5-15-2020 take care that without else ,error happens with empty files because data is pointer to vector of pointers to ticks .so when data is made inside read_bi5 ,it is made as null pointer and later it is assigned to vector if file has ticks.if file does not have ticks ,then it is just returned as null pointer .so when dereferencing null pointer we got error
    else if (data->size() != (raw_size / n47::ROW_SIZE)) {
        //??5-17-2020 isolate multithreaded error
        read_bi5_to_bin_lock.lock();

        BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Failure: Loaded " << data->size()
            << " ticks but file size indicates we should have loaded "
            << (raw_size / n47::ROW_SIZE) << std::endl;
        read_bi5_to_bin_lock.unlock();

        //5-14-2020 file is empty
        //return 0;
    }

    //??5-17-2020 isolate multithreaded error
    read_bi5_to_bin_lock.lock();

    BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "time, bid, bid_vol, ask, ask_vol" << std::endl;
    read_bi5_to_bin_lock.unlock();

    counter = 0;
    std::ofstream out_csv(filename_string_2_csv);
    if (data == 0)
    {

    }
    else if (data != 0)
    {
        ////read_bi5_to_bin_lock.lock();

        for (iter = data->begin(); iter != data->end(); iter++) {
            //5-11-2020 here i will save file.csv from data which is pointer to vector to pointers to ticks>>>>>>>here i should open file stream for output and save data to it
            out_csv << ((*iter)->epoch + (*iter)->td) << ", "
                << (*iter)->bid << ", " << (*iter)->bidv << ", "
                << (*iter)->ask << ", " << (*iter)->askv << std::endl;
            //??5-17-2020 isolate multithreaded error
            read_bi5_to_bin_lock.lock();

            BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) <<
                boost::this_thread::get_id() << "\t"<<((*iter)->epoch + (*iter)->td) << ", "
                << (*iter)->bid << ", " << (*iter)->bidv << ", "
                << (*iter)->ask << ", " << (*iter)->askv << std::endl;
            read_bi5_to_bin_lock.unlock();

            counter++;
        }
        ////read_bi5_to_bin_lock.unlock();

    }
    out_csv.close();
    //5-13-2020

    //??5-17-2020 isolate multithreaded error
    read_bi5_to_bin_lock.lock();

    BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << ".end." << std::endl << std::endl
        << "From " << raw_size << " bytes we read " << counter
        << " records." << std::endl
        << raw_size << " / " << n47::ROW_SIZE << " = "
        << (raw_size / n47::ROW_SIZE) << std::endl;
    read_bi5_to_bin_lock.unlock();


    delete data;
    delete[] buffer;    
    delete [] data_bin_buffer;
    return 0;
}

then inside read_bi5_to_bin function ,there will be a call to another function n47::lzma::decompress . this is the code of read_bi5_to_bin:

tick_data* read_bi5_to_bin(
    unsigned char *lzma_buffer, size_t lzma_buffer_size, pt::ptime epoch,
    float point_value, size_t *bytes_read, unsigned char* buffer_decompressed) {
    tick_data *result = 0;

    // decompress
    int status;
    buffer_decompressed = n47::lzma::decompress(lzma_buffer,
        lzma_buffer_size, &status, bytes_read);

    if (status != N47_E_OK) 
    {
        bytes_read = 0;
    }
    else {
        // convert to tick data (with read_bin).
        result = read_bin(buffer_decompressed, *bytes_read, epoch, point_value);
        //delete[] buffer;
    }

    return result;
}

then inside n47::lzma::decompress there will be making an array on heap called outBuffer .this buffer i need to retrieve at read_bi5_main in data_bin_buffer
this is code for n47::lzma::decompress

unsigned char *decompress(
        unsigned char *inBuffer, size_t inSize, int *status, size_t *outSize) {
    unsigned char *outBuffer = 0;
    elzma_file_format format = ELZMA_lzma;

    elzma_decompress_handle handle;
    handle = elzma_decompress_alloc();

    if (handle == 0) {
        *status = -1;
    } else {
        // decompression...
        datastream ds(inBuffer, inSize);
        *status = elzma_decompress_run(
            handle,
            inputCallback, static_cast<void*>(&ds),
            outputCallback, static_cast <void*>(&ds),
            format);

        if (*status == ELZMA_E_OK) {
            *outSize = ds.outData.size();
            outBuffer = new unsigned char[ ds.outData.size() ];
            std::copy(ds.outData.begin(), ds.outData.end(), outBuffer);
        }
        elzma_decompress_free(&handle);
    }

    return outBuffer;
}

when i run this it gives error.a very helpful one here told me that it is the data_bin_buffer which is the problem because it is just one byte.and advised me to avoid shared pointer.when i convert it to normal pointer it gives another error if initialzed it to nullptr.should i not initialize the pointer???

Answer 1

I found the problem in my code.
it is the way i pass pointer to unsigned char to read_bi5_to_bin .
at first I thought passing the pointer variable is correct"as if i am passing lvalue reference" but i was so much wrong.reference is different than pointer.
pointer variable is lvalue variable of type pointer.so it is copied into function making new lvalue variable in function .the new variable has same value of pointer passed.this value is the address to first unsigned char in array.
but what i really need is to pass pointer to pointer to unsigned char.then pointer to pointer will be copied into function.the value of copied variable will be address of pointer to unsigned char.this is what i need retrieve from function.
I used & operator to get pointer to pointer .then I read very useful comment suggesting using ** so i changed the code to use **.voila,it works
this is the modified code:

read_bi5_to_bin_lock2.lock();
    //5-20-2020
    //boost::shared_ptr<unsigned char> data_bin_buffer = boost::make_shared<unsigned char>() ;
    //n47::tick_data *data = n47::read_bi5_to_bin(
    //  buffer, buffer_size, epoch, PV, &raw_size, data_bin_buffer.get());
    unsigned char *data_bin_buffer = 0 ;
    n47::tick_data *data = n47::read_bi5_to_bin(
            buffer, buffer_size, epoch, PV, &raw_size, &data_bin_buffer);
    //5-11-2020 here i will save binary file
    //boost::unique_lock<boost::mutex> read_bi5_to_bin_lock(m_read_bi5_to_binMutex);
    std::string file_name_path_string=output_compressed_file_2(&data_bin_buffer, raw_size, filename_2_bin);
    read_bi5_to_bin_lock2.unlock();

and this is the modified code in read_bi5_to_bin:

tick_data* read_bi5_to_bin(
    unsigned char *lzma_buffer, size_t lzma_buffer_size, pt::ptime epoch,
    float point_value, size_t *bytes_read, unsigned char** buffer_decompressed) {
    tick_data *result = 0;

    // decompress
    int status;
    *buffer_decompressed = n47::lzma::decompress(lzma_buffer,
        lzma_buffer_size, &status, bytes_read);

thanks alot to DavidSchwartz for his very helpful comment.

Answer 2

Your read_bi5_to_bin code is broken.

tick_data* read_bi5_to_bin(
    unsigned char *lzma_buffer, size_t lzma_buffer_size, pt::ptime epoch,
    float point_value, size_t *bytes_read, unsigned char* buffer_decompressed) {
    tick_data *result = 0;

The code receives a pointer called buffer_decompressed.

    // decompress
    int status;
    buffer_decompressed = n47::lzma::decompress(lzma_buffer,
        lzma_buffer_size, &status, bytes_read);

Then it throws away the value it received and gets some other value.

    if (status != N47_E_OK) 
    {
        bytes_read = 0;
    }
    else {
        // convert to tick data (with read_bin).
        result = read_bin(buffer_decompressed, *bytes_read, epoch, point_value);
        //delete[] buffer;
    }

    return result;
}

And it never returns or frees the value it stored in buffer_decompressed.

Ooops.