Boost.Interprocess crashes when destroy an object in a mpi process

Question

I create a shm in parent process and then start mpiexec by system.

In these mpi booted processes, the shm is opened. It can find objects by find<Type>("name"), but when it tries destroy<Type>(name), there is an error.

void Foo::DumpInitialData()
{
    using namespace boost::interprocess;
    shared_memory_object::remove("MySHM");
    typedef allocator<char, managed_shared_memory::segment_manager>
        ShmCharAllocator;
    typedef std::basic_string<char, std::char_traits<char>, ShmCharAllocator>
        ShmStr;

    string try_str;
    {
        ostringstream try_ss;
        Foo split(*this, ...);
        BinarySerialization ser(split);
        ser.Dump(try_ss);
        try_str = try_ss.str();
    }
    size_t try_size = sizeof(ShmStr) + 512
        + (try_str.capacity() + 1) * sizeof(char);

    managed_shared_memory segment(create_only, "MySHM", 
        try_size * (split_size + 1));
    ShmCharAllocator alloc_chars(segment.get_segment_manager());
    segment.construct<ShmStr>(
            ("Mpi" + to_string(0)).c_str(
    ))(try_str, alloc_chars); 

    try {
        for (int i = 1; i < split_size; ++i) {
            stringstream ss;
            Foo split(*this, ...);
            BinarySerialization ser(split);
            ser.Dump(ss);

            segment.construct<ShmStr>(("Mpi" + to_string(i)).c_str())
                (ss.str(), alloc_chars);
        }
    } catch (interprocess_exception &e) {
        throw ...Exception(
            "No enough shared memeroy to passing split data."
        );
    }
}

...

void Foo::PumpMpiInitialData(int i)
{
    auto& world = mpi->world();
    vector<string> send(world.size());
    decltype(send)::value_type recv;
    if (i == 0) {
        using namespace boost::interprocess;
        typedef allocator<char, managed_shared_memory::segment_manager>
            ShmCharAllocator;
        typedef std::basic_string<
            char, std::char_traits<char>, ShmCharAllocator
        > ShmStr;
        {
            managed_shared_memory segment(
                open_only, "MySHM"
            );

            for_each(send | ba::indexed(), [&](const auto& s_it) {
                auto name = "Mpi" + to_string(s_it.index());
                const char* c_name = name.c_str();
                ShmStr* data = segment.find<ShmStr>(c_name).first;
                s_it.value() = string(*data);
                // This will failed.
                bool succ = segment.destroy<ShmStr>(c_name);
            });
        }
        shared_memory_object::remove("MySHM");
    }
    bm::scatter(world, send, recv, 0);

    BinarySerialization ser(*this);
    istringstream ss(recv);
    ser.Pump(ss);
}

The error is happened with the following call stacks:

>   ....exe!std::_Container_base12::_Orphan_all_unlocked_v3() Line 1243 C++
    ....exe!std::_Container_base12::_Orphan_all_locked_v3() Line 1098   C++
    ....exe!std::_Container_base12::_Orphan_all() Line 1260 C++
    ....exe!std::basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>::_Tidy_deallocate() Line 4618 C++
    ....exe!std::basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>::~basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>() Line 3005    C++
    ....exe!std::basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>::`scalar deleting destructor'(unsigned int)   C++
    ....exe!boost::interprocess::ipcdetail::placement_destroy<std::basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>>::destroy_n(void * mem, unsigned __int64 num, unsigned __int64 & destroyed) Line 61 C++
    ....exe!boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>::priv_generic_named_destroy<char>(const char * name, boost::interprocess::iset_index<boost::interprocess::ipcdetail::index_config<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>>> & index, boost::interprocess::ipcdetail::in_place_interface & table, boost::interprocess::ipcdetail::bool_<1> is_intrusive_index) Line 992    C++
    ....exe!boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>::destroy<std::basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>>(boost::interprocess::ipcdetail::char_ptr_holder<char> name) Line 553  C++
    ....exe!boost::interprocess::ipcdetail::basic_managed_memory_impl<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index,8>::destroy<std::basic_string<char,std::char_traits<char>,boost::interprocess::allocator<char,boost::interprocess::segment_manager<char,boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family,boost::interprocess::offset_ptr<void,__int64,unsigned __int64,0>,0>,boost::interprocess::iset_index>>>>(const char * name) Line 568    C++
    ....exe!<lambda_da719d1c74e37fe397eecaa291110c18>::operator()<boost::range::index_value<std::string &,__int64>>(const boost::range::index_value<std::string &,__int64> & s_it) Line 222 C++
...

A similar situation without mpi can work correctly as following.

int main(int argc, char *argv[])
{
   using namespace boost::interprocess;
   typedef allocator<char, managed_shared_memory::segment_manager>
        ShmCharAllocator;
   typedef std::basic_string<char, std::char_traits<char>, ShmCharAllocator>
        ShmStr;

   if(argc == 1){  //Parent process
      //Remove shared memory on construction and destruction
      struct shm_remove
      {
         shm_remove() { shared_memory_object::remove("MySharedMemory"); }
         ~shm_remove(){ shared_memory_object::remove("MySharedMemory"); }
      } remover;

      {

      //Construct managed shared memory
      managed_shared_memory segment(create_only, "MySharedMemory", 65536);

      const ShmCharAllocator alloc_inst(segment.get_segment_manager());
      //Create an object of MyType initialized to {0.0, 0}
      segment.construct<ShmStr>("MyType0")(string("abc"), alloc_inst);

      }

      //Launch child process
      std::string s(argv[0]); s += " child ";
      if(0 != std::system(s.c_str()))
         return 1;


   } else {
       {
           using namespace std::chrono_literals;
           std::this_thread::sleep_for(10s);
       }


      //Open managed shared memory
      managed_shared_memory segment(open_only, "MySharedMemory");

      const string str = "MyType0";
      auto* res = segment.find<ShmStr>(str.c_str()).first;
      string a = string(*res);
      //Length should be 1
      //if(res.second != 1)
      //    return 1;

      segment.destroy<ShmStr>(str.c_str());
   }
   return 0;
}

Answer 1

It's hard to see what all the magic numbers mean. It's a big red flag that you're dealing with sizeof(ShmStr) because it's not a trivial type.

It's also "weird" that you're using a managed segment, but try to do all the memory management manually. Why not store an actual queue inside the segment?

Note that you're seemingly emulating the array support using named allocations, but array support is built-in to segment managers.

Finally, I don't use MPI but the way I remember it has builtin support for boost serialization, which means you don't need to copy all data into a string stream, then into a string, that into a shared memory object etc. That's not optimal.

Next up, you're using the shared memory segment from multiple processes (MPI processes) but you're not locking anything. For example, destroy<> would modify the shared object that other processes are still reading. Also, if you are destroying from all processes then you will be destroying things multiple times (Undefined Behaviour).

Finally, if you're fine with

    bip::shared_memory_object::remove("MySHM");

in the end, don't bother destroying individual split elements?