c++ - How to send a set object in MPI_Send

Question

I searched to send a set object and the closest I found was with vector (it's different and don't work with set).

How can I send a set object in MPI_Send? (without using boost library) Anyone can put a simple example?

Answer 1

Whether you have to write a complex data structure to a file or over the network in MPI, the issues are the same; you have to extract the data into "Plain Old Data" (POD), save it, and then output it, and likewise be able to unpack the saved data into the same sort of structure. In general, this is called serialization.

For any given structure you can always write your own routines for doing this, but in C++, there's a framework in Boost called the Boost Serialization Library for doing this; it's a bit heavyweight for this example but it will work for most (all?) STL containers and there are hooks for adding support for your own classes.

The main trick with using Boost for this is that the Boost libraries (and all the examples) make it very easy to write the data to a file, but here you want to keep it in memory and send/receive it over the network; that means jumping through a couple more hoops to make sure the serialization is into an array you can access. This SO answer is very helpful in this regard.

So a complete working example looks like this:

#include <mpi.h>
#include <set>
#include <string>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/serialization/set.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/device/back_inserter.hpp>

int main(int argc,char** argv) {

    int size, rank;

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    if (size < 2) {
        if (rank == 0)
            std::cerr << "Require at least 2 tasks" << std::endl;
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

   const int lentag=0;
   const int datatag=1;
   if (rank == 0) {
        int nums[] = {1,4,9,16};
        std::set<int> send_set(nums, nums+4);

        std::cout << "Rank " << rank << " sending set: ";
        for (std::set<int>::iterator i=send_set.begin(); i!=send_set.end(); i++)
            std::cout << *i << " ";
        std::cout << std::endl;

        // We're going to serialize into a std::string of bytes, and then send this
        std::string serial_str;
        boost::iostreams::back_insert_device<std::string> inserter(serial_str);
        boost::iostreams::stream<boost::iostreams::back_insert_device<std::string> > s(inserter);
        boost::archive::binary_oarchive send_ar(s);

        send_ar << send_set;
        s.flush();
        int len = serial_str.size();

        // Send length, then data
        MPI_Send( &len, 1, MPI_INT, 1, lentag, MPI_COMM_WORLD );
        MPI_Send( (void *)serial_str.data(), len, MPI_BYTE, 1, datatag, MPI_COMM_WORLD );
    } else if (rank == 1) {
        int len;
        MPI_Recv( &len, 1, MPI_INT, 0, lentag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);

        char data[len+1];
        MPI_Recv( data, len, MPI_BYTE, 0, datatag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
        data[len] = '';

        boost::iostreams::basic_array_source<char> device(data, len);
        boost::iostreams::stream<boost::iostreams::basic_array_source<char> > s(device);
        boost::archive::binary_iarchive recv_ar(s);

        std::set<int> recv_set;
        recv_ar >> recv_set;

        std::cout << "Rank " << rank << " got set: ";
        for (std::set<int>::iterator i=recv_set.begin(); i!=recv_set.end(); i++)
            std::cout << *i << " ";
        std::cout << std::endl;
    }

    MPI_Finalize();
    return 0;
}

Running gives:

$ mpic++ mpi-set.cxx -o mpiset -lboost_serialization
$ mpirun -np 2 ./mpiset
Rank 0 sending set: 1 4 9 16
Rank 1 got set: 1 4 9 16

If you really don't want to use Boost, since you can't actually see directly into the set data structure, there's not much alternative but to extract the data into an array or vector, and send the data that way:

#include <mpi.h>
#include <set>
#include <vector>

int main(int argc,char** argv) {

    int size, rank;

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    if (size < 2) {
        if (rank == 0)
            std::cerr << "Require at least 2 tasks" << std::endl;
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

   const int lentag=0;
   const int datatag=1;
   if (rank == 0) {
        int nums[] = {1,4,9,16};
        std::set<int> send_set(nums, nums+4);

        std::cout << "Rank " << rank << " sending set: ";
        for (std::set<int>::iterator i=send_set.begin(); i!=send_set.end(); i++)
            std::cout << *i << " ";
        std::cout << std::endl;

        // Send length, then data
        int len = send_set.size();
        MPI_Send( &len, 1, MPI_INT, 1, lentag, MPI_COMM_WORLD );

        std::vector<int> send_vec;
        for (std::set<int>::iterator i=send_set.begin(); i!=send_set.end(); i++) 
            send_vec.push_back(*i);

        MPI_Send( send_vec.data(), len, MPI_INT, 1, datatag, MPI_COMM_WORLD );

    } else if (rank == 1) {

        int len;
        MPI_Recv( &len, 1, MPI_INT, 0, lentag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);

        int recv_data[len];
        MPI_Recv( recv_data, len, MPI_INT, 0, datatag, MPI_COMM_WORLD, MPI_STATUS_IGNORE);

        std::set<int> recv_set;
        for (int i=0; i<len; i++) 
            recv_set.insert(recv_data[i]);

        std::cout << "Rank " << rank << " got set: ";
        for (std::set<int>::iterator i=recv_set.begin(); i!=recv_set.end(); i++)
            std::cout << *i << " ";
        std::cout << std::endl;
    }

    MPI_Finalize();
    return 0;
}

And running gives:

$ mpicxx -o mpisetvector mpi-set-vector.cxx 
$ mpirun -np 2 mpisetvector
Rank 0 sending set: 1 4 9 16 
Rank 1 got set: 1 4 9 16 

But really, if you're going to be doing this with other types of object as well, Boost is the way to go.