MPI_Scatter and Gather for 2D matrix in MPI using C

Question

I am trying learn MPI and experimenting with the examples.

I am getting errors like

     Fatal error in PMPI_Scatter: Invalid buffer pointer, error stack:
     PMPI_Scatter(783): MPI_Scatter(sbuf=0x6021e0, scount=16, MPI_INT, rbuf=0x6021e0, rcount=16, MPI_INT, root=0, MPI_COMM_WORLD) failed
     PMPI_Scatter(710): Buffers must not be aliased

 ===================================================================================
 =   BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES
 =   EXIT CODE: 1
 =   CLEANING UP REMAINING PROCESSES
 =   YOU CAN IGNORE THE BELOW CLEANUP MESSAGES
===================================================================================

My code is :

   #include <mpi.h>
   #include <stdio.h>

   #define SIZE 8           /* Size of matrices */
   #define MAX_RAND     100

   int A[SIZE][SIZE], B[SIZE][1], C[SIZE][1],D[SIZE][SIZE],E[SIZE][1];

   void fill_matrix(int m[SIZE][SIZE])
   {
   //static int n=0;
    int i, j;

    printf("\n*****************************\n");
    for (i=0; i<SIZE; i++)
        {
            for (j=0; j<SIZE; j++){     
                m[i][j] = rand() % MAX_RAND;
                printf("%2d ", m[i][j]);
          }
        printf("\n");
        }
         printf("\n*****************************\n");
     }



    void fill_vector(int m[SIZE][1])
    {
    //static int n=0;
     int i, j;

     printf("\n*****************************\n");
      for (i=0; i<SIZE; i++)
       {
            for (j=0; j<1; j++){     
                m[i][j] = rand() % MAX_RAND;
                printf("%2d ", m[i][j]);
          }
        printf("\n");
        }
         printf("\n*****************************\n");
     }


    void print_matrix(int m[SIZE][SIZE])
    {
      int i, j = 0;
      for (i=0; i<SIZE; i++) {
        printf("\n\t| ");
        for (j=0; j<SIZE; j++)
          printf("%2d ", m[i][j]);
        printf("|");
      }
    }




    void print_vector(int m[SIZE][1])
    {
      int i, j = 0;
      for (i=0; i<SIZE; i++) {
        printf("\n\t| ");
        for (j=0; j<1; j++)
          printf("%2d ", m[i][j]);
        printf("|");
      }
    }


    int main(int argc, char *argv[])
    {
      int myrank, P, from, to, i, j, k;
    //  int tag = 666;      /* any value will do */
    //  MPI_Status status;

      MPI_Init (&argc, &argv);
      MPI_Comm_rank(MPI_COMM_WORLD, &myrank);   /* who am i */
      MPI_Comm_size(MPI_COMM_WORLD, &P); /* number of processors */


      if (SIZE%P!=0) {
        if (myrank==0) printf("Matrix size not divisible by number of processors\n");
        MPI_Finalize();
        exit(-1);
      }

      from = myrank * SIZE/P;
      to = ((myrank+1) * SIZE/P);

      /* Process 0 fills the input matrices and broadcasts them to the rest */
      /* (actually, only the relevant stripe of A is sent to each process) */

      if (myrank==0) {

    {
      //static int n=0;
      int i, j;

         printf("\n*****************************\n");
      for (i=0; i<SIZE; i++)
        {
            for (j=0; j<SIZE; j++){     
                A[i][j] = rand() % MAX_RAND;        
                printf("%d ", A[i][j]);
          }
        printf("\n");
        }
         printf("\n*****************************\n");
     }
        fill_vector(B);
      }

        int s=SIZE*SIZE/P;
     // printf("computing slice %d (from row %d to %d)\n", myrank, from, to-1);
      MPI_Bcast (B, SIZE*1, MPI_INT, 0, MPI_COMM_WORLD);
    //    printf("\n\n%d",s);
        //print_vector(s);
    //printf("\n\n");

      MPI_Scatter (&A, SIZE*SIZE/P, MPI_INT, &A[from], SIZE*SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);


      printf("computing slice %d (from row %d to %d)\n", myrank, from, to-1);
      for (i=from; i<to; i++) 
        for (j=0; j<SIZE; j++) {
          C[i][0]=0;
          for (k=0; k<SIZE; k++){
            C[i][0] += A[i][k]*B[k][0];
        }
        }

      MPI_Gather (&C[from], SIZE*SIZE/P, MPI_INT, &C, SIZE*SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);

      if (myrank==0) {
        printf("\n\n");

        {
            int i, j = 0;
                for (i=0; i<SIZE; i++) {
                        printf("\n\t| ");
                    for (j=0; j<SIZE; j++)
                        printf("%d ", A[i][j]);
                        printf("|");
            }
            }

        printf("\n\n");
        print_matrix(D);
        printf("\n\n\t       * \n");
        print_vector(B);
        printf("\n\n\t       = \n");
        print_vector(C);
        printf("\n\n");
        print_vector(E);
        printf("\n\n");
      }

      MPI_Finalize();
      return 0;
    }

As i am JAVA programmer i know very little about pointers, so if my problem sounds silly pardon me as i am still learning. What i am trying to do here is splitting A matrix rows to different processors and broadcasting the whole B Vector and multiplying both to get C vector which then again i want to receive using gather function.

Answer 1

Two things arose from your code:

• As you have guessed, one is an error related to pointers. MPI_Scatter() needs the pointer to the data to be sent and a pointer to the buffer where data will be received. For instance, since A is a two dimensional array (contiguous in memory):

  MPI_Scatter (&A[0][0], SIZE*SIZE/P, MPI_INT, &A[from][0], SIZE*SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);
  

where &A[0][0] is the pointer to the send buffer and &A[from][0] is the pointer to the receive buffer.

• The second problem the in the MPI_Gather(). Of course, the same error as the first one occurs as well. Moreover, C is a vector, not a matrix : the number of integers to send is much lower than SIZE*SIZE/P. Hence, the number of integers to be sent is SIZE/P.

   MPI_Gather (&C[from][0], SIZE/P, MPI_INT, &C[0][0], SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);
  

  where int C[SIZE][1] is a vector.

Here is your code with a few modifications:

#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>

#define SIZE 8           /* Size of matrices */
#define MAX_RAND     100

int A[SIZE][SIZE], B[SIZE][1], C[SIZE][1];//D[SIZE][SIZE],E[SIZE][1];

void fill_matrix(int m[SIZE][SIZE])
{
    //static int n=0;
    int i, j;

    printf("\n*****************************\n");
    for (i=0; i<SIZE; i++)
    {
        for (j=0; j<SIZE; j++){     
            m[i][j] = rand() % MAX_RAND;
            printf("%2d ", m[i][j]);
        }
        printf("\n");
    }
    printf("\n*****************************\n");
}



void fill_vector(int m[SIZE][1])
{
    //static int n=0;
    int i, j;

    printf("\n*****************************\n");
    for (i=0; i<SIZE; i++)
    {
        for (j=0; j<1; j++){     
            m[i][j] = rand() % MAX_RAND;
            printf("%2d ", m[i][j]);
        }
        printf("\n");
    }
    printf("\n*****************************\n");
}


void print_matrix(int m[SIZE][SIZE])
{
    int i, j = 0;
    for (i=0; i<SIZE; i++) {
        printf("\n\t| ");
        for (j=0; j<SIZE; j++)
            printf("%2d ", m[i][j]);
        printf("|");
    }
}




void print_vector(int m[SIZE][1])
{
    int i, j = 0;
    for (i=0; i<SIZE; i++) {
        printf("\n\t| ");
        for (j=0; j<1; j++)
            printf("%2d ", m[i][j]);
        printf("|");
    }
}


int main(int argc, char *argv[])
{
    int myrank, P, from, to, i, j, k;
    //  int tag = 666;      /* any value will do */
    //  MPI_Status status;

    MPI_Init (&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);   /* who am i */
    MPI_Comm_size(MPI_COMM_WORLD, &P); /* number of processors */


    if (SIZE%P!=0) {
        if (myrank==0) printf("Matrix size not divisible by number of processors\n");
        MPI_Finalize();
        exit(-1);
    }

    from = myrank * SIZE/P;
    to = ((myrank+1) * SIZE/P);

    /* Process 0 fills the input matrices and broadcasts them to the rest */
    /* (actually, only the relevant stripe of A is sent to each process) */

    if (myrank==0) {


        //static int n=0;
        int i, j;

        printf("\n*****************************\n");
        for (i=0; i<SIZE; i++)
        {
            for (j=0; j<SIZE; j++){     
                A[i][j] = rand() % MAX_RAND;        
                printf("%d ", A[i][j]);
            }
            printf("\n");

            printf("\n*****************************\n");
        }
        fill_vector(B);
    }

    //int s=SIZE*SIZE/P;
    // printf("computing slice %d (from row %d to %d)\n", myrank, from, to-1);
    MPI_Bcast (B, SIZE*1, MPI_INT, 0, MPI_COMM_WORLD);
    //    printf("\n\n%d",s);
    //print_vector(s);
    //printf("\n\n");
    if(myrank==0){
        MPI_Scatter (&A[0][0], SIZE*SIZE/P, MPI_INT, MPI_IN_PLACE, SIZE*SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);
    }else{
        MPI_Scatter (&A[0][0], SIZE*SIZE/P, MPI_INT, &A[from][0], SIZE*SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);
    }


    printf("computing slice %d (from row %d to %d)\n", myrank, from, to-1);
    for (i=from; i<to; i++) 
        for (j=0; j<SIZE; j++) {
            C[i][0]=0;
            for (k=0; k<SIZE; k++){
                C[i][0] += A[i][k]*B[k][0];
            }
        }

    if(myrank==0){
         MPI_Gather (MPI_IN_PLACE, SIZE/P, MPI_INT, &C[0][0], SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);
    }else{
         MPI_Gather (&C[from][0], SIZE/P, MPI_INT, &C[0][0], SIZE/P, MPI_INT, 0, MPI_COMM_WORLD);
    }

    if (myrank==0) {
        printf("\n\n");

        {
            int i, j = 0;
            for (i=0; i<SIZE; i++) {
                printf("\n\t| ");
                for (j=0; j<SIZE; j++)
                    printf("%d ", A[i][j]);
                printf("|");
            }
        }

        printf("\n\n");
        //   print_matrix(D);
        printf("\n\n\t       * \n");
        print_vector(B);
        printf("\n\n\t       = \n");
        print_vector(C);
        printf("\n\n");
        //   print_vector(E);
        //   printf("\n\n");
    }

    MPI_Finalize();
    return 0;
}

It can be compiled by mpicc main.c -o main -Wall and ran by mpirun -np 4 main.

I suppose than srand() is not used to get a reproducible result. If you plan to use larger arrays, you will need to allocate them. If it is the case, take a look at the following question : sending blocks of 2D array in C using MPI

Edit : I should have noticed that the send buffer and the receive buffer are identical. This is called buffer aliasing (see Mvapich2 buffer aliasing ) and the flag MPI_IN_PLACE must be used instead (see How does MPI_IN_PLACE work with MPI_Scatter? ). The code above is modified accordingly. Sorry for the answer that was previously incomplete !