/* L-6 MCS 572 timing to fan out a random matrix over 8 processors */

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "mpi.h"

#define size 2520  /* size of the problem, divisible by 2,3,4,..,10 */
#define tag 100    /* tag for sending a number */
#define v 1        /* verbose flag for debugging */

void random_matrix ( int n, int m, double *a );
/* generates a random n-by-m matrix */

void print_time ( int p, double time[p] );
/* prints the timing results in the array time of size p */

int main ( int argc, char *argv[] )
{
   int myid,k,p,s,i,j,d;
   double *wall_time,startwtime,endwtime,wtime;
   double *A = (double*)calloc(size*size,sizeof(double));
   double *my_A,*ptr_A,*ptr_my_A;
   MPI_Status status;
   MPI_Init(&argc,&argv);
   MPI_Comm_size(MPI_COMM_WORLD,&p);
   MPI_Comm_rank(MPI_COMM_WORLD,&myid);
   s = size/p;
   my_A = (double*)calloc(s*size,sizeof(double));
   srand(time(NULL));
   if(myid==0) /* manager allocates and initializes matrix */
   {
      wall_time = (double*) calloc(p,sizeof(double));
      startwtime = MPI_Wtime();
      random_matrix(size,size,A);
      for(k=0,ptr_A=A,ptr_my_A=my_A; k<s*size; k++,ptr_A++,ptr_my_A++)
         *ptr_my_A = *ptr_A;
   }
   else
      startwtime = MPI_Wtime();

   for(i=0,d=1; i<3; i++,d*=2) /* submatrix my_A is fanned out */
      for(j=0; j<d; j++)
         if(myid==j)
         {
            if(v>0) printf("Processor %d is sending to %d.\n",j,j+d);
            MPI_Send(my_A,s*size,MPI_DOUBLE,j+d,tag,MPI_COMM_WORLD);
         }
         else if(myid==j+d)
         {
            if(v>0) printf("Processor %d is receiving from %d.\n",j+d,j);
            MPI_Recv(my_A,s*size,MPI_DOUBLE,j,tag,MPI_COMM_WORLD,&status);
         }
   endwtime = MPI_Wtime();
   wtime = endwtime - startwtime;
   MPI_Gather(&wtime,1,MPI_DOUBLE,wall_time,1,MPI_DOUBLE,0,MPI_COMM_WORLD);
   if(myid==0) print_time(p,wall_time);
   MPI_Finalize();
   return 0;
}

void print_time ( int p, double time[p] )
{
   int i;

   printf("\nTotal wall time = %lf seconds on %d processors\n",time[0],p);
   for(i=1; i<p; i++)
      printf("Wall time on processor %d = %lf seconds.\n",i,time[i]);
}

void random_matrix ( int n, int m, double *a )
{
   int i,j;
   double *p;

   p = a;
   for(i=0; i<n; i++)
      for(j=0; j<m; j++)
         *(p++) = ((double) rand())/RAND_MAX;
}