1 #include <mpi.h> 2 #include <stdio.h> 3 #include <stdlib.h> 4 #include <unistd.h> 5 #include <set> 6 #include <string> 7 #include <sstream> 8 #include <cassert> 9 #include "phIO.h" 10 #include "posixio.h" 11 #include "phio_posix.h" 12 13 int openfile(const char* geomfilename, phio_fp& file) { 14 int commrank; 15 MPI_Comm_rank(MPI_COMM_WORLD,&commrank); 16 int err = posix_openfile_single(geomfilename, file); 17 int globalerr = 0; 18 MPI_Allreduce(&err, &globalerr, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); 19 if(err > 0 && !commrank) 20 fprintf(stderr, "failed to open file %s\n", geomfilename); 21 return globalerr; 22 } 23 24 int main(int argc, char* argv[]) { 25 MPI_Init(&argc,&argv); 26 int commrank,commsize; 27 MPI_Comm_rank(MPI_COMM_WORLD,&commrank); 28 MPI_Comm_size(MPI_COMM_WORLD,&commsize); 29 if( argc != 5 ) { 30 if( !commrank ) 31 fprintf(stderr, "Usage: %s <geombc posix file> <verbosity=0|1|2> <rankoffset> <outfile>\n",argv[0]); 32 fprintf(stderr, "verbosity=0 will only write to the specified \'outfile\'\n",argv[0]); 33 fprintf(stderr, "verbosity>0 will write to the specified \'outfile\' and to stdout\n",argv[0]); 34 MPI_Finalize(); 35 return 1; 36 } 37 const char* filename = argv[1]; 38 const int verbosity = atoi(argv[2]); 39 const int rankoffset = atoi(argv[3]); 40 char* outfilename = argv[4]; 41 const char* phrase = "ilwork"; 42 const char* type = "integer"; 43 const char* iotype = "binary"; 44 int* ilwork = NULL; 45 int len = 0; 46 int one = 2; 47 48 const int phastarank = commrank+1+rankoffset; 49 const int group = (commrank+rankoffset)/2048; 50 51 char geomfilename[4096]; 52 sprintf(geomfilename, "%s/%d/geombc.dat.%d",filename,group,phastarank); 53 54 phio_fp file; 55 posixio_setup(&(file), 'r'); 56 int err = openfile(geomfilename, file); 57 if(err > 0) { 58 if(!commrank) fprintf(stderr, "trying again without the sub-directory...\n"); 59 sprintf(geomfilename, "%s/geombc.dat.%d",filename,phastarank); 60 err = openfile(geomfilename, file); 61 assert(!err); 62 if(!commrank) 63 fprintf(stderr, "geombc files opened successfully\n"); 64 } 65 phio_readheader(file, phrase, &len, &one, type, iotype); 66 ilwork = (int*) malloc(len*sizeof(int)); 67 phio_readdatablock(file, phrase, ilwork, &len, type, iotype); 68 phio_closefile(file); 69 70 // Initialization 71 int itkbeg = 0; 72 int m = 0; 73 int idl = 0; 74 std::set<int> neighbors; 75 76 // Compute summary info first such as number of communication tasks, neighboring parts, onwer parts, etc 77 int numtask = ilwork[0]; 78 int numowner = 0; 79 for(int itask=0;itask<numtask;itask++) { 80 int iacc = ilwork [itkbeg + 2]; 81 int iother = ilwork [itkbeg + 3]; 82 int numseg = ilwork [itkbeg + 4]; 83 if(iacc == 1) numowner++; 84 neighbors.insert(iother); 85 itkbeg = itkbeg + 4 + 2*numseg; 86 } 87 assert(neighbors.size() != 0); 88 MPI_Status status; 89 MPI_File outfile; 90 MPI_File_open(MPI_COMM_WORLD,outfilename, 91 MPI_MODE_CREATE|MPI_MODE_WRONLY,MPI_INFO_NULL,&outfile); 92 std::string header("rank,peers,tasks,owned,notowned\n"); 93 if( !commrank ) //write header 94 MPI_File_write_at(outfile,0,(void*)header.c_str(),header.size(),MPI_BYTE,&status); 95 std::stringstream ss; 96 ss << phastarank << "," 97 << neighbors.size() << "," 98 << numtask << "," 99 << numowner << "," 100 << numtask-numowner << "\n"; 101 std::string s = ss.str(); 102 int size = s.size(); 103 int offset = 0; 104 MPI_Exscan(&size,&offset,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 105 offset += header.size(); 106 int ret = MPI_File_write_at(outfile,offset,(void*)s.c_str(),s.size(),MPI_BYTE,&status); 107 assert(ret == MPI_SUCCESS); 108 if( verbosity > 0 ) { 109 // Print now communication info 110 printf("\n"); 111 printf("Communication info for this part:\n"); 112 itkbeg = 0; 113 for(int itask=0;itask<numtask;itask++) { 114 int itag = ilwork [itkbeg + 1]; 115 int iacc = ilwork [itkbeg + 2]; 116 int iother = ilwork [itkbeg + 3]; 117 int numseg = ilwork [itkbeg + 4]; 118 int isgbeg = ilwork [itkbeg + 5]; 119 120 // Toal number of nodes involved in this communication (lfront), including all segments 121 int lfront = 0; 122 for(int is=1;is<=numseg;is++) { 123 int lenseg = ilwork [itkbeg + 4 + 2*is]; 124 lfront = lfront + lenseg; 125 } 126 printf("Communication %d:\ttag %d\townership %d\trank %d\tnumseg %d\tnumvtx %d\n",itask,itag,iacc,iother,numseg,lfront); 127 itkbeg = itkbeg + 4 + 2*numseg; 128 } 129 printf("\n"); 130 } 131 132 // Print now the raw ilwork array 133 if( verbosity > 1 ) { 134 printf("ilwork array:\n"); 135 for(int i=0;i<len;i++) { 136 printf("%d\n",ilwork[i]); 137 } 138 } 139 140 free(ilwork); 141 MPI_File_close(&outfile); 142 MPI_Finalize(); 143 return 0; 144 } 145