1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdio.h> 7 #include <string.h> 8 #include <ctype.h> 9 #include <stdlib.h> 10 #include <time.h> 11 #include <math.h> 12 #include "mpi.h" 13 #include "phastaIO.h" 14 #include "rdtsc.h" 15 #include <FCMangle.h> 16 #include "new_interface.h" 17 #include "phIO.h" 18 19 //MR CHANGE 20 #include "common_c.h" 21 //MR CHANGE END 22 23 #ifdef intel 24 #include <winsock2.h> 25 #else 26 #include <unistd.h> 27 #include <strings.h> 28 #endif 29 30 //extern double cpu_speed = 2600000000.0; //for Jaguar XT5 31 //extern double cpu_speed = 2100000000.0; //for Jaguar xt4 32 //extern double cpu_speed = 850000000.0; //for Intrepid 33 34 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 35 int isThisRank; 36 37 double *value = (double*)malloc(sizeof(double)); 38 *value = 1.0*(*ivalue); 39 40 rgetMinMaxAvg(value,stats,statRanks); 41 42 /* MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 43 isThisRank=workfc.numpe+1; 44 if(*value==stats[0]) 45 isThisRank=workfc.myrank; 46 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 47 48 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 49 isThisRank=workfc.numpe+1; 50 if(*value==stats[1]) 51 isThisRank=workfc.myrank; 52 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 53 54 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 55 stats[2] /= workfc.numpe; */ 56 57 free(value); 58 } 59 60 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 61 int isThisRank; 62 63 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 64 isThisRank=workfc.numpe+1; 65 if(*value==stats[0]) 66 isThisRank=workfc.myrank; 67 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 68 69 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 70 isThisRank=workfc.numpe+1; 71 if(*value==stats[1]) 72 isThisRank=workfc.myrank; 73 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 74 75 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 76 stats[2] /= workfc.numpe; 77 78 double sqValue = (*value)*(*value), sqValueAvg = 0.; 79 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 80 sqValueAvg /= workfc.numpe; 81 // stats[3] = sqValueAvg; 82 83 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 84 } 85 86 void print_mesh_stats(void) { 87 int statRanks[2]; 88 double iStats[4], rStats[4]; 89 90 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 91 if(workfc.myrank==workfc.master) 92 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 93 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 94 if(workfc.myrank==workfc.master) 95 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 96 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 97 if(workfc.myrank==workfc.master) 98 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 99 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 100 if(workfc.myrank==workfc.master) { 101 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 102 printf("\n"); 103 } 104 } 105 106 void print_mpi_stats(void) { 107 int statRanks[2]; 108 double iStats[4], rStats[4]; 109 110 // NS equations 111 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 112 if(workfc.myrank==workfc.master) 113 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 114 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 115 if(workfc.myrank==workfc.master) 116 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 117 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 118 if(workfc.myrank==workfc.master) 119 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 120 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 121 if(workfc.myrank==workfc.master) 122 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 123 124 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 125 if(workfc.myrank==workfc.master) 126 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 127 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 128 if(workfc.myrank==workfc.master) 129 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 130 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 131 if(workfc.myrank==workfc.master) 132 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 133 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 134 if(workfc.myrank==workfc.master) 135 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 136 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 137 if(workfc.myrank==workfc.master) { 138 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 139 printf("\n"); 140 } 141 // Scalars 142 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 143 if(workfc.myrank==workfc.master) 144 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 145 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 146 if(workfc.myrank==workfc.master) 147 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 148 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 149 if(workfc.myrank==workfc.master) 150 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 151 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 152 if(workfc.myrank==workfc.master) 153 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 154 155 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 156 if(workfc.myrank==workfc.master) 157 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 158 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 159 if(workfc.myrank==workfc.master) 160 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 161 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 162 if(workfc.myrank==workfc.master) 163 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 164 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 165 if(workfc.myrank==workfc.master) 166 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 167 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 168 if(workfc.myrank==workfc.master) 169 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 170 171 172 } 173 174 //void print_system_stats(double tcorecp[2]) { 175 void print_system_stats(double *tcorecp, double *tcorecpscal) { 176 int statRanks[2]; 177 double iStats[4], rStats[4]; 178 double syst_assembly, syst_solve; 179 180 // NS equations 181 syst_assembly = tcorecp[0]; 182 syst_solve = tcorecp[1]; 183 184 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 185 if(workfc.myrank==workfc.master) 186 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 187 188 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 189 if(workfc.myrank==workfc.master) 190 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 191 192 // Scalars 193 syst_assembly = tcorecpscal[0]; 194 syst_solve = tcorecpscal[1]; 195 196 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 197 if(workfc.myrank==workfc.master) 198 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 199 200 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 201 if(workfc.myrank==workfc.master) { 202 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 203 printf("\n"); 204 } 205 //printf("rank %d - syst_assembly %f - syst_solve %f\n",workfc.myrank,syst_assembly,syst_solve); 206 } 207 208 209 210 void countfieldstowriterestart() 211 { 212 int nfields; 213 214 // printf("TEST: %d %d %d %d %d\n",timdat.istep,timdat.itseq,inpdat.nstep[0],inpdat.nstep[1],timdat.lstep); 215 216 nfields = 2; //solution, time derivatives 217 218 if(outpar.ivort == 1){ 219 nfields++; //vorticity 220 } 221 222 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 223 nfields++; //instantaneous wss in bflux.f 224 } 225 226 // if(ideformwall.eq.1) not handled yet 227 228 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 229 230 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 231 nfields = nfields +2; 232 233 //if Print Error Indicators = true (call write_error in itrdrv) 234 if(turbvar.ierrcalc == 1){ 235 nfields++; 236 } 237 238 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 239 if(outpar.ioybar == 1){ 240 nfields++; //ybar 241 242 //phase average fields 243 if(outpar.nphasesincycle >0) { 244 nfields = nfields + outpar.nphasesincycle; 245 } 246 247 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 248 nfields++; //wssbar 249 } 250 251 } 252 253 if(turbvari.irans < 0) { 254 nfields++; //dwal 255 } 256 257 } 258 259 outpar.nsynciofieldswriterestart = nfields; 260 261 if(workfc.myrank == 0) { 262 printf("Number of fields to write in restart files: %d\n", nfields); 263 } 264 } 265 266 267 void 268 Write_Restart( int* pid, 269 int* stepno, 270 int* nshg, 271 int* numVars, 272 double* array1, 273 double* array2 ) { 274 275 char fname[255]; 276 char rfile[60]; 277 char existingfile[30], linkfile[30]; 278 int irstou; 279 int magic_number = 362436; 280 int* mptr = &magic_number; 281 // time_t timenow = time ( &timenow); 282 double version=0.0; 283 int isize, nitems; 284 int iarray[10]; 285 286 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 287 openfile_(rfile,"write", &irstou); 288 289 // writing the top ascii header for the restart file 290 291 writestring_( &irstou,"# PHASTA Input File Version 2.0\n"); 292 writestring_( &irstou, 293 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 294 295 bzero( (void*)fname, 255 ); 296 sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); 297 writestring_( &irstou, fname ); 298 299 bzero( (void*)fname, 255 ); 300 gethostname(fname,255); 301 writestring_( &irstou,"# This result was produced on: "); 302 writestring_( &irstou, fname ); 303 writestring_( &irstou,"\n"); 304 305 bzero( (void*)fname, 255 ); 306 sprintf(fname,"# %s\n", ctime( &timenow )); 307 writestring_( &irstou, fname ); 308 309 isize = 1; 310 nitems = 1; 311 iarray[ 0 ] = 1; 312 writeheader_( &irstou, "byteorder magic number ", 313 (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); 314 315 nitems = 1; 316 writedatablock_( &irstou, "byteorder magic number ", 317 (void*)mptr, &nitems, "integer", phasta_iotype ); 318 319 320 bzero( (void*)fname, 255 ); 321 sprintf(fname,"number of modes : < 0 > %d\n", *nshg); 322 writestring_( &irstou, fname ); 323 324 bzero( (void*)fname, 255 ); 325 sprintf(fname,"number of variables : < 0 > %d\n", *numVars); 326 writestring_( &irstou, fname ); 327 328 329 isize = (*nshg)*(*numVars); 330 nitems = 3; 331 iarray[ 0 ] = (*nshg); 332 iarray[ 1 ] = (*numVars); 333 iarray[ 2 ] = (*stepno); 334 writeheader_( &irstou, "solution ", 335 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 336 337 338 nitems = (*nshg)*(*numVars); 339 writedatablock_( &irstou, "solution ", 340 (void*)(array1), &nitems, "double", phasta_iotype ); 341 342 343 344 nitems = 3; 345 writeheader_( &irstou, "time derivative of solution ", 346 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 347 348 349 nitems = (*nshg)*(*numVars); 350 writedatablock_( &irstou, "time derivative of solution ", 351 (void*)(array2), &nitems, "double", phasta_iotype ); 352 353 354 closefile_( &irstou, "write" ); 355 */ 356 //MPI_Barrier(MPI_COMM_WORLD); 357 358 /////////////////////////////// Start of writing using new-lib //////////////////////////// 359 360 //MR CHANGE 361 int nfiles; 362 int nfields; 363 int numparts; 364 int irank; 365 int nprocs; 366 367 // First, count the number of fields to write and store the result in 368 countfieldstowriterestart(); 369 370 // Retrieve and compute the parameters required for SyncIO 371 nfiles = outpar.nsynciofiles; 372 nfields = outpar.nsynciofieldswriterestart; 373 numparts = workfc.numpe; 374 irank = *pid; //workfc.myrank; 375 nprocs = workfc.numpe; 376 //MR CHANGE END 377 int nppf = numparts/nfiles; 378 int GPID; 379 380 // Calculate number of parts each proc deal with and where it start and end ... 381 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 382 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 383 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 384 385 int descriptor; 386 char filename[255],path[255],fieldtag_s[255]; 387 bzero((void*)filename,255); 388 bzero((void*)fieldtag_s,255); 389 390 phio_restartname(stepno, filename); 391 phio_openfile_write(filename, &nfiles, &nfields, &nppf, &f_descriptor); 392 393 //MR CHANGE 394 // Measure the time - End of timer 395 // MPI_Barrier(MPI_COMM_WORLD); 396 // timer_end = rdtsc(); 397 // time_span=(double)((timer_end-timer_start)/cpu_speed); 398 // if (*pid==0) { 399 // printf("Time: 'openfile' of %s with %d fields and %d files is: %f s\n",filename,nfields,nfiles,time_span); 400 // printf("*****************************\n"); 401 // } 402 //MR CHANGE END 403 404 field_flag=0; 405 406 int i; 407 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 408 // GPID : global part id, corresponds to rank ... 409 // e.g : (in this example) 410 // proc 0 : 1--4 411 // proc 1 : 5--8 ... 412 GPID = startpart + i; 413 414 // Write solution field ... 415 sprintf(fieldtag_s,"solution@%d",GPID); 416 417 isize = (*nshg)*(*numVars); 418 nitems = 3; 419 iarray[ 0 ] = (*nshg); 420 iarray[ 1 ] = (*numVars); 421 iarray[ 2 ] = (*stepno); 422 423 //MR CHANGE 424 // Measure the time - Start the timer 425 // MPI_Barrier(MPI_COMM_WORLD); 426 // timer_start = rdtsc(); 427 //MR CHANGE END 428 429 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 430 431 //MR CHANGE 432 // Measure the time - End of timer 433 // MPI_Barrier(MPI_COMM_WORLD); 434 // timer_end = rdtsc(); 435 // time_span=(double)((timer_end-timer_start)/cpu_speed); 436 // if (*pid==0) { 437 // printf("\n*****************************\n"); 438 // printf("Time: header for 'Solution': %f s\n",time_span); 439 // } 440 //MR CHANGE END 441 442 nitems = (*nshg)*(*numVars); 443 444 //MR CHANGE 445 // Measure the time - Start the timer 446 // MPI_Barrier(MPI_COMM_WORLD); 447 // timer_start = rdtsc(); 448 //MR CHANGE END 449 450 writedatablock( &f_descriptor, fieldtag_s, (void*)(array1), &isize, "double", phasta_iotype ); 451 452 //MR CHANGE 453 // Measure the time - End of timer 454 // MPI_Barrier(MPI_COMM_WORLD); 455 // timer_end = rdtsc(); 456 // time_span=(double)((timer_end-timer_start)/cpu_speed); 457 458 // int isizemin,isizemax,isizetot; 459 // double sizemin,sizemax,sizeavg,sizetot,rate; 460 461 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 462 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 463 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 464 465 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 466 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 467 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 468 // sizeavg=(double)(1.0*sizetot/workfc.numpe); 469 // rate=(double)(1.0*sizetot/time_span); 470 471 // if (*pid==0) { 472 // printf("Time: block for 'Solution': %f s\n",time_span); 473 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 474 475 // } 476 //MR CHANGE END 477 478 } 479 field_flag++; 480 481 for ( i = 0; i < nppp; i++) { 482 483 // GPID : global part id, corresponds to rank ... 484 // e.g : (in this example) 485 // proc 0 : 1--4 486 // proc 1 : 5--8 ... 487 GPID = startpart + i; 488 489 // Write solution field ... 490 sprintf(fieldtag_s,"time derivative of solution@%d",GPID); 491 492 isize = (*nshg)*(*numVars); 493 nitems = 3; 494 iarray[ 0 ] = (*nshg); 495 iarray[ 1 ] = (*numVars); 496 iarray[ 2 ] = (*stepno); 497 498 //MR CHANGE 499 // Measure the time - Start the timer 500 // MPI_Barrier(MPI_COMM_WORLD); 501 // timer_start = rdtsc(); 502 //MR CHANGE END 503 504 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 505 506 //MR CHANGE 507 // Measure the time - End of timer 508 // MPI_Barrier(MPI_COMM_WORLD); 509 // timer_end = rdtsc(); 510 // time_span=(double)((timer_end-timer_start)/cpu_speed); 511 // if (*pid==0) { 512 // printf("Time: header for 'Time Derivative of solution': %f s\n",time_span); 513 // } 514 //MR CHANGE END 515 516 nitems = (*nshg)*(*numVars); 517 518 //MR CHANGE 519 // Measure the time - Start the timer 520 // MPI_Barrier(MPI_COMM_WORLD); 521 // timer_start = rdtsc(); 522 //MR CHANGE END 523 524 writedatablock( &f_descriptor, fieldtag_s, (void*)(array2), &isize, "double", phasta_iotype ); 525 526 //MR CHANGE 527 // Measure the time - End of timer 528 // MPI_Barrier(MPI_COMM_WORLD); 529 // timer_end = rdtsc(); 530 // time_span=(double)((timer_end-timer_start)/cpu_speed); 531 532 // int isizemin,isizemax,isizetot; 533 // double sizemin,sizemax,sizeavg,sizetot,rate; 534 535 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 536 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 537 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 538 539 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 540 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 541 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 542 // sizeavg=sizetot/workfc.numpe; 543 // rate=sizetot/time_span; 544 545 // if (*pid==0) { 546 // printf("Time: block for 'Time Derivative of Solution': %f s\n",time_span); 547 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 548 // printf("*****************************\n"); 549 550 // } 551 //MR CHANGE END 552 553 } 554 field_flag++; 555 556 if (field_flag==nfields){ 557 558 //MR CHANGE 559 // Measure the time - Start the timer 560 // MPI_Barrier(MPI_COMM_WORLD); 561 // timer_start = rdtsc(); 562 //MR CHANGE END 563 564 //MR CHANGE 565 // Measure the time - End of timer 566 // MPI_Barrier(MPI_COMM_WORLD); 567 // timer_end = rdtsc(); 568 // time_span=(double)((timer_end-timer_start)/cpu_speed); 569 // if (*pid==0) { 570 // printf("\n*****************************\n"); 571 // printf("Time: 'closefile' is: %f s\n",time_span); 572 // } 573 //MR CHANGE END 574 575 //MR CHANGE 576 // Measure the time - Start the timer 577 // MPI_Barrier(MPI_COMM_WORLD); 578 // timer_start = rdtsc(); 579 //MR CHANGE END 580 581 phio_closefile_write(&f_descriptor); 582 583 //MR CHANGE 584 // Measure the time - End of timer 585 // MPI_Barrier(MPI_COMM_WORLD); 586 // timer_end = rdtsc(); 587 // time_span=(double)((timer_end-timer_start)/cpu_speed); 588 if (*pid==0) { 589 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 590 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 591 printf("\n"); 592 // printf("*****************************\n"); 593 } 594 } 595 //MR CHANGE END 596 597 598 599 /////////////////////////////////////////////////////////////////////////////////////////// 600 601 /* create a soft link of the restart we just wrote to restart.latest 602 this is the file the next run will always try to start from */ 603 604 /* sprintf( linkfile, "restart.latest.%d", *pid+1 ); 605 unlink( linkfile ); 606 sprintf( existingfile, "restart.%d.%d", *stepno, *pid+1 ); 607 link( existingfile, linkfile ); 608 */ 609 } 610 611 void 612 Write_Error( int* pid, 613 int* stepno, 614 int* nshg, 615 int* numVars, 616 double* array1 ) { 617 618 619 char fname[255]; 620 char rfile[60]; 621 int irstou; 622 int magic_number = 362436; 623 int* mptr = &magic_number; 624 //printf("Time is commented\n"); 625 //time_t timenow = time ( &timenow); 626 //printf("Yes\n"); 627 double version=0.0; 628 int isize, nitems; 629 int iarray[10]; 630 631 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 632 openfile_(rfile,"append", &irstou); 633 634 isize = (*nshg)*(*numVars); 635 nitems = 3; 636 iarray[ 0 ] = (*nshg); 637 iarray[ 1 ] = (*numVars); 638 iarray[ 2 ] = (*stepno); 639 writeheader_( &irstou, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 640 641 642 nitems = (*nshg)*(*numVars); 643 writedatablock_( &irstou, "errors ", (void*)(array1), &nitems, "double", phasta_iotype ); 644 645 closefile_( &irstou, "append" );*/ 646 647 /////////////////////////////// Start of writing using new-lib //////////////////////////// 648 649 int nfiles; 650 int nfields; 651 int numparts; 652 int irank; 653 int nprocs; 654 655 // unsigned long long timer_start; 656 // unsigned long long timer_end; 657 // double time_span; 658 659 nfiles = outpar.nsynciofiles; 660 nfields = outpar.nsynciofieldswriterestart; 661 numparts = workfc.numpe; 662 irank = *pid; //workfc.myrank; 663 nprocs = workfc.numpe; 664 665 int nppf = numparts/nfiles; 666 int GPID; 667 668 // Calculate number of parts each proc deal with and where it start and end ... 669 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 670 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 671 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 672 673 field_flag++; 674 675 char fieldtag[255]; 676 677 int i; 678 for ( i = 0; i < nppp; i++ ) { 679 GPID = startpart + i; 680 sprintf(fieldtag,"errors@%d",GPID); 681 682 if(*pid==0) { 683 // printf("\n*****************************\n"); 684 printf("\n"); 685 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldtag); 686 } 687 688 isize = (*nshg)*(*numVars); 689 nitems = 3; 690 iarray[ 0 ] = (*nshg); 691 iarray[ 1 ] = (*numVars); 692 iarray[ 2 ] = (*stepno); 693 694 //MR CHANGE 695 // Measure the time - Start the timer 696 // MPI_Barrier(MPI_COMM_WORLD); 697 // timer_start = rdtsc(); 698 //MR CHANGE END 699 700 writeheader( &f_descriptor, fieldtag, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 701 702 //MR CHANGE 703 // Measure the time - End of timer 704 // MPI_Barrier(MPI_COMM_WORLD); 705 // timer_end = rdtsc(); 706 // time_span=(double)((timer_end-timer_start)/cpu_speed); 707 // if (*pid==0) { 708 // printf("Time: header for 'error': %f s\n",time_span); 709 // } 710 //MR CHANGE END 711 712 //MR CHANGE 713 // Measure the time - Start the timer 714 // MPI_Barrier(MPI_COMM_WORLD); 715 // timer_start = rdtsc(); 716 //MR CHANGE END 717 718 writedatablock( &f_descriptor, fieldtag, (void*)array1, &isize, "double", phasta_iotype ); 719 720 //MR CHANGE 721 // Measure the time - End of timer 722 // MPI_Barrier(MPI_COMM_WORLD); 723 // timer_end = rdtsc(); 724 // time_span=(double)((timer_end-timer_start)/cpu_speed); 725 726 // int isizemin,isizemax,isizetot; 727 // double sizemin,sizemax,sizeavg,sizetot,rate; 728 729 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 730 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 731 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 732 733 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 734 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 735 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 736 // sizeavg=sizetot/workfc.numpe; 737 // rate=sizetot/time_span; 738 739 // if (*pid==0) { 740 // printf("Time: block for 'error': %f s\n",time_span); 741 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 742 // printf("*****************************\n"); 743 // } 744 //MR CHANGE END 745 746 } 747 748 // MPI_Barrier(MPI_COMM_WORLD); 749 // timer_end = rdtsc(); 750 // time_span=(double)(timer_end-timer_start)/cpu_speed; 751 752 // if (*pid==0) { 753 // printf("Field 'error' written in: %f s\n",time_span); 754 // printf("Write field '%s' finished! \n",fieldtag); 755 // } 756 757 if (field_flag==nfields){ 758 759 //MR CHANGE 760 // Measure the time - Start the timer 761 // MPI_Barrier(MPI_COMM_WORLD); 762 // timer_start = rdtsc(); 763 //MR CHANGE END 764 765 //MR CHANGE 766 // Measure the time - End of timer 767 // MPI_Barrier(MPI_COMM_WORLD); 768 // timer_end = rdtsc(); 769 // time_span=(double)((timer_end-timer_start)/cpu_speed); 770 // if (*pid==0) { 771 // printf("\n*****************************\n"); 772 // printf("Time: 'closefile' is: %f s\n",time_span); 773 // } 774 //MR CHANGE END 775 776 //MR CHANGE 777 // Measure the time - Start the timer 778 // MPI_Barrier(MPI_COMM_WORLD); 779 // timer_start = rdtsc(); 780 //MR CHANGE END 781 782 phio_closefile_write(&f_descriptor); 783 784 //MR CHANGE 785 // Measure the time - End of timer 786 // MPI_Barrier(MPI_COMM_WORLD); 787 // timer_end = rdtsc(); 788 // time_span=(double)((timer_end-timer_start)/cpu_speed); 789 if (*pid==0) { 790 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 791 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 792 printf("\n"); 793 // printf("*****************************\n"); 794 } 795 } 796 //MR CHANGE END 797 798 /////////////////////////////////////////////////////////////////////////////////////////// 799 800 801 } 802 803 804 void 805 Write_Displ( int* pid, 806 int* stepno, 807 int* nshg, 808 int* numVars, 809 double* array1 ) { 810 fprintf(stderr, "This function is dead...exiting\n"); 811 exit(1); 812 } 813 814 void 815 Write_Field( int *pid, 816 char* filemode, 817 char* fieldtag, 818 int* tagsize, 819 void* array, 820 char* arraytype, 821 int* nshg, 822 int* numvars, 823 int* stepno) { 824 825 //printf("Rank is %d, field is %s, tagsize is %d, nshg is %d, numvars is %d\n",*pid,fieldtag,*tagsize,*nshg,*numvars); 826 827 // char rfile[32]; 828 // assuming restart.sn.(pid+1) 829 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 830 831 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 832 strncpy(fieldlabel, fieldtag, *tagsize); 833 fieldlabel[*tagsize] = '\0'; 834 835 int irstou; 836 int magic_number = 362436; 837 int* mptr = &magic_number; 838 double version=0.0; 839 int isize, nitems; 840 int iarray[10]; 841 842 char fmode[10]; 843 if(!strncmp(filemode,"w",1)) 844 strcpy(fmode,"write"); 845 else // default is append 846 strcpy(fmode,"append"); 847 848 char datatype[10]; 849 if(!strncmp(arraytype,"i",1)) 850 strcpy(datatype,"int"); 851 else // default is double 852 strcpy(datatype,"double"); 853 854 /* openfile_(rfile, fmode, &irstou); 855 856 nitems = 3; // assuming field will write 3 items in iarray 857 iarray[ 0 ] = (*nshg); 858 iarray[ 1 ] = (*numvars); 859 iarray[ 2 ] = (*stepno); 860 861 isize = (*nshg)*(*numvars); 862 writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 863 864 nitems = (*nshg)*(*numvars); 865 writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 866 closefile_( &irstou, fmode); 867 */ 868 /////////////////////////////// Start of writing using new-lib //////////////////////////// 869 870 int nfiles; 871 int nfields; 872 int numparts; 873 int irank; 874 int nprocs; 875 876 // unsigned long long timer_start; 877 // unsigned long long timer_end; 878 // double time_span; 879 880 nfiles = outpar.nsynciofiles; 881 nfields = outpar.nsynciofieldswriterestart; 882 numparts = workfc.numpe; 883 irank = *pid; //workfc.myrank; 884 nprocs = workfc.numpe; 885 886 int nppf = numparts/nfiles; 887 int GPID; 888 889 // Calculate number of parts each proc deal with and where it start and end ... 890 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 891 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 892 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 893 894 char filename[255],path[255],fieldtag_s[255]; 895 bzero((void*)filename,255); 896 bzero((void*)fieldtag_s,255); 897 898 strncpy(fieldlabel, fieldtag, *tagsize); 899 900 field_flag++; 901 if(*pid==0) { 902 // printf("\n*****************************\n"); 903 printf("\n"); 904 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 905 } 906 907 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 908 909 // MPI_Barrier(MPI_COMM_WORLD); 910 // timer_start = rdtsc(); 911 912 int i; 913 for ( i = 0; i < nppp; i++ ) { 914 GPID = startpart + i; 915 916 // Write solution field ... 917 sprintf(fieldtag_s,"%s@%d",fieldlabel,GPID); 918 919 isize = (*nshg)*(*numvars); 920 nitems = 3; 921 iarray[ 0 ] = (*nshg); 922 iarray[ 1 ] = (*numvars); 923 iarray[ 2 ] = (*stepno); 924 925 //MR CHANGE 926 // Measure the time - Start the timer 927 // MPI_Barrier(MPI_COMM_WORLD); 928 // timer_start = rdtsc(); 929 //MR CHANGE END 930 931 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, datatype, phasta_iotype); 932 933 //MR CHANGE 934 // Measure the time - End of timer 935 // MPI_Barrier(MPI_COMM_WORLD); 936 // timer_end = rdtsc(); 937 // time_span=(double)((timer_end-timer_start)/cpu_speed); 938 // if (*pid==0) { 939 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 940 // } 941 //MR CHANGE END 942 943 nitems = (*nshg)*(*numvars); 944 945 //MR CHANGE 946 // Measure the time - Start the timer 947 // MPI_Barrier(MPI_COMM_WORLD); 948 // timer_start = rdtsc(); 949 //MR CHANGE END 950 951 writedatablock( &f_descriptor, fieldtag_s, array, &isize, datatype, phasta_iotype ); 952 953 //MR CHANGE 954 // Measure the time - End of timer 955 // MPI_Barrier(MPI_COMM_WORLD); 956 // timer_end = rdtsc(); 957 // time_span=(double)((timer_end-timer_start)/cpu_speed); 958 959 // int isizemin,isizemax,isizetot; 960 // double sizemin,sizemax,sizeavg,sizetot,rate; 961 962 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 963 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 964 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 965 966 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 967 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 968 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 969 // sizeavg=sizetot/workfc.numpe; 970 // rate=sizetot/time_span; 971 972 // if (*pid==0) { 973 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 974 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 975 // printf("*****************************\n"); 976 // } 977 //MR CHANGE END 978 979 } 980 981 // MPI_Barrier(MPI_COMM_WORLD); 982 // timer_end = rdtsc(); 983 // time_span=(double)(timer_end-timer_start)/cpu_speed; 984 985 // if (*pid==0) { 986 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 987 // printf("Write field '%s' finished! \n",fieldtag_s); 988 // } 989 990 // if (field_flag==nfields){ 991 // closefile_(&f_descriptor, "write"); 992 // finalizephmpiio_(&f_descriptor); 993 // if(*pid==0) { 994 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 995 // printf("\n*****************************\n"); 996 // } 997 // } 998 999 if (field_flag==nfields){ 1000 1001 //MR CHANGE 1002 // Measure the time - Start the timer 1003 // MPI_Barrier(MPI_COMM_WORLD); 1004 // timer_start = rdtsc(); 1005 //MR CHANGE END 1006 1007 //MR CHANGE 1008 // Measure the time - End of timer 1009 // MPI_Barrier(MPI_COMM_WORLD); 1010 // timer_end = rdtsc(); 1011 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1012 // if (*pid==0) { 1013 // printf("\n*****************************\n"); 1014 // printf("Time: 'closefile' is: %f s\n",time_span); 1015 // } 1016 //MR CHANGE END 1017 1018 //MR CHANGE 1019 // Measure the time - Start the timer 1020 // MPI_Barrier(MPI_COMM_WORLD); 1021 // timer_start = rdtsc(); 1022 //MR CHANGE END 1023 1024 phio_closefile_write(&f_descriptor); 1025 1026 //MR CHANGE 1027 // Measure the time - End of timer 1028 // MPI_Barrier(MPI_COMM_WORLD); 1029 // timer_end = rdtsc(); 1030 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1031 if (*pid==0) { 1032 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1033 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1034 printf("\n"); 1035 // printf("*****************************\n"); 1036 } 1037 } 1038 //MR CHANGE END 1039 1040 /////////////////////////////////////////////////////////////////////////////////////////// 1041 1042 free(fieldlabel); 1043 } 1044 1045 //MR CHANGE 1046 1047 //MR CHANGE 1048 void 1049 Write_PhAvg2( int* pid, 1050 char* filemode, 1051 char* fieldtag, 1052 int* tagsize, 1053 int* iphase, 1054 int* nphasesincycle, 1055 void* array, 1056 char* arraytype, 1057 int* nshg, 1058 int* numvars, 1059 int* stepno) { 1060 1061 // char rfile[32]; 1062 // assuming restart.sn.(pid+1) 1063 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 1064 1065 int addtagsize=0; // phase number is added to the name of the field 1066 if(*iphase<10) 1067 addtagsize=1; 1068 else if(*iphase<100) 1069 addtagsize=2; 1070 else if(*iphase<1000) 1071 addtagsize=3; 1072 1073 int tagsize2; 1074 tagsize2=*tagsize+addtagsize; 1075 1076 // char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 1077 // strncpy(fieldlabel, fieldtag, *tagsize); 1078 // fieldlabel[*tagsize] = '\0'; 1079 1080 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 1081 strncpy(fieldlabel, fieldtag, *tagsize); 1082 fieldlabel[tagsize2] = '\0'; 1083 1084 char straddtagsize[10]; 1085 sprintf(straddtagsize,"%d",*iphase); 1086 1087 if(*iphase<10) { 1088 fieldlabel[tagsize2-1]=straddtagsize[0]; 1089 } 1090 else if(*iphase<100) { 1091 fieldlabel[tagsize2-2]=straddtagsize[0]; 1092 fieldlabel[tagsize2-1]=straddtagsize[1]; 1093 } 1094 else if(*iphase<1000) { 1095 fieldlabel[tagsize2-3]=straddtagsize[0]; 1096 fieldlabel[tagsize2-2]=straddtagsize[1]; 1097 fieldlabel[tagsize2-1]=straddtagsize[2]; 1098 } 1099 1100 int irstou; 1101 int magic_number = 362436; 1102 int* mptr = &magic_number; 1103 double version=0.0; 1104 int isize, nitems; 1105 int iarray[10]; 1106 1107 char fmode[10]; 1108 if(!strncmp(filemode,"w",1)) 1109 strcpy(fmode,"write"); 1110 else // default is append 1111 strcpy(fmode,"append"); 1112 1113 char datatype[10]; 1114 if(!strncmp(arraytype,"i",1)) 1115 strcpy(datatype,"int"); 1116 else // default is double 1117 strcpy(datatype,"double"); 1118 1119 // 1120 // // if(*iphase==1) //open the file but then keep it open for the remaining cycles 1121 // openfile_(rfile, fmode, &irstou); 1122 // 1123 // // printf("iphase: %d - pid: %d - irstou %d\n",*iphase,*pid,irstou); 1124 // 1125 // 1126 // nitems = 3; // assuming field will write 3 items in iarray 1127 // iarray[ 0 ] = (*nshg); 1128 // iarray[ 1 ] = (*numvars); 1129 // iarray[ 2 ] = (*stepno); 1130 // 1131 // isize = (*nshg)*(*numvars); 1132 // writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 1133 // 1134 // nitems = (*nshg)*(*numvars); 1135 // writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 1136 // 1137 // // if(*iphase==*nphasesincycle) //close the file after nphasesincycle 1138 // closefile_( &irstou, fmode); 1139 // 1140 1141 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1142 1143 int nfiles; 1144 int nfields; 1145 int numparts; 1146 int irank; 1147 int nprocs; 1148 // unsigned long long timer_start; 1149 // unsigned long long timer_end; 1150 // double time_span; 1151 1152 nfiles = outpar.nsynciofiles; 1153 nfields = outpar.nsynciofieldswriterestart; 1154 numparts = workfc.numpe; 1155 irank = *pid; //workfc.myrank; 1156 nprocs = workfc.numpe; 1157 1158 int nppf = numparts/nfiles; 1159 int GPID; 1160 1161 // Calculate number of parts each proc deal with and where it start and end ... 1162 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1163 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1164 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1165 1166 //int descriptor; 1167 char filename[255],path[255],fieldtag_s[255]; 1168 bzero((void*)filename,255); 1169 bzero((void*)fieldtag_s,255); 1170 1171 // char * namer; 1172 // namer = strtok(fieldlabel," "); 1173 // strncpy(fieldlabel, fieldtag, *tagsize); 1174 1175 field_flag++; 1176 if(*pid==0) { 1177 // printf("\n*****************************\n"); 1178 printf("\n"); 1179 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 1180 } 1181 1182 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 1183 1184 int i; 1185 for ( i = 0; i < nppp; i++ ) { 1186 GPID = startpart + i; 1187 1188 // Write solution field ... 1189 sprintf(fieldtag_s,"%s@%d",fieldlabel,GPID); 1190 1191 //printf("This is %d and fieldtag_s is %s \n",myrank,fieldtag_s); 1192 1193 isize = (*nshg)*(*numvars); 1194 nitems = 3; 1195 iarray[ 0 ] = (*nshg); 1196 iarray[ 1 ] = (*numvars); 1197 iarray[ 2 ] = (*stepno); 1198 1199 //MR CHANGE 1200 // Measure the time - Start the timer 1201 // MPI_Barrier(MPI_COMM_WORLD); 1202 // timer_start = rdtsc(); 1203 //MR CHANGE END 1204 1205 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 1206 1207 //MR CHANGE 1208 // Measure the time - End of timer 1209 // MPI_Barrier(MPI_COMM_WORLD); 1210 // timer_end = rdtsc(); 1211 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1212 // if (*pid==0) { 1213 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 1214 // } 1215 //MR CHANGE END 1216 1217 nitems = (*nshg)*(*numvars); 1218 1219 //MR CHANGE 1220 // Measure the time - Start the timer 1221 // MPI_Barrier(MPI_COMM_WORLD); 1222 // timer_start = rdtsc(); 1223 //MR CHANGE END 1224 1225 writedatablock( &f_descriptor, fieldtag_s, array, &isize, "double", phasta_iotype ); 1226 1227 //MR CHANGE 1228 // Measure the time - End of timer 1229 // MPI_Barrier(MPI_COMM_WORLD); 1230 // timer_end = rdtsc(); 1231 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1232 1233 // int isizemin,isizemax,isizetot; 1234 // double sizemin,sizemax,sizeavg,sizetot,rate; 1235 1236 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 1237 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 1238 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 1239 1240 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 1241 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 1242 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 1243 // sizeavg=sizetot/workfc.numpe; 1244 // rate=sizetot/time_span; 1245 1246 // if (*pid==0) { 1247 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 1248 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 1249 // printf("*****************************\n"); 1250 // } 1251 //MR CHANGE END 1252 1253 } 1254 1255 // if (*pid==0) { 1256 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 1257 // printf("Write field '%s' finished! \n",fieldtag_s); 1258 // } 1259 1260 // 1261 // if (field_flag==nfields){ 1262 // closefile_(&f_descriptor, "write"); 1263 // finalizephmpiio_(&f_descriptor); 1264 // if(*pid==0) { 1265 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1266 // printf("\n*****************************\n"); 1267 // } 1268 // } 1269 1270 if (field_flag==nfields){ 1271 1272 //MR CHANGE 1273 // Measure the time - Start the timer 1274 // MPI_Barrier(MPI_COMM_WORLD); 1275 // timer_start = rdtsc(); 1276 //MR CHANGE END 1277 1278 //MR CHANGE 1279 // Measure the time - End of timer 1280 // MPI_Barrier(MPI_COMM_WORLD); 1281 // timer_end = rdtsc(); 1282 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1283 // if (*pid==0) { 1284 // printf("\n*****************************\n"); 1285 // printf("Time: 'closefile' is: %f s\n",time_span); 1286 // } 1287 //MR CHANGE END 1288 1289 //MR CHANGE 1290 // Measure the time - Start the timer 1291 // MPI_Barrier(MPI_COMM_WORLD); 1292 // timer_start = rdtsc(); 1293 //MR CHANGE END 1294 1295 phio_closefile_write(&f_descriptor); 1296 1297 //MR CHANGE 1298 // Measure the time - End of timer 1299 // MPI_Barrier(MPI_COMM_WORLD); 1300 // timer_end = rdtsc(); 1301 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1302 if (*pid==0) { 1303 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1304 // printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1305 printf("\n"); 1306 // printf("*****************************\n"); 1307 } 1308 } 1309 //MR CHANGE END 1310 1311 /////////////////////////////////////////////////////////////////////////////////////////// 1312 1313 free(fieldlabel); 1314 } 1315 1316 1317 void 1318 Write_d2wall( int* pid, 1319 int* numnp, 1320 double* array1 ) { 1321 1322 // time_t timenow = time ( &timenow); 1323 int isize, nitems; 1324 int iarray[10]; 1325 1326 // MPI_Barrier(MPI_COMM_WORLD); 1327 1328 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1329 1330 int nfiles; 1331 int nfields; 1332 int numparts; 1333 int irank; 1334 int nprocs; 1335 1336 // First, count the number of fields to write and store the result in 1337 //countfieldstowriterestart(); 1338 1339 // Retrieve and compute the parameters required for SyncIO 1340 nfiles = outpar.nsynciofiles; 1341 nfields = 1; //outpar.nsynciofieldswriterestart; // Only the distance to the walls in d2wall 1342 numparts = workfc.numpe; 1343 irank = *pid; //workfc.myrank; 1344 nprocs = workfc.numpe; 1345 int nppf = numparts/nfiles; 1346 int GPID; 1347 1348 // Calculate number of parts each proc deal with and where it start and end ... 1349 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1350 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1351 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1352 1353 int descriptor; 1354 char filename[255],path[255],fieldtag_s[255]; 1355 bzero((void*)filename,255); 1356 bzero((void*)fieldtag_s,255); 1357 1358 phio_openfile_write("d2wall.", &nfiles, &nfields, &nppf, &f_descriptor); 1359 1360 field_flag=0; 1361 1362 int i; 1363 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 1364 // GPID : global part id, corresponds to rank ... 1365 // e.g : (in this example) 1366 // proc 0 : 1--4 1367 // proc 1 : 5--8 ... 1368 GPID = startpart + i; 1369 1370 // Write solution field ... 1371 sprintf(fieldtag_s,"d2wall@%d",GPID); 1372 1373 isize = (*numnp); 1374 nitems = 2; 1375 iarray[ 0 ] = (*numnp); 1376 iarray[ 1 ] = 1; //numVars = 1 1377 1378 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 1379 1380 //nitems = (*nshg)*(*numVars); 1381 //nitems = (*numnp); 1382 1383 writedatablock( &f_descriptor, fieldtag_s, (void*)(array1), &isize, "double", phasta_iotype ); 1384 1385 1386 } 1387 field_flag++; 1388 1389 if (field_flag==nfields){ 1390 phio_closefile_write(&f_descriptor); 1391 if (irank==0) { 1392 printf("\n"); 1393 } 1394 } 1395 } 1396 1397