1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdio.h> 7 #include <string.h> 8 #include <ctype.h> 9 #include <stdlib.h> 10 #include <time.h> 11 #include <math.h> 12 #include "mpi.h" 13 #include "phastaIO.h" 14 #include "rdtsc.h" 15 #include <FCMangle.h> 16 #include "new_interface.h" 17 #include "phIO.h" 18 19 //MR CHANGE 20 #include "common_c.h" 21 //MR CHANGE END 22 23 #ifdef intel 24 #include <winsock2.h> 25 #else 26 #include <unistd.h> 27 #include <strings.h> 28 #endif 29 30 //extern double cpu_speed = 2600000000.0; //for Jaguar XT5 31 //extern double cpu_speed = 2100000000.0; //for Jaguar xt4 32 //extern double cpu_speed = 850000000.0; //for Intrepid 33 34 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 35 int isThisRank; 36 37 double *value = (double*)malloc(sizeof(double)); 38 *value = 1.0*(*ivalue); 39 40 rgetMinMaxAvg(value,stats,statRanks); 41 42 /* MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 43 isThisRank=workfc.numpe+1; 44 if(*value==stats[0]) 45 isThisRank=workfc.myrank; 46 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 47 48 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 49 isThisRank=workfc.numpe+1; 50 if(*value==stats[1]) 51 isThisRank=workfc.myrank; 52 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 53 54 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 55 stats[2] /= workfc.numpe; */ 56 57 free(value); 58 } 59 60 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 61 int isThisRank; 62 63 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 64 isThisRank=workfc.numpe+1; 65 if(*value==stats[0]) 66 isThisRank=workfc.myrank; 67 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 68 69 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 70 isThisRank=workfc.numpe+1; 71 if(*value==stats[1]) 72 isThisRank=workfc.myrank; 73 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 74 75 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 76 stats[2] /= workfc.numpe; 77 78 double sqValue = (*value)*(*value), sqValueAvg = 0.; 79 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 80 sqValueAvg /= workfc.numpe; 81 // stats[3] = sqValueAvg; 82 83 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 84 } 85 86 void print_mesh_stats(void) { 87 int statRanks[2]; 88 double iStats[4], rStats[4]; 89 90 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 91 if(workfc.myrank==workfc.master) 92 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 93 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 94 if(workfc.myrank==workfc.master) 95 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 96 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 97 if(workfc.myrank==workfc.master) 98 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 99 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 100 if(workfc.myrank==workfc.master) { 101 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 102 printf("\n"); 103 } 104 } 105 106 void print_mpi_stats(void) { 107 int statRanks[2]; 108 double iStats[4], rStats[4]; 109 110 // NS equations 111 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 112 if(workfc.myrank==workfc.master) 113 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 114 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 115 if(workfc.myrank==workfc.master) 116 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 117 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 118 if(workfc.myrank==workfc.master) 119 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 120 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 121 if(workfc.myrank==workfc.master) 122 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 123 124 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 125 if(workfc.myrank==workfc.master) 126 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 127 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 128 if(workfc.myrank==workfc.master) 129 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 130 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 131 if(workfc.myrank==workfc.master) 132 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 133 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 134 if(workfc.myrank==workfc.master) 135 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 136 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 137 if(workfc.myrank==workfc.master) { 138 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 139 printf("\n"); 140 } 141 // Scalars 142 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 143 if(workfc.myrank==workfc.master) 144 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 145 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 146 if(workfc.myrank==workfc.master) 147 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 148 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 149 if(workfc.myrank==workfc.master) 150 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 151 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 152 if(workfc.myrank==workfc.master) 153 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 154 155 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 156 if(workfc.myrank==workfc.master) 157 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 158 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 159 if(workfc.myrank==workfc.master) 160 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 161 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 162 if(workfc.myrank==workfc.master) 163 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 164 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 165 if(workfc.myrank==workfc.master) 166 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 167 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 168 if(workfc.myrank==workfc.master) 169 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 170 171 172 } 173 174 //void print_system_stats(double tcorecp[2]) { 175 void print_system_stats(double *tcorecp, double *tcorecpscal) { 176 int statRanks[2]; 177 double iStats[4], rStats[4]; 178 double syst_assembly, syst_solve; 179 180 // NS equations 181 syst_assembly = tcorecp[0]; 182 syst_solve = tcorecp[1]; 183 184 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 185 if(workfc.myrank==workfc.master) 186 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 187 188 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 189 if(workfc.myrank==workfc.master) 190 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 191 192 // Scalars 193 syst_assembly = tcorecpscal[0]; 194 syst_solve = tcorecpscal[1]; 195 196 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 197 if(workfc.myrank==workfc.master) 198 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 199 200 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 201 if(workfc.myrank==workfc.master) { 202 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 203 printf("\n"); 204 } 205 //printf("rank %d - syst_assembly %f - syst_solve %f\n",workfc.myrank,syst_assembly,syst_solve); 206 } 207 208 209 210 void countfieldstowriterestart() 211 { 212 int nfields; 213 214 // printf("TEST: %d %d %d %d %d\n",timdat.istep,timdat.itseq,inpdat.nstep[0],inpdat.nstep[1],timdat.lstep); 215 216 nfields = 2; //solution, time derivatives 217 218 if(outpar.ivort == 1){ 219 nfields++; //vorticity 220 } 221 222 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 223 nfields++; //instantaneous wss in bflux.f 224 } 225 226 // if(ideformwall.eq.1) not handled yet 227 228 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 229 230 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 231 nfields = nfields +2; 232 233 //if Print Error Indicators = true (call write_error in itrdrv) 234 if(turbvar.ierrcalc == 1){ 235 nfields++; 236 } 237 238 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 239 if(outpar.ioybar == 1){ 240 nfields++; //ybar 241 242 //phase average fields 243 if(outpar.nphasesincycle >0) { 244 nfields = nfields + outpar.nphasesincycle; 245 } 246 247 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 248 nfields++; //wssbar 249 } 250 251 } 252 253 if(turbvari.irans < 0) { 254 nfields++; //dwal 255 } 256 257 } 258 259 outpar.nsynciofieldswriterestart = nfields; 260 261 if(workfc.myrank == 0) { 262 printf("Number of fields to write in restart files: %d\n", nfields); 263 } 264 } 265 266 267 void 268 Write_Restart( int* pid, 269 int* stepno, 270 int* nshg, 271 int* numVars, 272 double* array1, 273 double* array2 ) { 274 275 char fname[255]; 276 char rfile[60]; 277 char existingfile[30], linkfile[30]; 278 int irstou; 279 int magic_number = 362436; 280 int* mptr = &magic_number; 281 // time_t timenow = time ( &timenow); 282 double version=0.0; 283 int isize, nitems; 284 int iarray[10]; 285 286 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 287 openfile_(rfile,"write", &irstou); 288 289 // writing the top ascii header for the restart file 290 291 writestring_( &irstou,"# PHASTA Input File Version 2.0\n"); 292 writestring_( &irstou, 293 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 294 295 bzero( (void*)fname, 255 ); 296 sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); 297 writestring_( &irstou, fname ); 298 299 bzero( (void*)fname, 255 ); 300 gethostname(fname,255); 301 writestring_( &irstou,"# This result was produced on: "); 302 writestring_( &irstou, fname ); 303 writestring_( &irstou,"\n"); 304 305 bzero( (void*)fname, 255 ); 306 sprintf(fname,"# %s\n", ctime( &timenow )); 307 writestring_( &irstou, fname ); 308 309 isize = 1; 310 nitems = 1; 311 iarray[ 0 ] = 1; 312 writeheader_( &irstou, "byteorder magic number ", 313 (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); 314 315 nitems = 1; 316 writedatablock_( &irstou, "byteorder magic number ", 317 (void*)mptr, &nitems, "integer", phasta_iotype ); 318 319 320 bzero( (void*)fname, 255 ); 321 sprintf(fname,"number of modes : < 0 > %d\n", *nshg); 322 writestring_( &irstou, fname ); 323 324 bzero( (void*)fname, 255 ); 325 sprintf(fname,"number of variables : < 0 > %d\n", *numVars); 326 writestring_( &irstou, fname ); 327 328 329 isize = (*nshg)*(*numVars); 330 nitems = 3; 331 iarray[ 0 ] = (*nshg); 332 iarray[ 1 ] = (*numVars); 333 iarray[ 2 ] = (*stepno); 334 writeheader_( &irstou, "solution ", 335 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 336 337 338 nitems = (*nshg)*(*numVars); 339 writedatablock_( &irstou, "solution ", 340 (void*)(array1), &nitems, "double", phasta_iotype ); 341 342 343 344 nitems = 3; 345 writeheader_( &irstou, "time derivative of solution ", 346 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 347 348 349 nitems = (*nshg)*(*numVars); 350 writedatablock_( &irstou, "time derivative of solution ", 351 (void*)(array2), &nitems, "double", phasta_iotype ); 352 353 354 closefile_( &irstou, "write" ); 355 */ 356 //MPI_Barrier(MPI_COMM_WORLD); 357 358 /////////////////////////////// Start of writing using new-lib //////////////////////////// 359 360 //MR CHANGE 361 int nfiles; 362 int nfields; 363 int numparts; 364 int irank; 365 int nprocs; 366 367 // First, count the number of fields to write and store the result in 368 countfieldstowriterestart(); 369 370 // Retrieve and compute the parameters required for SyncIO 371 nfiles = outpar.nsynciofiles; 372 nfields = outpar.nsynciofieldswriterestart; 373 numparts = workfc.numpe; 374 irank = *pid; //workfc.myrank; 375 nprocs = workfc.numpe; 376 //MR CHANGE END 377 int nppf = numparts/nfiles; 378 int GPID; 379 380 // Calculate number of parts each proc deal with and where it start and end ... 381 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 382 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 383 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 384 385 int descriptor; 386 char filename[255],path[255]; 387 bzero((void*)filename,255); 388 389 phio_restartname(stepno, filename); 390 phio_openfile_write(filename, &nfiles, &nfields, &nppf, &f_descriptor); 391 392 //MR CHANGE 393 // Measure the time - End of timer 394 // MPI_Barrier(MPI_COMM_WORLD); 395 // timer_end = rdtsc(); 396 // time_span=(double)((timer_end-timer_start)/cpu_speed); 397 // if (*pid==0) { 398 // printf("Time: 'openfile' of %s with %d fields and %d files is: %f s\n",filename,nfields,nfiles,time_span); 399 // printf("*****************************\n"); 400 // } 401 //MR CHANGE END 402 403 field_flag=0; 404 405 int i; 406 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 407 // GPID : global part id, corresponds to rank ... 408 // e.g : (in this example) 409 // proc 0 : 1--4 410 // proc 1 : 5--8 ... 411 GPID = startpart + i; 412 413 // Write solution field ... 414 isize = (*nshg)*(*numVars); 415 nitems = 3; 416 iarray[ 0 ] = (*nshg); 417 iarray[ 1 ] = (*numVars); 418 iarray[ 2 ] = (*stepno); 419 420 //MR CHANGE 421 // Measure the time - Start the timer 422 // MPI_Barrier(MPI_COMM_WORLD); 423 // timer_start = rdtsc(); 424 //MR CHANGE END 425 426 phio_writeheader( &f_descriptor, "solution", (void*)iarray, &nitems, 427 &isize, "double", phasta_iotype); 428 429 //MR CHANGE 430 // Measure the time - End of timer 431 // MPI_Barrier(MPI_COMM_WORLD); 432 // timer_end = rdtsc(); 433 // time_span=(double)((timer_end-timer_start)/cpu_speed); 434 // if (*pid==0) { 435 // printf("\n*****************************\n"); 436 // printf("Time: header for 'Solution': %f s\n",time_span); 437 // } 438 //MR CHANGE END 439 440 nitems = (*nshg)*(*numVars); 441 442 //MR CHANGE 443 // Measure the time - Start the timer 444 // MPI_Barrier(MPI_COMM_WORLD); 445 // timer_start = rdtsc(); 446 //MR CHANGE END 447 448 phio_writedatablock(&f_descriptor, "solution", (void*)(array1), 449 &isize, "double", phasta_iotype ); 450 451 //MR CHANGE 452 // Measure the time - End of timer 453 // MPI_Barrier(MPI_COMM_WORLD); 454 // timer_end = rdtsc(); 455 // time_span=(double)((timer_end-timer_start)/cpu_speed); 456 457 // int isizemin,isizemax,isizetot; 458 // double sizemin,sizemax,sizeavg,sizetot,rate; 459 460 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 461 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 462 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 463 464 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 465 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 466 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 467 // sizeavg=(double)(1.0*sizetot/workfc.numpe); 468 // rate=(double)(1.0*sizetot/time_span); 469 470 // if (*pid==0) { 471 // printf("Time: block for 'Solution': %f s\n",time_span); 472 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 473 474 // } 475 //MR CHANGE END 476 477 } 478 field_flag++; 479 480 for ( i = 0; i < nppp; i++) { 481 482 // GPID : global part id, corresponds to rank ... 483 // e.g : (in this example) 484 // proc 0 : 1--4 485 // proc 1 : 5--8 ... 486 GPID = startpart + i; 487 488 // Write solution field ... 489 isize = (*nshg)*(*numVars); 490 nitems = 3; 491 iarray[ 0 ] = (*nshg); 492 iarray[ 1 ] = (*numVars); 493 iarray[ 2 ] = (*stepno); 494 495 //MR CHANGE 496 // Measure the time - Start the timer 497 // MPI_Barrier(MPI_COMM_WORLD); 498 // timer_start = rdtsc(); 499 //MR CHANGE END 500 501 phio_writeheader( &f_descriptor, "time derivative of solution", 502 (void*)iarray, &nitems, &isize, "double", phasta_iotype); 503 504 //MR CHANGE 505 // Measure the time - End of timer 506 // MPI_Barrier(MPI_COMM_WORLD); 507 // timer_end = rdtsc(); 508 // time_span=(double)((timer_end-timer_start)/cpu_speed); 509 // if (*pid==0) { 510 // printf("Time: header for 'Time Derivative of solution': %f s\n",time_span); 511 // } 512 //MR CHANGE END 513 514 nitems = (*nshg)*(*numVars); 515 516 //MR CHANGE 517 // Measure the time - Start the timer 518 // MPI_Barrier(MPI_COMM_WORLD); 519 // timer_start = rdtsc(); 520 //MR CHANGE END 521 522 phio_writedatablock( &f_descriptor, "time derivative of solution", 523 (void*)(array2), &isize, "double", phasta_iotype ); 524 525 //MR CHANGE 526 // Measure the time - End of timer 527 // MPI_Barrier(MPI_COMM_WORLD); 528 // timer_end = rdtsc(); 529 // time_span=(double)((timer_end-timer_start)/cpu_speed); 530 531 // int isizemin,isizemax,isizetot; 532 // double sizemin,sizemax,sizeavg,sizetot,rate; 533 534 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 535 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 536 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 537 538 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 539 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 540 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 541 // sizeavg=sizetot/workfc.numpe; 542 // rate=sizetot/time_span; 543 544 // if (*pid==0) { 545 // printf("Time: block for 'Time Derivative of Solution': %f s\n",time_span); 546 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 547 // printf("*****************************\n"); 548 549 // } 550 //MR CHANGE END 551 552 } 553 field_flag++; 554 555 if (field_flag==nfields){ 556 557 //MR CHANGE 558 // Measure the time - Start the timer 559 // MPI_Barrier(MPI_COMM_WORLD); 560 // timer_start = rdtsc(); 561 //MR CHANGE END 562 563 //MR CHANGE 564 // Measure the time - End of timer 565 // MPI_Barrier(MPI_COMM_WORLD); 566 // timer_end = rdtsc(); 567 // time_span=(double)((timer_end-timer_start)/cpu_speed); 568 // if (*pid==0) { 569 // printf("\n*****************************\n"); 570 // printf("Time: 'closefile' is: %f s\n",time_span); 571 // } 572 //MR CHANGE END 573 574 //MR CHANGE 575 // Measure the time - Start the timer 576 // MPI_Barrier(MPI_COMM_WORLD); 577 // timer_start = rdtsc(); 578 //MR CHANGE END 579 580 phio_closefile_write(&f_descriptor); 581 582 //MR CHANGE 583 // Measure the time - End of timer 584 // MPI_Barrier(MPI_COMM_WORLD); 585 // timer_end = rdtsc(); 586 // time_span=(double)((timer_end-timer_start)/cpu_speed); 587 if (*pid==0) { 588 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 589 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 590 printf("\n"); 591 // printf("*****************************\n"); 592 } 593 } 594 //MR CHANGE END 595 596 597 598 /////////////////////////////////////////////////////////////////////////////////////////// 599 600 /* create a soft link of the restart we just wrote to restart.latest 601 this is the file the next run will always try to start from */ 602 603 /* sprintf( linkfile, "restart.latest.%d", *pid+1 ); 604 unlink( linkfile ); 605 sprintf( existingfile, "restart.%d.%d", *stepno, *pid+1 ); 606 link( existingfile, linkfile ); 607 */ 608 } 609 610 void 611 Write_Error( int* pid, 612 int* stepno, 613 int* nshg, 614 int* numVars, 615 double* array1 ) { 616 617 618 char fname[255]; 619 char rfile[60]; 620 int irstou; 621 int magic_number = 362436; 622 int* mptr = &magic_number; 623 //printf("Time is commented\n"); 624 //time_t timenow = time ( &timenow); 625 //printf("Yes\n"); 626 double version=0.0; 627 int isize, nitems; 628 int iarray[10]; 629 630 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 631 openfile_(rfile,"append", &irstou); 632 633 isize = (*nshg)*(*numVars); 634 nitems = 3; 635 iarray[ 0 ] = (*nshg); 636 iarray[ 1 ] = (*numVars); 637 iarray[ 2 ] = (*stepno); 638 writeheader_( &irstou, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 639 640 641 nitems = (*nshg)*(*numVars); 642 writedatablock_( &irstou, "errors ", (void*)(array1), &nitems, "double", phasta_iotype ); 643 644 closefile_( &irstou, "append" );*/ 645 646 /////////////////////////////// Start of writing using new-lib //////////////////////////// 647 648 int nfiles; 649 int nfields; 650 int numparts; 651 int irank; 652 int nprocs; 653 654 // unsigned long long timer_start; 655 // unsigned long long timer_end; 656 // double time_span; 657 658 nfiles = outpar.nsynciofiles; 659 nfields = outpar.nsynciofieldswriterestart; 660 numparts = workfc.numpe; 661 irank = *pid; //workfc.myrank; 662 nprocs = workfc.numpe; 663 664 int nppf = numparts/nfiles; 665 int GPID; 666 667 // Calculate number of parts each proc deal with and where it start and end ... 668 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 669 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 670 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 671 672 field_flag++; 673 674 int i; 675 for ( i = 0; i < nppp; i++ ) { 676 GPID = startpart + i; 677 678 if(*pid==0) { 679 // printf("\n*****************************\n"); 680 printf("\n"); 681 printf("The %d/%d th field to be written is 'errors'\n",field_flag,nfields); 682 } 683 684 isize = (*nshg)*(*numVars); 685 nitems = 3; 686 iarray[ 0 ] = (*nshg); 687 iarray[ 1 ] = (*numVars); 688 iarray[ 2 ] = (*stepno); 689 690 //MR CHANGE 691 // Measure the time - Start the timer 692 // MPI_Barrier(MPI_COMM_WORLD); 693 // timer_start = rdtsc(); 694 //MR CHANGE END 695 696 phio_writeheader( &f_descriptor, "errors", (void*)iarray, &nitems, 697 &isize, "double", phasta_iotype); 698 699 //MR CHANGE 700 // Measure the time - End of timer 701 // MPI_Barrier(MPI_COMM_WORLD); 702 // timer_end = rdtsc(); 703 // time_span=(double)((timer_end-timer_start)/cpu_speed); 704 // if (*pid==0) { 705 // printf("Time: header for 'error': %f s\n",time_span); 706 // } 707 //MR CHANGE END 708 709 //MR CHANGE 710 // Measure the time - Start the timer 711 // MPI_Barrier(MPI_COMM_WORLD); 712 // timer_start = rdtsc(); 713 //MR CHANGE END 714 715 phio_writedatablock( &f_descriptor, "errors", (void*)array1, &isize, 716 "double", phasta_iotype ); 717 718 //MR CHANGE 719 // Measure the time - End of timer 720 // MPI_Barrier(MPI_COMM_WORLD); 721 // timer_end = rdtsc(); 722 // time_span=(double)((timer_end-timer_start)/cpu_speed); 723 724 // int isizemin,isizemax,isizetot; 725 // double sizemin,sizemax,sizeavg,sizetot,rate; 726 727 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 728 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 729 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 730 731 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 732 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 733 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 734 // sizeavg=sizetot/workfc.numpe; 735 // rate=sizetot/time_span; 736 737 // if (*pid==0) { 738 // printf("Time: block for 'error': %f s\n",time_span); 739 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 740 // printf("*****************************\n"); 741 // } 742 //MR CHANGE END 743 744 } 745 746 // MPI_Barrier(MPI_COMM_WORLD); 747 // timer_end = rdtsc(); 748 // time_span=(double)(timer_end-timer_start)/cpu_speed; 749 750 // if (*pid==0) { 751 // printf("Field 'error' written in: %f s\n",time_span); 752 // printf("Write field '%s' finished! \n",fieldtag); 753 // } 754 755 if (field_flag==nfields){ 756 757 //MR CHANGE 758 // Measure the time - Start the timer 759 // MPI_Barrier(MPI_COMM_WORLD); 760 // timer_start = rdtsc(); 761 //MR CHANGE END 762 763 //MR CHANGE 764 // Measure the time - End of timer 765 // MPI_Barrier(MPI_COMM_WORLD); 766 // timer_end = rdtsc(); 767 // time_span=(double)((timer_end-timer_start)/cpu_speed); 768 // if (*pid==0) { 769 // printf("\n*****************************\n"); 770 // printf("Time: 'closefile' is: %f s\n",time_span); 771 // } 772 //MR CHANGE END 773 774 //MR CHANGE 775 // Measure the time - Start the timer 776 // MPI_Barrier(MPI_COMM_WORLD); 777 // timer_start = rdtsc(); 778 //MR CHANGE END 779 780 phio_closefile_write(&f_descriptor); 781 782 //MR CHANGE 783 // Measure the time - End of timer 784 // MPI_Barrier(MPI_COMM_WORLD); 785 // timer_end = rdtsc(); 786 // time_span=(double)((timer_end-timer_start)/cpu_speed); 787 if (*pid==0) { 788 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 789 printf("Last field %d 'errors' finished! \n",nfields); 790 printf("\n"); 791 // printf("*****************************\n"); 792 } 793 } 794 //MR CHANGE END 795 796 /////////////////////////////////////////////////////////////////////////////////////////// 797 798 799 } 800 801 802 void 803 Write_Displ( int* pid, 804 int* stepno, 805 int* nshg, 806 int* numVars, 807 double* array1 ) { 808 fprintf(stderr, "This function is dead...exiting\n"); 809 exit(1); 810 } 811 812 void 813 Write_Field( int *pid, 814 char* filemode, 815 char* fieldtag, 816 int* tagsize, 817 void* array, 818 char* arraytype, 819 int* nshg, 820 int* numvars, 821 int* stepno) { 822 823 //printf("Rank is %d, field is %s, tagsize is %d, nshg is %d, numvars is %d\n",*pid,fieldtag,*tagsize,*nshg,*numvars); 824 825 // char rfile[32]; 826 // assuming restart.sn.(pid+1) 827 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 828 829 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 830 strncpy(fieldlabel, fieldtag, *tagsize); 831 fieldlabel[*tagsize] = '\0'; 832 833 int irstou; 834 int magic_number = 362436; 835 int* mptr = &magic_number; 836 double version=0.0; 837 int isize, nitems; 838 int iarray[10]; 839 840 char fmode[10]; 841 if(!strncmp(filemode,"w",1)) 842 strcpy(fmode,"write"); 843 else // default is append 844 strcpy(fmode,"append"); 845 846 char datatype[10]; 847 if(!strncmp(arraytype,"i",1)) 848 strcpy(datatype,"int"); 849 else // default is double 850 strcpy(datatype,"double"); 851 852 /* openfile_(rfile, fmode, &irstou); 853 854 nitems = 3; // assuming field will write 3 items in iarray 855 iarray[ 0 ] = (*nshg); 856 iarray[ 1 ] = (*numvars); 857 iarray[ 2 ] = (*stepno); 858 859 isize = (*nshg)*(*numvars); 860 writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 861 862 nitems = (*nshg)*(*numvars); 863 writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 864 closefile_( &irstou, fmode); 865 */ 866 /////////////////////////////// Start of writing using new-lib //////////////////////////// 867 868 int nfiles; 869 int nfields; 870 int numparts; 871 int irank; 872 int nprocs; 873 874 // unsigned long long timer_start; 875 // unsigned long long timer_end; 876 // double time_span; 877 878 nfiles = outpar.nsynciofiles; 879 nfields = outpar.nsynciofieldswriterestart; 880 numparts = workfc.numpe; 881 irank = *pid; //workfc.myrank; 882 nprocs = workfc.numpe; 883 884 int nppf = numparts/nfiles; 885 int GPID; 886 887 // Calculate number of parts each proc deal with and where it start and end ... 888 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 889 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 890 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 891 892 char filename[255],path[255]; 893 bzero((void*)filename,255); 894 895 strncpy(fieldlabel, fieldtag, *tagsize); 896 897 field_flag++; 898 if(*pid==0) { 899 // printf("\n*****************************\n"); 900 printf("\n"); 901 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 902 } 903 904 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 905 906 // MPI_Barrier(MPI_COMM_WORLD); 907 // timer_start = rdtsc(); 908 909 int i; 910 for ( i = 0; i < nppp; i++ ) { 911 GPID = startpart + i; 912 913 // Write solution field ... 914 isize = (*nshg)*(*numvars); 915 nitems = 3; 916 iarray[ 0 ] = (*nshg); 917 iarray[ 1 ] = (*numvars); 918 iarray[ 2 ] = (*stepno); 919 920 //MR CHANGE 921 // Measure the time - Start the timer 922 // MPI_Barrier(MPI_COMM_WORLD); 923 // timer_start = rdtsc(); 924 //MR CHANGE END 925 926 phio_writeheader( &f_descriptor, fieldlabel, (void*)iarray, &nitems, 927 &isize, datatype, phasta_iotype); 928 929 //MR CHANGE 930 // Measure the time - End of timer 931 // MPI_Barrier(MPI_COMM_WORLD); 932 // timer_end = rdtsc(); 933 // time_span=(double)((timer_end-timer_start)/cpu_speed); 934 // if (*pid==0) { 935 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 936 // } 937 //MR CHANGE END 938 939 nitems = (*nshg)*(*numvars); 940 941 //MR CHANGE 942 // Measure the time - Start the timer 943 // MPI_Barrier(MPI_COMM_WORLD); 944 // timer_start = rdtsc(); 945 //MR CHANGE END 946 947 phio_writedatablock( &f_descriptor, fieldlabel, array, &isize, 948 datatype, phasta_iotype ); 949 950 //MR CHANGE 951 // Measure the time - End of timer 952 // MPI_Barrier(MPI_COMM_WORLD); 953 // timer_end = rdtsc(); 954 // time_span=(double)((timer_end-timer_start)/cpu_speed); 955 956 // int isizemin,isizemax,isizetot; 957 // double sizemin,sizemax,sizeavg,sizetot,rate; 958 959 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 960 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 961 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 962 963 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 964 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 965 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 966 // sizeavg=sizetot/workfc.numpe; 967 // rate=sizetot/time_span; 968 969 // if (*pid==0) { 970 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 971 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 972 // printf("*****************************\n"); 973 // } 974 //MR CHANGE END 975 976 } 977 978 // MPI_Barrier(MPI_COMM_WORLD); 979 // timer_end = rdtsc(); 980 // time_span=(double)(timer_end-timer_start)/cpu_speed; 981 982 // if (*pid==0) { 983 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 984 // printf("Write field '%s' finished! \n",fieldtag_s); 985 // } 986 987 // if (field_flag==nfields){ 988 // closefile_(&f_descriptor, "write"); 989 // finalizephmpiio_(&f_descriptor); 990 // if(*pid==0) { 991 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 992 // printf("\n*****************************\n"); 993 // } 994 // } 995 996 if (field_flag==nfields){ 997 998 //MR CHANGE 999 // Measure the time - Start the timer 1000 // MPI_Barrier(MPI_COMM_WORLD); 1001 // timer_start = rdtsc(); 1002 //MR CHANGE END 1003 1004 //MR CHANGE 1005 // Measure the time - End of timer 1006 // MPI_Barrier(MPI_COMM_WORLD); 1007 // timer_end = rdtsc(); 1008 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1009 // if (*pid==0) { 1010 // printf("\n*****************************\n"); 1011 // printf("Time: 'closefile' is: %f s\n",time_span); 1012 // } 1013 //MR CHANGE END 1014 1015 //MR CHANGE 1016 // Measure the time - Start the timer 1017 // MPI_Barrier(MPI_COMM_WORLD); 1018 // timer_start = rdtsc(); 1019 //MR CHANGE END 1020 1021 phio_closefile_write(&f_descriptor); 1022 1023 //MR CHANGE 1024 // Measure the time - End of timer 1025 // MPI_Barrier(MPI_COMM_WORLD); 1026 // timer_end = rdtsc(); 1027 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1028 if (*pid==0) { 1029 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1030 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1031 printf("\n"); 1032 // printf("*****************************\n"); 1033 } 1034 } 1035 //MR CHANGE END 1036 1037 /////////////////////////////////////////////////////////////////////////////////////////// 1038 1039 free(fieldlabel); 1040 } 1041 1042 //MR CHANGE 1043 1044 //MR CHANGE 1045 void 1046 Write_PhAvg2( int* pid, 1047 char* filemode, 1048 char* fieldtag, 1049 int* tagsize, 1050 int* iphase, 1051 int* nphasesincycle, 1052 void* array, 1053 char* arraytype, 1054 int* nshg, 1055 int* numvars, 1056 int* stepno) { 1057 1058 // char rfile[32]; 1059 // assuming restart.sn.(pid+1) 1060 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 1061 1062 int addtagsize=0; // phase number is added to the name of the field 1063 if(*iphase<10) 1064 addtagsize=1; 1065 else if(*iphase<100) 1066 addtagsize=2; 1067 else if(*iphase<1000) 1068 addtagsize=3; 1069 1070 int tagsize2; 1071 tagsize2=*tagsize+addtagsize; 1072 1073 // char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 1074 // strncpy(fieldlabel, fieldtag, *tagsize); 1075 // fieldlabel[*tagsize] = '\0'; 1076 1077 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 1078 strncpy(fieldlabel, fieldtag, *tagsize); 1079 fieldlabel[tagsize2] = '\0'; 1080 1081 char straddtagsize[10]; 1082 sprintf(straddtagsize,"%d",*iphase); 1083 1084 if(*iphase<10) { 1085 fieldlabel[tagsize2-1]=straddtagsize[0]; 1086 } 1087 else if(*iphase<100) { 1088 fieldlabel[tagsize2-2]=straddtagsize[0]; 1089 fieldlabel[tagsize2-1]=straddtagsize[1]; 1090 } 1091 else if(*iphase<1000) { 1092 fieldlabel[tagsize2-3]=straddtagsize[0]; 1093 fieldlabel[tagsize2-2]=straddtagsize[1]; 1094 fieldlabel[tagsize2-1]=straddtagsize[2]; 1095 } 1096 1097 int irstou; 1098 int magic_number = 362436; 1099 int* mptr = &magic_number; 1100 double version=0.0; 1101 int isize, nitems; 1102 int iarray[10]; 1103 1104 char fmode[10]; 1105 if(!strncmp(filemode,"w",1)) 1106 strcpy(fmode,"write"); 1107 else // default is append 1108 strcpy(fmode,"append"); 1109 1110 char datatype[10]; 1111 if(!strncmp(arraytype,"i",1)) 1112 strcpy(datatype,"int"); 1113 else // default is double 1114 strcpy(datatype,"double"); 1115 1116 // 1117 // // if(*iphase==1) //open the file but then keep it open for the remaining cycles 1118 // openfile_(rfile, fmode, &irstou); 1119 // 1120 // // printf("iphase: %d - pid: %d - irstou %d\n",*iphase,*pid,irstou); 1121 // 1122 // 1123 // nitems = 3; // assuming field will write 3 items in iarray 1124 // iarray[ 0 ] = (*nshg); 1125 // iarray[ 1 ] = (*numvars); 1126 // iarray[ 2 ] = (*stepno); 1127 // 1128 // isize = (*nshg)*(*numvars); 1129 // writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 1130 // 1131 // nitems = (*nshg)*(*numvars); 1132 // writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 1133 // 1134 // // if(*iphase==*nphasesincycle) //close the file after nphasesincycle 1135 // closefile_( &irstou, fmode); 1136 // 1137 1138 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1139 1140 int nfiles; 1141 int nfields; 1142 int numparts; 1143 int irank; 1144 int nprocs; 1145 // unsigned long long timer_start; 1146 // unsigned long long timer_end; 1147 // double time_span; 1148 1149 nfiles = outpar.nsynciofiles; 1150 nfields = outpar.nsynciofieldswriterestart; 1151 numparts = workfc.numpe; 1152 irank = *pid; //workfc.myrank; 1153 nprocs = workfc.numpe; 1154 1155 int nppf = numparts/nfiles; 1156 int GPID; 1157 1158 // Calculate number of parts each proc deal with and where it start and end ... 1159 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1160 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1161 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1162 1163 //int descriptor; 1164 char filename[255],path[255]; 1165 bzero((void*)filename,255); 1166 1167 // char * namer; 1168 // namer = strtok(fieldlabel," "); 1169 // strncpy(fieldlabel, fieldtag, *tagsize); 1170 1171 field_flag++; 1172 if(*pid==0) { 1173 // printf("\n*****************************\n"); 1174 printf("\n"); 1175 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 1176 } 1177 1178 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 1179 1180 int i; 1181 for ( i = 0; i < nppp; i++ ) { 1182 GPID = startpart + i; 1183 1184 // Write solution field ... 1185 //printf("This is %d and fieldtag_s is %s \n",myrank,fieldtag_s); 1186 1187 isize = (*nshg)*(*numvars); 1188 nitems = 3; 1189 iarray[ 0 ] = (*nshg); 1190 iarray[ 1 ] = (*numvars); 1191 iarray[ 2 ] = (*stepno); 1192 1193 //MR CHANGE 1194 // Measure the time - Start the timer 1195 // MPI_Barrier(MPI_COMM_WORLD); 1196 // timer_start = rdtsc(); 1197 //MR CHANGE END 1198 1199 phio_writeheader( &f_descriptor, fieldlabel, (void*)iarray, &nitems, 1200 &isize, "double", phasta_iotype); 1201 1202 //MR CHANGE 1203 // Measure the time - End of timer 1204 // MPI_Barrier(MPI_COMM_WORLD); 1205 // timer_end = rdtsc(); 1206 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1207 // if (*pid==0) { 1208 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 1209 // } 1210 //MR CHANGE END 1211 1212 nitems = (*nshg)*(*numvars); 1213 1214 //MR CHANGE 1215 // Measure the time - Start the timer 1216 // MPI_Barrier(MPI_COMM_WORLD); 1217 // timer_start = rdtsc(); 1218 //MR CHANGE END 1219 1220 phio_writedatablock( &f_descriptor, fieldlabel, array, &isize, 1221 "double", phasta_iotype ); 1222 1223 //MR CHANGE 1224 // Measure the time - End of timer 1225 // MPI_Barrier(MPI_COMM_WORLD); 1226 // timer_end = rdtsc(); 1227 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1228 1229 // int isizemin,isizemax,isizetot; 1230 // double sizemin,sizemax,sizeavg,sizetot,rate; 1231 1232 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 1233 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 1234 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 1235 1236 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 1237 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 1238 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 1239 // sizeavg=sizetot/workfc.numpe; 1240 // rate=sizetot/time_span; 1241 1242 // if (*pid==0) { 1243 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 1244 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 1245 // printf("*****************************\n"); 1246 // } 1247 //MR CHANGE END 1248 1249 } 1250 1251 // if (*pid==0) { 1252 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 1253 // printf("Write field '%s' finished! \n",fieldtag_s); 1254 // } 1255 1256 // 1257 // if (field_flag==nfields){ 1258 // closefile_(&f_descriptor, "write"); 1259 // finalizephmpiio_(&f_descriptor); 1260 // if(*pid==0) { 1261 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1262 // printf("\n*****************************\n"); 1263 // } 1264 // } 1265 1266 if (field_flag==nfields){ 1267 1268 //MR CHANGE 1269 // Measure the time - Start the timer 1270 // MPI_Barrier(MPI_COMM_WORLD); 1271 // timer_start = rdtsc(); 1272 //MR CHANGE END 1273 1274 //MR CHANGE 1275 // Measure the time - End of timer 1276 // MPI_Barrier(MPI_COMM_WORLD); 1277 // timer_end = rdtsc(); 1278 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1279 // if (*pid==0) { 1280 // printf("\n*****************************\n"); 1281 // printf("Time: 'closefile' is: %f s\n",time_span); 1282 // } 1283 //MR CHANGE END 1284 1285 //MR CHANGE 1286 // Measure the time - Start the timer 1287 // MPI_Barrier(MPI_COMM_WORLD); 1288 // timer_start = rdtsc(); 1289 //MR CHANGE END 1290 1291 phio_closefile_write(&f_descriptor); 1292 1293 //MR CHANGE 1294 // Measure the time - End of timer 1295 // MPI_Barrier(MPI_COMM_WORLD); 1296 // timer_end = rdtsc(); 1297 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1298 if (*pid==0) { 1299 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1300 // printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1301 printf("\n"); 1302 // printf("*****************************\n"); 1303 } 1304 } 1305 //MR CHANGE END 1306 1307 /////////////////////////////////////////////////////////////////////////////////////////// 1308 1309 free(fieldlabel); 1310 } 1311 1312 1313 void 1314 Write_d2wall( int* pid, 1315 int* numnp, 1316 double* array1 ) { 1317 1318 // time_t timenow = time ( &timenow); 1319 int isize, nitems; 1320 int iarray[10]; 1321 1322 // MPI_Barrier(MPI_COMM_WORLD); 1323 1324 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1325 1326 int nfiles; 1327 int nfields; 1328 int numparts; 1329 int irank; 1330 int nprocs; 1331 1332 // First, count the number of fields to write and store the result in 1333 //countfieldstowriterestart(); 1334 1335 // Retrieve and compute the parameters required for SyncIO 1336 nfiles = outpar.nsynciofiles; 1337 nfields = 1; //outpar.nsynciofieldswriterestart; // Only the distance to the walls in d2wall 1338 numparts = workfc.numpe; 1339 irank = *pid; //workfc.myrank; 1340 nprocs = workfc.numpe; 1341 int nppf = numparts/nfiles; 1342 int GPID; 1343 1344 // Calculate number of parts each proc deal with and where it start and end ... 1345 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1346 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1347 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1348 1349 int descriptor; 1350 char filename[255],path[255]; 1351 bzero((void*)filename,255); 1352 1353 phio_openfile_write("d2wall.", &nfiles, &nfields, &nppf, &f_descriptor); 1354 1355 field_flag=0; 1356 1357 int i; 1358 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 1359 // GPID : global part id, corresponds to rank ... 1360 // e.g : (in this example) 1361 // proc 0 : 1--4 1362 // proc 1 : 5--8 ... 1363 GPID = startpart + i; 1364 1365 // Write solution field ... 1366 isize = (*numnp); 1367 nitems = 2; 1368 iarray[ 0 ] = (*numnp); 1369 iarray[ 1 ] = 1; //numVars = 1 1370 1371 phio_writeheader( &f_descriptor, "d2wall", (void*)iarray, &nitems, 1372 &isize, "double", phasta_iotype); 1373 1374 //nitems = (*nshg)*(*numVars); 1375 //nitems = (*numnp); 1376 1377 phio_writedatablock( &f_descriptor, "d2wall", (void*)(array1), &isize, 1378 "double", phasta_iotype ); 1379 } 1380 field_flag++; 1381 1382 if (field_flag==nfields){ 1383 phio_closefile_write(&f_descriptor); 1384 if (irank==0) { 1385 printf("\n"); 1386 } 1387 } 1388 } 1389 1390