1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdio.h> 7 #include <string.h> 8 #include <ctype.h> 9 #include <stdlib.h> 10 #include <time.h> 11 #include <math.h> 12 #include "mpi.h" 13 #include "phastaIO.h" 14 #include "rdtsc.h" 15 #include <FCMangle.h> 16 #include "new_interface.h" 17 #include "phIO.h" 18 19 //MR CHANGE 20 #include "common_c.h" 21 //MR CHANGE END 22 23 #ifdef intel 24 #include <winsock2.h> 25 #else 26 #include <unistd.h> 27 #include <strings.h> 28 #endif 29 30 //extern double cpu_speed = 2600000000.0; //for Jaguar XT5 31 //extern double cpu_speed = 2100000000.0; //for Jaguar xt4 32 //extern double cpu_speed = 850000000.0; //for Intrepid 33 34 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 35 int isThisRank; 36 37 double *value = (double*)malloc(sizeof(double)); 38 *value = 1.0*(*ivalue); 39 40 rgetMinMaxAvg(value,stats,statRanks); 41 42 /* MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 43 isThisRank=workfc.numpe+1; 44 if(*value==stats[0]) 45 isThisRank=workfc.myrank; 46 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 47 48 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 49 isThisRank=workfc.numpe+1; 50 if(*value==stats[1]) 51 isThisRank=workfc.myrank; 52 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 53 54 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 55 stats[2] /= workfc.numpe; */ 56 57 free(value); 58 } 59 60 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 61 int isThisRank; 62 63 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 64 isThisRank=workfc.numpe+1; 65 if(*value==stats[0]) 66 isThisRank=workfc.myrank; 67 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 68 69 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 70 isThisRank=workfc.numpe+1; 71 if(*value==stats[1]) 72 isThisRank=workfc.myrank; 73 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 74 75 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 76 stats[2] /= workfc.numpe; 77 78 double sqValue = (*value)*(*value), sqValueAvg = 0.; 79 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 80 sqValueAvg /= workfc.numpe; 81 // stats[3] = sqValueAvg; 82 83 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 84 } 85 86 void print_mesh_stats(void) { 87 int statRanks[2]; 88 double iStats[4], rStats[4]; 89 90 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 91 if(workfc.myrank==workfc.master) 92 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 93 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 94 if(workfc.myrank==workfc.master) 95 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 96 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 97 if(workfc.myrank==workfc.master) 98 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 99 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 100 if(workfc.myrank==workfc.master) { 101 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 102 printf("\n"); 103 } 104 } 105 106 void print_mpi_stats(void) { 107 int statRanks[2]; 108 double iStats[4], rStats[4]; 109 110 // NS equations 111 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 112 if(workfc.myrank==workfc.master) 113 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 114 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 115 if(workfc.myrank==workfc.master) 116 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 117 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 118 if(workfc.myrank==workfc.master) 119 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 120 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 121 if(workfc.myrank==workfc.master) 122 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 123 124 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 125 if(workfc.myrank==workfc.master) 126 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 127 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 128 if(workfc.myrank==workfc.master) 129 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 130 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 131 if(workfc.myrank==workfc.master) 132 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 133 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 134 if(workfc.myrank==workfc.master) 135 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 136 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 137 if(workfc.myrank==workfc.master) { 138 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 139 printf("\n"); 140 } 141 // Scalars 142 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 143 if(workfc.myrank==workfc.master) 144 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 145 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 146 if(workfc.myrank==workfc.master) 147 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 148 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 149 if(workfc.myrank==workfc.master) 150 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 151 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 152 if(workfc.myrank==workfc.master) 153 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 154 155 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 156 if(workfc.myrank==workfc.master) 157 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 158 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 159 if(workfc.myrank==workfc.master) 160 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 161 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 162 if(workfc.myrank==workfc.master) 163 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 164 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 165 if(workfc.myrank==workfc.master) 166 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 167 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 168 if(workfc.myrank==workfc.master) 169 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 170 171 172 } 173 174 //void print_system_stats(double tcorecp[2]) { 175 void print_system_stats(double *tcorecp, double *tcorecpscal) { 176 int statRanks[2]; 177 double iStats[4], rStats[4]; 178 double syst_assembly, syst_solve; 179 180 // NS equations 181 syst_assembly = tcorecp[0]; 182 syst_solve = tcorecp[1]; 183 184 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 185 if(workfc.myrank==workfc.master) 186 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 187 188 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 189 if(workfc.myrank==workfc.master) 190 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 191 192 // Scalars 193 syst_assembly = tcorecpscal[0]; 194 syst_solve = tcorecpscal[1]; 195 196 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 197 if(workfc.myrank==workfc.master) 198 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 199 200 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 201 if(workfc.myrank==workfc.master) { 202 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 203 printf("\n"); 204 } 205 //printf("rank %d - syst_assembly %f - syst_solve %f\n",workfc.myrank,syst_assembly,syst_solve); 206 } 207 208 209 210 void countfieldstowriterestart() 211 { 212 int nfields; 213 214 // printf("TEST: %d %d %d %d %d\n",timdat.istep,timdat.itseq,inpdat.nstep[0],inpdat.nstep[1],timdat.lstep); 215 216 nfields = 2; //solution, time derivatives 217 218 if(outpar.ivort == 1){ 219 nfields++; //vorticity 220 } 221 222 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 223 nfields++; //instantaneous wss in bflux.f 224 } 225 226 // if(ideformwall.eq.1) not handled yet 227 228 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 229 230 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 231 if(matdat.matflg[0][0] ==-1) { 232 nfields = nfields +2; 233 } 234 235 //if Print Error Indicators = true (call write_error in itrdrv) 236 if(turbvar.ierrcalc == 1){ 237 nfields++; 238 } 239 240 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 241 if(outpar.ioybar == 1){ 242 nfields++; //ybar 243 244 //phase average fields 245 if(outpar.nphasesincycle >0) { 246 nfields = nfields + outpar.nphasesincycle; 247 } 248 249 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 250 nfields++; //wssbar 251 } 252 253 } 254 255 if(turbvari.irans < 0) { 256 nfields++; //dwal 257 } 258 259 } 260 261 outpar.nsynciofieldswriterestart = nfields; 262 263 if(workfc.myrank == 0) { 264 printf("Number of fields to write in restart files: %d\n", nfields); 265 } 266 } 267 268 269 void 270 Write_Restart( int* pid, 271 int* stepno, 272 int* nshg, 273 int* numVars, 274 double* array1, 275 double* array2 ) { 276 277 char fname[255]; 278 char rfile[60]; 279 char existingfile[30], linkfile[30]; 280 int irstou; 281 int magic_number = 362436; 282 int* mptr = &magic_number; 283 // time_t timenow = time ( &timenow); 284 double version=0.0; 285 int isize, nitems; 286 int iarray[10]; 287 288 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 289 openfile_(rfile,"write", &irstou); 290 291 // writing the top ascii header for the restart file 292 293 writestring_( &irstou,"# PHASTA Input File Version 2.0\n"); 294 writestring_( &irstou, 295 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 296 297 bzero( (void*)fname, 255 ); 298 sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); 299 writestring_( &irstou, fname ); 300 301 bzero( (void*)fname, 255 ); 302 gethostname(fname,255); 303 writestring_( &irstou,"# This result was produced on: "); 304 writestring_( &irstou, fname ); 305 writestring_( &irstou,"\n"); 306 307 bzero( (void*)fname, 255 ); 308 sprintf(fname,"# %s\n", ctime( &timenow )); 309 writestring_( &irstou, fname ); 310 311 isize = 1; 312 nitems = 1; 313 iarray[ 0 ] = 1; 314 writeheader_( &irstou, "byteorder magic number ", 315 (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); 316 317 nitems = 1; 318 writedatablock_( &irstou, "byteorder magic number ", 319 (void*)mptr, &nitems, "integer", phasta_iotype ); 320 321 322 bzero( (void*)fname, 255 ); 323 sprintf(fname,"number of modes : < 0 > %d\n", *nshg); 324 writestring_( &irstou, fname ); 325 326 bzero( (void*)fname, 255 ); 327 sprintf(fname,"number of variables : < 0 > %d\n", *numVars); 328 writestring_( &irstou, fname ); 329 330 331 isize = (*nshg)*(*numVars); 332 nitems = 3; 333 iarray[ 0 ] = (*nshg); 334 iarray[ 1 ] = (*numVars); 335 iarray[ 2 ] = (*stepno); 336 writeheader_( &irstou, "solution ", 337 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 338 339 340 nitems = (*nshg)*(*numVars); 341 writedatablock_( &irstou, "solution ", 342 (void*)(array1), &nitems, "double", phasta_iotype ); 343 344 345 346 nitems = 3; 347 writeheader_( &irstou, "time derivative of solution ", 348 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 349 350 351 nitems = (*nshg)*(*numVars); 352 writedatablock_( &irstou, "time derivative of solution ", 353 (void*)(array2), &nitems, "double", phasta_iotype ); 354 355 356 closefile_( &irstou, "write" ); 357 */ 358 //MPI_Barrier(MPI_COMM_WORLD); 359 360 /////////////////////////////// Start of writing using new-lib //////////////////////////// 361 362 //MR CHANGE 363 int nfiles; 364 int nfields; 365 int numparts; 366 int irank; 367 int nprocs; 368 int nppf; 369 370 // First, count the number of fields to write and store the result in 371 countfieldstowriterestart(); 372 373 // Retrieve and compute the parameters required for SyncIO 374 nfiles = outpar.nsynciofiles; 375 nfields = outpar.nsynciofieldswriterestart; 376 numparts = workfc.numpe; 377 irank = *pid; //workfc.myrank; 378 nprocs = workfc.numpe; 379 int nppp = numparts/nprocs; // always 1 for PHASTA 380 //MR CHANGE END 381 382 int descriptor; 383 char filename[255]; 384 bzero((void*)filename,255); 385 386 if(nfiles == 0 ){ 387 sprintf(filename,"restart.%d.", *stepno); 388 nppf=1; 389 } else { 390 nppf=numparts/nfiles; 391 sprintf(filename,"restart-dat.%d.", *stepno); 392 } 393 phio_openfile_write(filename, &nfiles, &nfields, &nppf, &f_descriptor); 394 395 //MR CHANGE 396 // Measure the time - End of timer 397 // MPI_Barrier(MPI_COMM_WORLD); 398 // timer_end = rdtsc(); 399 // time_span=(double)((timer_end-timer_start)/cpu_speed); 400 // if (*pid==0) { 401 // printf("Time: 'openfile' of %s with %d fields and %d files is: %f s\n",filename,nfields,nfiles,time_span); 402 // printf("*****************************\n"); 403 // } 404 //MR CHANGE END 405 406 field_flag=0; 407 408 int i; 409 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 410 // Write solution field ... 411 isize = (*nshg)*(*numVars); 412 nitems = 3; 413 iarray[ 0 ] = (*nshg); 414 iarray[ 1 ] = (*numVars); 415 iarray[ 2 ] = (*stepno); 416 417 //MR CHANGE 418 // Measure the time - Start the timer 419 // MPI_Barrier(MPI_COMM_WORLD); 420 // timer_start = rdtsc(); 421 //MR CHANGE END 422 423 phio_writeheader(f_descriptor, "solution", (void*)iarray, &nitems, 424 &isize, "double", phasta_iotype); 425 426 //MR CHANGE 427 // Measure the time - End of timer 428 // MPI_Barrier(MPI_COMM_WORLD); 429 // timer_end = rdtsc(); 430 // time_span=(double)((timer_end-timer_start)/cpu_speed); 431 // if (*pid==0) { 432 // printf("\n*****************************\n"); 433 // printf("Time: header for 'Solution': %f s\n",time_span); 434 // } 435 //MR CHANGE END 436 437 nitems = (*nshg)*(*numVars); 438 439 //MR CHANGE 440 // Measure the time - Start the timer 441 // MPI_Barrier(MPI_COMM_WORLD); 442 // timer_start = rdtsc(); 443 //MR CHANGE END 444 445 phio_writedatablock(f_descriptor, "solution", (void*)(array1), 446 &isize, "double", phasta_iotype ); 447 448 //MR CHANGE 449 // Measure the time - End of timer 450 // MPI_Barrier(MPI_COMM_WORLD); 451 // timer_end = rdtsc(); 452 // time_span=(double)((timer_end-timer_start)/cpu_speed); 453 454 // int isizemin,isizemax,isizetot; 455 // double sizemin,sizemax,sizeavg,sizetot,rate; 456 457 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 458 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 459 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 460 461 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 462 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 463 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 464 // sizeavg=(double)(1.0*sizetot/workfc.numpe); 465 // rate=(double)(1.0*sizetot/time_span); 466 467 // if (*pid==0) { 468 // printf("Time: block for 'Solution': %f s\n",time_span); 469 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 470 471 // } 472 //MR CHANGE END 473 474 } 475 field_flag++; 476 477 for ( i = 0; i < nppp; i++) { 478 479 480 // Write solution field ... 481 isize = (*nshg)*(*numVars); 482 nitems = 3; 483 iarray[ 0 ] = (*nshg); 484 iarray[ 1 ] = (*numVars); 485 iarray[ 2 ] = (*stepno); 486 487 //MR CHANGE 488 // Measure the time - Start the timer 489 // MPI_Barrier(MPI_COMM_WORLD); 490 // timer_start = rdtsc(); 491 //MR CHANGE END 492 493 phio_writeheader(f_descriptor, "time derivative of solution", 494 (void*)iarray, &nitems, &isize, "double", phasta_iotype); 495 496 //MR CHANGE 497 // Measure the time - End of timer 498 // MPI_Barrier(MPI_COMM_WORLD); 499 // timer_end = rdtsc(); 500 // time_span=(double)((timer_end-timer_start)/cpu_speed); 501 // if (*pid==0) { 502 // printf("Time: header for 'Time Derivative of solution': %f s\n",time_span); 503 // } 504 //MR CHANGE END 505 506 nitems = (*nshg)*(*numVars); 507 508 //MR CHANGE 509 // Measure the time - Start the timer 510 // MPI_Barrier(MPI_COMM_WORLD); 511 // timer_start = rdtsc(); 512 //MR CHANGE END 513 514 phio_writedatablock(f_descriptor, "time derivative of solution", 515 (void*)(array2), &isize, "double", phasta_iotype ); 516 517 //MR CHANGE 518 // Measure the time - End of timer 519 // MPI_Barrier(MPI_COMM_WORLD); 520 // timer_end = rdtsc(); 521 // time_span=(double)((timer_end-timer_start)/cpu_speed); 522 523 // int isizemin,isizemax,isizetot; 524 // double sizemin,sizemax,sizeavg,sizetot,rate; 525 526 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 527 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 528 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 529 530 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 531 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 532 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 533 // sizeavg=sizetot/workfc.numpe; 534 // rate=sizetot/time_span; 535 536 // if (*pid==0) { 537 // printf("Time: block for 'Time Derivative of Solution': %f s\n",time_span); 538 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 539 // printf("*****************************\n"); 540 541 // } 542 //MR CHANGE END 543 544 } 545 field_flag++; 546 547 if (field_flag==nfields){ 548 549 //MR CHANGE 550 // Measure the time - Start the timer 551 // MPI_Barrier(MPI_COMM_WORLD); 552 // timer_start = rdtsc(); 553 //MR CHANGE END 554 555 //MR CHANGE 556 // Measure the time - End of timer 557 // MPI_Barrier(MPI_COMM_WORLD); 558 // timer_end = rdtsc(); 559 // time_span=(double)((timer_end-timer_start)/cpu_speed); 560 // if (*pid==0) { 561 // printf("\n*****************************\n"); 562 // printf("Time: 'closefile' is: %f s\n",time_span); 563 // } 564 //MR CHANGE END 565 566 //MR CHANGE 567 // Measure the time - Start the timer 568 // MPI_Barrier(MPI_COMM_WORLD); 569 // timer_start = rdtsc(); 570 //MR CHANGE END 571 572 phio_closefile_write(f_descriptor); 573 574 //MR CHANGE 575 // Measure the time - End of timer 576 // MPI_Barrier(MPI_COMM_WORLD); 577 // timer_end = rdtsc(); 578 // time_span=(double)((timer_end-timer_start)/cpu_speed); 579 if (*pid==0) { 580 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 581 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 582 printf("\n"); 583 // printf("*****************************\n"); 584 } 585 } 586 //MR CHANGE END 587 588 589 590 /////////////////////////////////////////////////////////////////////////////////////////// 591 592 /* create a soft link of the restart we just wrote to restart.latest 593 this is the file the next run will always try to start from */ 594 595 /* sprintf( linkfile, "restart.latest.%d", *pid+1 ); 596 unlink( linkfile ); 597 sprintf( existingfile, "restart.%d.%d", *stepno, *pid+1 ); 598 link( existingfile, linkfile ); 599 */ 600 } 601 602 void 603 Write_Error( int* pid, 604 int* stepno, 605 int* nshg, 606 int* numVars, 607 double* array1 ) { 608 609 610 char fname[255]; 611 char rfile[60]; 612 int irstou; 613 int magic_number = 362436; 614 int* mptr = &magic_number; 615 //printf("Time is commented\n"); 616 //time_t timenow = time ( &timenow); 617 //printf("Yes\n"); 618 double version=0.0; 619 int isize, nitems; 620 int iarray[10]; 621 622 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 623 openfile_(rfile,"append", &irstou); 624 625 isize = (*nshg)*(*numVars); 626 nitems = 3; 627 iarray[ 0 ] = (*nshg); 628 iarray[ 1 ] = (*numVars); 629 iarray[ 2 ] = (*stepno); 630 writeheader_( &irstou, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 631 632 633 nitems = (*nshg)*(*numVars); 634 writedatablock_( &irstou, "errors ", (void*)(array1), &nitems, "double", phasta_iotype ); 635 636 closefile_( &irstou, "append" );*/ 637 638 /////////////////////////////// Start of writing using new-lib //////////////////////////// 639 640 int nfiles; 641 int nfields; 642 int numparts; 643 int irank; 644 int nprocs; 645 646 // unsigned long long timer_start; 647 // unsigned long long timer_end; 648 // double time_span; 649 650 nfiles = outpar.nsynciofiles; 651 nfields = outpar.nsynciofieldswriterestart; 652 numparts = workfc.numpe; 653 irank = *pid; //workfc.myrank; 654 nprocs = workfc.numpe; 655 656 657 // Calculate number of parts each proc deal with and where it start and end ... 658 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 659 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 660 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 661 662 field_flag++; 663 664 int i; 665 for ( i = 0; i < nppp; i++ ) { 666 667 if(*pid==0) { 668 // printf("\n*****************************\n"); 669 printf("\n"); 670 printf("The %d/%d th field to be written is 'errors'\n",field_flag,nfields); 671 } 672 673 isize = (*nshg)*(*numVars); 674 nitems = 3; 675 iarray[ 0 ] = (*nshg); 676 iarray[ 1 ] = (*numVars); 677 iarray[ 2 ] = (*stepno); 678 679 //MR CHANGE 680 // Measure the time - Start the timer 681 // MPI_Barrier(MPI_COMM_WORLD); 682 // timer_start = rdtsc(); 683 //MR CHANGE END 684 685 phio_writeheader(f_descriptor, "errors", (void*)iarray, &nitems, 686 &isize, "double", phasta_iotype); 687 688 //MR CHANGE 689 // Measure the time - End of timer 690 // MPI_Barrier(MPI_COMM_WORLD); 691 // timer_end = rdtsc(); 692 // time_span=(double)((timer_end-timer_start)/cpu_speed); 693 // if (*pid==0) { 694 // printf("Time: header for 'error': %f s\n",time_span); 695 // } 696 //MR CHANGE END 697 698 //MR CHANGE 699 // Measure the time - Start the timer 700 // MPI_Barrier(MPI_COMM_WORLD); 701 // timer_start = rdtsc(); 702 //MR CHANGE END 703 704 phio_writedatablock(f_descriptor, "errors", (void*)array1, &isize, 705 "double", phasta_iotype ); 706 707 //MR CHANGE 708 // Measure the time - End of timer 709 // MPI_Barrier(MPI_COMM_WORLD); 710 // timer_end = rdtsc(); 711 // time_span=(double)((timer_end-timer_start)/cpu_speed); 712 713 // int isizemin,isizemax,isizetot; 714 // double sizemin,sizemax,sizeavg,sizetot,rate; 715 716 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 717 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 718 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 719 720 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 721 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 722 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 723 // sizeavg=sizetot/workfc.numpe; 724 // rate=sizetot/time_span; 725 726 // if (*pid==0) { 727 // printf("Time: block for 'error': %f s\n",time_span); 728 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 729 // printf("*****************************\n"); 730 // } 731 //MR CHANGE END 732 733 } 734 735 // MPI_Barrier(MPI_COMM_WORLD); 736 // timer_end = rdtsc(); 737 // time_span=(double)(timer_end-timer_start)/cpu_speed; 738 739 // if (*pid==0) { 740 // printf("Field 'error' written in: %f s\n",time_span); 741 // printf("Write field '%s' finished! \n",fieldtag); 742 // } 743 744 if (field_flag==nfields){ 745 746 //MR CHANGE 747 // Measure the time - Start the timer 748 // MPI_Barrier(MPI_COMM_WORLD); 749 // timer_start = rdtsc(); 750 //MR CHANGE END 751 752 //MR CHANGE 753 // Measure the time - End of timer 754 // MPI_Barrier(MPI_COMM_WORLD); 755 // timer_end = rdtsc(); 756 // time_span=(double)((timer_end-timer_start)/cpu_speed); 757 // if (*pid==0) { 758 // printf("\n*****************************\n"); 759 // printf("Time: 'closefile' is: %f s\n",time_span); 760 // } 761 //MR CHANGE END 762 763 //MR CHANGE 764 // Measure the time - Start the timer 765 // MPI_Barrier(MPI_COMM_WORLD); 766 // timer_start = rdtsc(); 767 //MR CHANGE END 768 769 phio_closefile_write(f_descriptor); 770 771 //MR CHANGE 772 // Measure the time - End of timer 773 // MPI_Barrier(MPI_COMM_WORLD); 774 // timer_end = rdtsc(); 775 // time_span=(double)((timer_end-timer_start)/cpu_speed); 776 if (*pid==0) { 777 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 778 printf("Last field %d 'errors' finished! \n",nfields); 779 printf("\n"); 780 // printf("*****************************\n"); 781 } 782 } 783 //MR CHANGE END 784 785 /////////////////////////////////////////////////////////////////////////////////////////// 786 787 788 } 789 790 791 void 792 Write_Displ( int* pid, 793 int* stepno, 794 int* nshg, 795 int* numVars, 796 double* array1 ) { 797 fprintf(stderr, "This function is dead...exiting\n"); 798 exit(1); 799 } 800 801 void 802 Write_Field( int *pid, 803 char* filemode, 804 char* fieldtag, 805 int* tagsize, 806 void* array, 807 char* arraytype, 808 int* nshg, 809 int* numvars, 810 int* stepno) { 811 812 //printf("Rank is %d, field is %s, tagsize is %d, nshg is %d, numvars is %d\n",*pid,fieldtag,*tagsize,*nshg,*numvars); 813 814 // char rfile[32]; 815 // assuming restart.sn.(pid+1) 816 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 817 818 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 819 strncpy(fieldlabel, fieldtag, *tagsize); 820 fieldlabel[*tagsize] = '\0'; 821 822 int irstou; 823 int magic_number = 362436; 824 int* mptr = &magic_number; 825 double version=0.0; 826 int isize, nitems; 827 int iarray[10]; 828 829 char fmode[10]; 830 if(!strncmp(filemode,"w",1)) 831 strcpy(fmode,"write"); 832 else // default is append 833 strcpy(fmode,"append"); 834 835 char datatype[10]; 836 if(!strncmp(arraytype,"i",1)) 837 strcpy(datatype,"int"); 838 else // default is double 839 strcpy(datatype,"double"); 840 841 /* openfile_(rfile, fmode, &irstou); 842 843 nitems = 3; // assuming field will write 3 items in iarray 844 iarray[ 0 ] = (*nshg); 845 iarray[ 1 ] = (*numvars); 846 iarray[ 2 ] = (*stepno); 847 848 isize = (*nshg)*(*numvars); 849 writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 850 851 nitems = (*nshg)*(*numvars); 852 writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 853 closefile_( &irstou, fmode); 854 */ 855 /////////////////////////////// Start of writing using new-lib //////////////////////////// 856 857 int nfiles; 858 int nfields; 859 int numparts; 860 int irank; 861 int nprocs; 862 863 // unsigned long long timer_start; 864 // unsigned long long timer_end; 865 // double time_span; 866 867 nfiles = outpar.nsynciofiles; 868 nfields = outpar.nsynciofieldswriterestart; 869 numparts = workfc.numpe; 870 irank = *pid; //workfc.myrank; 871 nprocs = workfc.numpe; 872 873 874 // Calculate number of parts each proc deal with and where it start and end ... 875 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 876 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 877 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 878 879 880 strncpy(fieldlabel, fieldtag, *tagsize); 881 882 field_flag++; 883 if(*pid==0) { 884 // printf("\n*****************************\n"); 885 printf("\n"); 886 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 887 } 888 889 890 // MPI_Barrier(MPI_COMM_WORLD); 891 // timer_start = rdtsc(); 892 893 int i; 894 for ( i = 0; i < nppp; i++ ) { 895 896 // Write solution field ... 897 isize = (*nshg)*(*numvars); 898 nitems = 3; 899 iarray[ 0 ] = (*nshg); 900 iarray[ 1 ] = (*numvars); 901 iarray[ 2 ] = (*stepno); 902 903 //MR CHANGE 904 // Measure the time - Start the timer 905 // MPI_Barrier(MPI_COMM_WORLD); 906 // timer_start = rdtsc(); 907 //MR CHANGE END 908 909 phio_writeheader(f_descriptor, fieldlabel, (void*)iarray, &nitems, 910 &isize, datatype, phasta_iotype); 911 912 //MR CHANGE 913 // Measure the time - End of timer 914 // MPI_Barrier(MPI_COMM_WORLD); 915 // timer_end = rdtsc(); 916 // time_span=(double)((timer_end-timer_start)/cpu_speed); 917 // if (*pid==0) { 918 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 919 // } 920 //MR CHANGE END 921 922 nitems = (*nshg)*(*numvars); 923 924 //MR CHANGE 925 // Measure the time - Start the timer 926 // MPI_Barrier(MPI_COMM_WORLD); 927 // timer_start = rdtsc(); 928 //MR CHANGE END 929 930 phio_writedatablock(f_descriptor, fieldlabel, array, &isize, 931 datatype, phasta_iotype ); 932 933 //MR CHANGE 934 // Measure the time - End of timer 935 // MPI_Barrier(MPI_COMM_WORLD); 936 // timer_end = rdtsc(); 937 // time_span=(double)((timer_end-timer_start)/cpu_speed); 938 939 // int isizemin,isizemax,isizetot; 940 // double sizemin,sizemax,sizeavg,sizetot,rate; 941 942 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 943 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 944 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 945 946 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 947 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 948 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 949 // sizeavg=sizetot/workfc.numpe; 950 // rate=sizetot/time_span; 951 952 // if (*pid==0) { 953 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 954 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 955 // printf("*****************************\n"); 956 // } 957 //MR CHANGE END 958 959 } 960 961 // MPI_Barrier(MPI_COMM_WORLD); 962 // timer_end = rdtsc(); 963 // time_span=(double)(timer_end-timer_start)/cpu_speed; 964 965 // if (*pid==0) { 966 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 967 // printf("Write field '%s' finished! \n",fieldtag_s); 968 // } 969 970 // if (field_flag==nfields){ 971 // closefile_(&f_descriptor, "write"); 972 // finalizephmpiio_(&f_descriptor); 973 // if(*pid==0) { 974 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 975 // printf("\n*****************************\n"); 976 // } 977 // } 978 979 if (field_flag==nfields){ 980 981 //MR CHANGE 982 // Measure the time - Start the timer 983 // MPI_Barrier(MPI_COMM_WORLD); 984 // timer_start = rdtsc(); 985 //MR CHANGE END 986 987 //MR CHANGE 988 // Measure the time - End of timer 989 // MPI_Barrier(MPI_COMM_WORLD); 990 // timer_end = rdtsc(); 991 // time_span=(double)((timer_end-timer_start)/cpu_speed); 992 // if (*pid==0) { 993 // printf("\n*****************************\n"); 994 // printf("Time: 'closefile' is: %f s\n",time_span); 995 // } 996 //MR CHANGE END 997 998 //MR CHANGE 999 // Measure the time - Start the timer 1000 // MPI_Barrier(MPI_COMM_WORLD); 1001 // timer_start = rdtsc(); 1002 //MR CHANGE END 1003 1004 phio_closefile_write(f_descriptor); 1005 1006 //MR CHANGE 1007 // Measure the time - End of timer 1008 // MPI_Barrier(MPI_COMM_WORLD); 1009 // timer_end = rdtsc(); 1010 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1011 if (*pid==0) { 1012 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1013 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1014 printf("\n"); 1015 // printf("*****************************\n"); 1016 } 1017 } 1018 //MR CHANGE END 1019 1020 /////////////////////////////////////////////////////////////////////////////////////////// 1021 1022 free(fieldlabel); 1023 } 1024 1025 //MR CHANGE 1026 1027 //MR CHANGE 1028 void 1029 Write_PhAvg2( int* pid, 1030 char* filemode, 1031 char* fieldtag, 1032 int* tagsize, 1033 int* iphase, 1034 int* nphasesincycle, 1035 void* array, 1036 char* arraytype, 1037 int* nshg, 1038 int* numvars, 1039 int* stepno) { 1040 1041 // char rfile[32]; 1042 // assuming restart.sn.(pid+1) 1043 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 1044 1045 int addtagsize=0; // phase number is added to the name of the field 1046 if(*iphase<10) 1047 addtagsize=1; 1048 else if(*iphase<100) 1049 addtagsize=2; 1050 else if(*iphase<1000) 1051 addtagsize=3; 1052 1053 int tagsize2; 1054 tagsize2=*tagsize+addtagsize; 1055 1056 // char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 1057 // strncpy(fieldlabel, fieldtag, *tagsize); 1058 // fieldlabel[*tagsize] = '\0'; 1059 1060 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 1061 strncpy(fieldlabel, fieldtag, *tagsize); 1062 fieldlabel[tagsize2] = '\0'; 1063 1064 char straddtagsize[10]; 1065 sprintf(straddtagsize,"%d",*iphase); 1066 1067 if(*iphase<10) { 1068 fieldlabel[tagsize2-1]=straddtagsize[0]; 1069 } 1070 else if(*iphase<100) { 1071 fieldlabel[tagsize2-2]=straddtagsize[0]; 1072 fieldlabel[tagsize2-1]=straddtagsize[1]; 1073 } 1074 else if(*iphase<1000) { 1075 fieldlabel[tagsize2-3]=straddtagsize[0]; 1076 fieldlabel[tagsize2-2]=straddtagsize[1]; 1077 fieldlabel[tagsize2-1]=straddtagsize[2]; 1078 } 1079 1080 int irstou; 1081 int magic_number = 362436; 1082 int* mptr = &magic_number; 1083 double version=0.0; 1084 int isize, nitems; 1085 int iarray[10]; 1086 1087 char fmode[10]; 1088 if(!strncmp(filemode,"w",1)) 1089 strcpy(fmode,"write"); 1090 else // default is append 1091 strcpy(fmode,"append"); 1092 1093 char datatype[10]; 1094 if(!strncmp(arraytype,"i",1)) 1095 strcpy(datatype,"int"); 1096 else // default is double 1097 strcpy(datatype,"double"); 1098 1099 // 1100 // // if(*iphase==1) //open the file but then keep it open for the remaining cycles 1101 // openfile_(rfile, fmode, &irstou); 1102 // 1103 // // printf("iphase: %d - pid: %d - irstou %d\n",*iphase,*pid,irstou); 1104 // 1105 // 1106 // nitems = 3; // assuming field will write 3 items in iarray 1107 // iarray[ 0 ] = (*nshg); 1108 // iarray[ 1 ] = (*numvars); 1109 // iarray[ 2 ] = (*stepno); 1110 // 1111 // isize = (*nshg)*(*numvars); 1112 // writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 1113 // 1114 // nitems = (*nshg)*(*numvars); 1115 // writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 1116 // 1117 // // if(*iphase==*nphasesincycle) //close the file after nphasesincycle 1118 // closefile_( &irstou, fmode); 1119 // 1120 1121 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1122 1123 int nfiles; 1124 int nfields; 1125 int numparts; 1126 int irank; 1127 int nprocs; 1128 // unsigned long long timer_start; 1129 // unsigned long long timer_end; 1130 // double time_span; 1131 1132 nfiles = outpar.nsynciofiles; 1133 nfields = outpar.nsynciofieldswriterestart; 1134 numparts = workfc.numpe; 1135 irank = *pid; //workfc.myrank; 1136 nprocs = workfc.numpe; 1137 1138 1139 // Calculate number of parts each proc deal with and where it start and end ... 1140 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1141 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1142 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1143 1144 //int descriptor; 1145 1146 // char * namer; 1147 // namer = strtok(fieldlabel," "); 1148 // strncpy(fieldlabel, fieldtag, *tagsize); 1149 1150 field_flag++; 1151 if(*pid==0) { 1152 // printf("\n*****************************\n"); 1153 printf("\n"); 1154 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 1155 } 1156 1157 1158 int i; 1159 for ( i = 0; i < nppp; i++ ) { 1160 1161 // Write solution field ... 1162 //printf("This is %d and fieldtag_s is %s \n",myrank,fieldtag_s); 1163 1164 isize = (*nshg)*(*numvars); 1165 nitems = 3; 1166 iarray[ 0 ] = (*nshg); 1167 iarray[ 1 ] = (*numvars); 1168 iarray[ 2 ] = (*stepno); 1169 1170 //MR CHANGE 1171 // Measure the time - Start the timer 1172 // MPI_Barrier(MPI_COMM_WORLD); 1173 // timer_start = rdtsc(); 1174 //MR CHANGE END 1175 1176 phio_writeheader(f_descriptor, fieldlabel, (void*)iarray, &nitems, 1177 &isize, "double", phasta_iotype); 1178 1179 //MR CHANGE 1180 // Measure the time - End of timer 1181 // MPI_Barrier(MPI_COMM_WORLD); 1182 // timer_end = rdtsc(); 1183 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1184 // if (*pid==0) { 1185 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 1186 // } 1187 //MR CHANGE END 1188 1189 nitems = (*nshg)*(*numvars); 1190 1191 //MR CHANGE 1192 // Measure the time - Start the timer 1193 // MPI_Barrier(MPI_COMM_WORLD); 1194 // timer_start = rdtsc(); 1195 //MR CHANGE END 1196 1197 phio_writedatablock(f_descriptor, fieldlabel, array, &isize, 1198 "double", phasta_iotype ); 1199 1200 //MR CHANGE 1201 // Measure the time - End of timer 1202 // MPI_Barrier(MPI_COMM_WORLD); 1203 // timer_end = rdtsc(); 1204 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1205 1206 // int isizemin,isizemax,isizetot; 1207 // double sizemin,sizemax,sizeavg,sizetot,rate; 1208 1209 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 1210 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 1211 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 1212 1213 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 1214 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 1215 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 1216 // sizeavg=sizetot/workfc.numpe; 1217 // rate=sizetot/time_span; 1218 1219 // if (*pid==0) { 1220 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 1221 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 1222 // printf("*****************************\n"); 1223 // } 1224 //MR CHANGE END 1225 1226 } 1227 1228 // if (*pid==0) { 1229 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 1230 // printf("Write field '%s' finished! \n",fieldtag_s); 1231 // } 1232 1233 // 1234 // if (field_flag==nfields){ 1235 // closefile_(&f_descriptor, "write"); 1236 // finalizephmpiio_(&f_descriptor); 1237 // if(*pid==0) { 1238 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1239 // printf("\n*****************************\n"); 1240 // } 1241 // } 1242 1243 if (field_flag==nfields){ 1244 1245 //MR CHANGE 1246 // Measure the time - Start the timer 1247 // MPI_Barrier(MPI_COMM_WORLD); 1248 // timer_start = rdtsc(); 1249 //MR CHANGE END 1250 1251 //MR CHANGE 1252 // Measure the time - End of timer 1253 // MPI_Barrier(MPI_COMM_WORLD); 1254 // timer_end = rdtsc(); 1255 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1256 // if (*pid==0) { 1257 // printf("\n*****************************\n"); 1258 // printf("Time: 'closefile' is: %f s\n",time_span); 1259 // } 1260 //MR CHANGE END 1261 1262 //MR CHANGE 1263 // Measure the time - Start the timer 1264 // MPI_Barrier(MPI_COMM_WORLD); 1265 // timer_start = rdtsc(); 1266 //MR CHANGE END 1267 1268 phio_closefile_write(f_descriptor); 1269 1270 //MR CHANGE 1271 // Measure the time - End of timer 1272 // MPI_Barrier(MPI_COMM_WORLD); 1273 // timer_end = rdtsc(); 1274 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1275 if (*pid==0) { 1276 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1277 // printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1278 printf("\n"); 1279 // printf("*****************************\n"); 1280 } 1281 } 1282 //MR CHANGE END 1283 1284 /////////////////////////////////////////////////////////////////////////////////////////// 1285 1286 free(fieldlabel); 1287 } 1288 1289 1290 void 1291 Write_d2wall( int* pid, 1292 int* numnp, 1293 double* array1 ) { 1294 1295 // time_t timenow = time ( &timenow); 1296 int isize, nitems; 1297 int iarray[10]; 1298 1299 // MPI_Barrier(MPI_COMM_WORLD); 1300 1301 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1302 1303 int nfiles; 1304 int nfields; 1305 int numparts; 1306 int irank; 1307 int nprocs; 1308 1309 // First, count the number of fields to write and store the result in 1310 //countfieldstowriterestart(); 1311 1312 // Retrieve and compute the parameters required for SyncIO 1313 nfiles = outpar.nsynciofiles; 1314 nfields = 1; //outpar.nsynciofieldswriterestart; // Only the distance to the walls in d2wall 1315 numparts = workfc.numpe; 1316 irank = *pid; //workfc.myrank; 1317 nprocs = workfc.numpe; 1318 1319 // Calculate number of parts each proc deal with and where it start and end ... 1320 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1321 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1322 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1323 1324 int descriptor; 1325 int nppf=numparts/nfiles; 1326 1327 phio_openfile_write("d2wall.", &nfiles, &nfields, &nppf, &f_descriptor); 1328 1329 field_flag=0; 1330 1331 int i; 1332 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 1333 1334 // Write solution field ... 1335 isize = (*numnp); 1336 nitems = 2; 1337 iarray[ 0 ] = (*numnp); 1338 iarray[ 1 ] = 1; //numVars = 1 1339 1340 phio_writeheader( &f_descriptor, "d2wall", (void*)iarray, &nitems, 1341 &isize, "double", phasta_iotype); 1342 1343 //nitems = (*nshg)*(*numVars); 1344 //nitems = (*numnp); 1345 1346 phio_writedatablock( &f_descriptor, "d2wall", (void*)(array1), &isize, 1347 "double", phasta_iotype ); 1348 } 1349 field_flag++; 1350 1351 if (field_flag==nfields){ 1352 phio_closefile_write(&f_descriptor); 1353 if (irank==0) { 1354 printf("\n"); 1355 } 1356 } 1357 } 1358 1359