1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdio.h> 7 #include <string.h> 8 #include <ctype.h> 9 #include <stdlib.h> 10 #include <time.h> 11 #include <math.h> 12 #include "mpi.h" 13 #include "phastaIO.h" 14 #include "rdtsc.h" 15 #include <FCMangle.h> 16 #include "new_interface.h" 17 #include "phIO.h" 18 19 //MR CHANGE 20 #include "common_c.h" 21 //MR CHANGE END 22 23 #ifdef intel 24 #include <winsock2.h> 25 #else 26 #include <unistd.h> 27 #include <strings.h> 28 #endif 29 30 //extern double cpu_speed = 2600000000.0; //for Jaguar XT5 31 //extern double cpu_speed = 2100000000.0; //for Jaguar xt4 32 //extern double cpu_speed = 850000000.0; //for Intrepid 33 34 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 35 int isThisRank; 36 37 double *value = (double*)malloc(sizeof(double)); 38 *value = 1.0*(*ivalue); 39 40 rgetMinMaxAvg(value,stats,statRanks); 41 42 /* MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 43 isThisRank=workfc.numpe+1; 44 if(*value==stats[0]) 45 isThisRank=workfc.myrank; 46 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 47 48 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 49 isThisRank=workfc.numpe+1; 50 if(*value==stats[1]) 51 isThisRank=workfc.myrank; 52 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 53 54 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 55 stats[2] /= workfc.numpe; */ 56 57 free(value); 58 } 59 60 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 61 int isThisRank; 62 63 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 64 isThisRank=workfc.numpe+1; 65 if(*value==stats[0]) 66 isThisRank=workfc.myrank; 67 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 68 69 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 70 isThisRank=workfc.numpe+1; 71 if(*value==stats[1]) 72 isThisRank=workfc.myrank; 73 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 74 75 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 76 stats[2] /= workfc.numpe; 77 78 double sqValue = (*value)*(*value), sqValueAvg = 0.; 79 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 80 sqValueAvg /= workfc.numpe; 81 // stats[3] = sqValueAvg; 82 83 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 84 } 85 86 void print_mesh_stats(void) { 87 int statRanks[2]; 88 double iStats[4], rStats[4]; 89 90 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 91 if(workfc.myrank==workfc.master) 92 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 93 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 94 if(workfc.myrank==workfc.master) 95 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 96 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 97 if(workfc.myrank==workfc.master) 98 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 99 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 100 if(workfc.myrank==workfc.master) { 101 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 102 printf("\n"); 103 } 104 } 105 106 void print_mpi_stats(void) { 107 int statRanks[2]; 108 double iStats[4], rStats[4]; 109 110 // NS equations 111 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 112 if(workfc.myrank==workfc.master) 113 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 114 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 115 if(workfc.myrank==workfc.master) 116 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 117 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 118 if(workfc.myrank==workfc.master) 119 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 120 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 121 if(workfc.myrank==workfc.master) 122 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 123 124 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 125 if(workfc.myrank==workfc.master) 126 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 127 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 128 if(workfc.myrank==workfc.master) 129 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 130 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 131 if(workfc.myrank==workfc.master) 132 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 133 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 134 if(workfc.myrank==workfc.master) 135 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 136 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 137 if(workfc.myrank==workfc.master) { 138 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 139 printf("\n"); 140 } 141 // Scalars 142 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 143 if(workfc.myrank==workfc.master) 144 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 145 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 146 if(workfc.myrank==workfc.master) 147 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 148 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 149 if(workfc.myrank==workfc.master) 150 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 151 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 152 if(workfc.myrank==workfc.master) 153 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 154 155 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 156 if(workfc.myrank==workfc.master) 157 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 158 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 159 if(workfc.myrank==workfc.master) 160 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 161 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 162 if(workfc.myrank==workfc.master) 163 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 164 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 165 if(workfc.myrank==workfc.master) 166 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 167 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 168 if(workfc.myrank==workfc.master) 169 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 170 171 172 } 173 174 //void print_system_stats(double tcorecp[2]) { 175 void print_system_stats(double *tcorecp, double *tcorecpscal) { 176 int statRanks[2]; 177 double iStats[4], rStats[4]; 178 double syst_assembly, syst_solve; 179 180 // NS equations 181 syst_assembly = tcorecp[0]; 182 syst_solve = tcorecp[1]; 183 184 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 185 if(workfc.myrank==workfc.master) 186 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 187 188 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 189 if(workfc.myrank==workfc.master) 190 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 191 192 // Scalars 193 syst_assembly = tcorecpscal[0]; 194 syst_solve = tcorecpscal[1]; 195 196 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 197 if(workfc.myrank==workfc.master) 198 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 199 200 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 201 if(workfc.myrank==workfc.master) { 202 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 203 printf("\n"); 204 } 205 //printf("rank %d - syst_assembly %f - syst_solve %f\n",workfc.myrank,syst_assembly,syst_solve); 206 } 207 208 209 210 void countfieldstowriterestart() 211 { 212 int nfields; 213 214 // printf("TEST: %d %d %d %d %d\n",timdat.istep,timdat.itseq,inpdat.nstep[0],inpdat.nstep[1],timdat.lstep); 215 216 nfields = 2; //solution, time derivatives 217 218 if(outpar.ivort == 1){ 219 nfields++; //vorticity 220 } 221 222 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 223 nfields++; //instantaneous wss in bflux.f 224 } 225 226 // if(ideformwall.eq.1) not handled yet 227 228 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 229 230 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 231 nfields = nfields +2; 232 233 //if Print Error Indicators = true (call write_error in itrdrv) 234 if(turbvar.ierrcalc == 1){ 235 nfields++; 236 } 237 238 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 239 if(outpar.ioybar == 1){ 240 nfields++; //ybar 241 242 //phase average fields 243 if(outpar.nphasesincycle >0) { 244 nfields = nfields + outpar.nphasesincycle; 245 } 246 247 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 248 nfields++; //wssbar 249 } 250 251 } 252 253 if(turbvari.irans < 0) { 254 nfields++; //dwal 255 } 256 257 } 258 259 outpar.nsynciofieldswriterestart = nfields; 260 261 if(workfc.myrank == 0) { 262 printf("Number of fields to write in restart files: %d\n", nfields); 263 } 264 } 265 266 267 void 268 Write_Restart( int* pid, 269 int* stepno, 270 int* nshg, 271 int* numVars, 272 double* array1, 273 double* array2 ) { 274 275 char fname[255]; 276 char rfile[60]; 277 char existingfile[30], linkfile[30]; 278 int irstou; 279 int magic_number = 362436; 280 int* mptr = &magic_number; 281 // time_t timenow = time ( &timenow); 282 double version=0.0; 283 int isize, nitems; 284 int iarray[10]; 285 286 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 287 openfile_(rfile,"write", &irstou); 288 289 // writing the top ascii header for the restart file 290 291 writestring_( &irstou,"# PHASTA Input File Version 2.0\n"); 292 writestring_( &irstou, 293 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 294 295 bzero( (void*)fname, 255 ); 296 sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); 297 writestring_( &irstou, fname ); 298 299 bzero( (void*)fname, 255 ); 300 gethostname(fname,255); 301 writestring_( &irstou,"# This result was produced on: "); 302 writestring_( &irstou, fname ); 303 writestring_( &irstou,"\n"); 304 305 bzero( (void*)fname, 255 ); 306 sprintf(fname,"# %s\n", ctime( &timenow )); 307 writestring_( &irstou, fname ); 308 309 isize = 1; 310 nitems = 1; 311 iarray[ 0 ] = 1; 312 writeheader_( &irstou, "byteorder magic number ", 313 (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); 314 315 nitems = 1; 316 writedatablock_( &irstou, "byteorder magic number ", 317 (void*)mptr, &nitems, "integer", phasta_iotype ); 318 319 320 bzero( (void*)fname, 255 ); 321 sprintf(fname,"number of modes : < 0 > %d\n", *nshg); 322 writestring_( &irstou, fname ); 323 324 bzero( (void*)fname, 255 ); 325 sprintf(fname,"number of variables : < 0 > %d\n", *numVars); 326 writestring_( &irstou, fname ); 327 328 329 isize = (*nshg)*(*numVars); 330 nitems = 3; 331 iarray[ 0 ] = (*nshg); 332 iarray[ 1 ] = (*numVars); 333 iarray[ 2 ] = (*stepno); 334 writeheader_( &irstou, "solution ", 335 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 336 337 338 nitems = (*nshg)*(*numVars); 339 writedatablock_( &irstou, "solution ", 340 (void*)(array1), &nitems, "double", phasta_iotype ); 341 342 343 344 nitems = 3; 345 writeheader_( &irstou, "time derivative of solution ", 346 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 347 348 349 nitems = (*nshg)*(*numVars); 350 writedatablock_( &irstou, "time derivative of solution ", 351 (void*)(array2), &nitems, "double", phasta_iotype ); 352 353 354 closefile_( &irstou, "write" ); 355 */ 356 //MPI_Barrier(MPI_COMM_WORLD); 357 358 /////////////////////////////// Start of writing using new-lib //////////////////////////// 359 360 //MR CHANGE 361 int nfiles; 362 int nfields; 363 int numparts; 364 int irank; 365 int nprocs; 366 367 // First, count the number of fields to write and store the result in 368 countfieldstowriterestart(); 369 370 // Retrieve and compute the parameters required for SyncIO 371 nfiles = outpar.nsynciofiles; 372 nfields = outpar.nsynciofieldswriterestart; 373 numparts = workfc.numpe; 374 irank = *pid; //workfc.myrank; 375 nprocs = workfc.numpe; 376 //MR CHANGE END 377 int nppf = numparts/nfiles; 378 int GPID; 379 380 // Calculate number of parts each proc deal with and where it start and end ... 381 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 382 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 383 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 384 385 int descriptor; 386 char filename[255],path[255],fieldtag_s[255]; 387 bzero((void*)filename,255); 388 bzero((void*)fieldtag_s,255); 389 390 phio_restartname(stepno, filename); 391 phio_openfile_write(filename, &nfiles, &nfields, &nppf, &f_descriptor); 392 393 //MR CHANGE 394 // Measure the time - End of timer 395 // MPI_Barrier(MPI_COMM_WORLD); 396 // timer_end = rdtsc(); 397 // time_span=(double)((timer_end-timer_start)/cpu_speed); 398 // if (*pid==0) { 399 // printf("Time: 'openfile' of %s with %d fields and %d files is: %f s\n",filename,nfields,nfiles,time_span); 400 // printf("*****************************\n"); 401 // } 402 //MR CHANGE END 403 404 field_flag=0; 405 406 int i; 407 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 408 // GPID : global part id, corresponds to rank ... 409 // e.g : (in this example) 410 // proc 0 : 1--4 411 // proc 1 : 5--8 ... 412 GPID = startpart + i; 413 414 // Write solution field ... 415 sprintf(fieldtag_s,"solution@%d",GPID); 416 417 isize = (*nshg)*(*numVars); 418 nitems = 3; 419 iarray[ 0 ] = (*nshg); 420 iarray[ 1 ] = (*numVars); 421 iarray[ 2 ] = (*stepno); 422 423 //MR CHANGE 424 // Measure the time - Start the timer 425 // MPI_Barrier(MPI_COMM_WORLD); 426 // timer_start = rdtsc(); 427 //MR CHANGE END 428 429 phio_writeheader( &f_descriptor, "solution", (void*)iarray, &nitems, 430 &isize, "double", phasta_iotype); 431 432 //MR CHANGE 433 // Measure the time - End of timer 434 // MPI_Barrier(MPI_COMM_WORLD); 435 // timer_end = rdtsc(); 436 // time_span=(double)((timer_end-timer_start)/cpu_speed); 437 // if (*pid==0) { 438 // printf("\n*****************************\n"); 439 // printf("Time: header for 'Solution': %f s\n",time_span); 440 // } 441 //MR CHANGE END 442 443 nitems = (*nshg)*(*numVars); 444 445 //MR CHANGE 446 // Measure the time - Start the timer 447 // MPI_Barrier(MPI_COMM_WORLD); 448 // timer_start = rdtsc(); 449 //MR CHANGE END 450 451 phio_writedatablock(&f_descriptor, "solution", (void*)(array1), 452 &isize, "double", phasta_iotype ); 453 454 //MR CHANGE 455 // Measure the time - End of timer 456 // MPI_Barrier(MPI_COMM_WORLD); 457 // timer_end = rdtsc(); 458 // time_span=(double)((timer_end-timer_start)/cpu_speed); 459 460 // int isizemin,isizemax,isizetot; 461 // double sizemin,sizemax,sizeavg,sizetot,rate; 462 463 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 464 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 465 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 466 467 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 468 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 469 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 470 // sizeavg=(double)(1.0*sizetot/workfc.numpe); 471 // rate=(double)(1.0*sizetot/time_span); 472 473 // if (*pid==0) { 474 // printf("Time: block for 'Solution': %f s\n",time_span); 475 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 476 477 // } 478 //MR CHANGE END 479 480 } 481 field_flag++; 482 483 for ( i = 0; i < nppp; i++) { 484 485 // GPID : global part id, corresponds to rank ... 486 // e.g : (in this example) 487 // proc 0 : 1--4 488 // proc 1 : 5--8 ... 489 GPID = startpart + i; 490 491 // Write solution field ... 492 sprintf(fieldtag_s,"time derivative of solution@%d",GPID); 493 494 isize = (*nshg)*(*numVars); 495 nitems = 3; 496 iarray[ 0 ] = (*nshg); 497 iarray[ 1 ] = (*numVars); 498 iarray[ 2 ] = (*stepno); 499 500 //MR CHANGE 501 // Measure the time - Start the timer 502 // MPI_Barrier(MPI_COMM_WORLD); 503 // timer_start = rdtsc(); 504 //MR CHANGE END 505 506 phio_writeheader( &f_descriptor, "time derivative of solution", 507 (void*)iarray, &nitems, &isize, "double", phasta_iotype); 508 509 //MR CHANGE 510 // Measure the time - End of timer 511 // MPI_Barrier(MPI_COMM_WORLD); 512 // timer_end = rdtsc(); 513 // time_span=(double)((timer_end-timer_start)/cpu_speed); 514 // if (*pid==0) { 515 // printf("Time: header for 'Time Derivative of solution': %f s\n",time_span); 516 // } 517 //MR CHANGE END 518 519 nitems = (*nshg)*(*numVars); 520 521 //MR CHANGE 522 // Measure the time - Start the timer 523 // MPI_Barrier(MPI_COMM_WORLD); 524 // timer_start = rdtsc(); 525 //MR CHANGE END 526 527 phio_writedatablock( &f_descriptor, "time derivative of solution", 528 (void*)(array2), &isize, "double", phasta_iotype ); 529 530 //MR CHANGE 531 // Measure the time - End of timer 532 // MPI_Barrier(MPI_COMM_WORLD); 533 // timer_end = rdtsc(); 534 // time_span=(double)((timer_end-timer_start)/cpu_speed); 535 536 // int isizemin,isizemax,isizetot; 537 // double sizemin,sizemax,sizeavg,sizetot,rate; 538 539 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 540 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 541 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 542 543 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 544 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 545 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 546 // sizeavg=sizetot/workfc.numpe; 547 // rate=sizetot/time_span; 548 549 // if (*pid==0) { 550 // printf("Time: block for 'Time Derivative of Solution': %f s\n",time_span); 551 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 552 // printf("*****************************\n"); 553 554 // } 555 //MR CHANGE END 556 557 } 558 field_flag++; 559 560 if (field_flag==nfields){ 561 562 //MR CHANGE 563 // Measure the time - Start the timer 564 // MPI_Barrier(MPI_COMM_WORLD); 565 // timer_start = rdtsc(); 566 //MR CHANGE END 567 568 //MR CHANGE 569 // Measure the time - End of timer 570 // MPI_Barrier(MPI_COMM_WORLD); 571 // timer_end = rdtsc(); 572 // time_span=(double)((timer_end-timer_start)/cpu_speed); 573 // if (*pid==0) { 574 // printf("\n*****************************\n"); 575 // printf("Time: 'closefile' is: %f s\n",time_span); 576 // } 577 //MR CHANGE END 578 579 //MR CHANGE 580 // Measure the time - Start the timer 581 // MPI_Barrier(MPI_COMM_WORLD); 582 // timer_start = rdtsc(); 583 //MR CHANGE END 584 585 phio_closefile_write(&f_descriptor); 586 587 //MR CHANGE 588 // Measure the time - End of timer 589 // MPI_Barrier(MPI_COMM_WORLD); 590 // timer_end = rdtsc(); 591 // time_span=(double)((timer_end-timer_start)/cpu_speed); 592 if (*pid==0) { 593 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 594 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 595 printf("\n"); 596 // printf("*****************************\n"); 597 } 598 } 599 //MR CHANGE END 600 601 602 603 /////////////////////////////////////////////////////////////////////////////////////////// 604 605 /* create a soft link of the restart we just wrote to restart.latest 606 this is the file the next run will always try to start from */ 607 608 /* sprintf( linkfile, "restart.latest.%d", *pid+1 ); 609 unlink( linkfile ); 610 sprintf( existingfile, "restart.%d.%d", *stepno, *pid+1 ); 611 link( existingfile, linkfile ); 612 */ 613 } 614 615 void 616 Write_Error( int* pid, 617 int* stepno, 618 int* nshg, 619 int* numVars, 620 double* array1 ) { 621 622 623 char fname[255]; 624 char rfile[60]; 625 int irstou; 626 int magic_number = 362436; 627 int* mptr = &magic_number; 628 //printf("Time is commented\n"); 629 //time_t timenow = time ( &timenow); 630 //printf("Yes\n"); 631 double version=0.0; 632 int isize, nitems; 633 int iarray[10]; 634 635 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 636 openfile_(rfile,"append", &irstou); 637 638 isize = (*nshg)*(*numVars); 639 nitems = 3; 640 iarray[ 0 ] = (*nshg); 641 iarray[ 1 ] = (*numVars); 642 iarray[ 2 ] = (*stepno); 643 writeheader_( &irstou, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 644 645 646 nitems = (*nshg)*(*numVars); 647 writedatablock_( &irstou, "errors ", (void*)(array1), &nitems, "double", phasta_iotype ); 648 649 closefile_( &irstou, "append" );*/ 650 651 /////////////////////////////// Start of writing using new-lib //////////////////////////// 652 653 int nfiles; 654 int nfields; 655 int numparts; 656 int irank; 657 int nprocs; 658 659 // unsigned long long timer_start; 660 // unsigned long long timer_end; 661 // double time_span; 662 663 nfiles = outpar.nsynciofiles; 664 nfields = outpar.nsynciofieldswriterestart; 665 numparts = workfc.numpe; 666 irank = *pid; //workfc.myrank; 667 nprocs = workfc.numpe; 668 669 int nppf = numparts/nfiles; 670 int GPID; 671 672 // Calculate number of parts each proc deal with and where it start and end ... 673 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 674 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 675 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 676 677 field_flag++; 678 679 char fieldtag[255]; 680 681 int i; 682 for ( i = 0; i < nppp; i++ ) { 683 GPID = startpart + i; 684 sprintf(fieldtag,"errors@%d",GPID); 685 686 if(*pid==0) { 687 // printf("\n*****************************\n"); 688 printf("\n"); 689 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldtag); 690 } 691 692 isize = (*nshg)*(*numVars); 693 nitems = 3; 694 iarray[ 0 ] = (*nshg); 695 iarray[ 1 ] = (*numVars); 696 iarray[ 2 ] = (*stepno); 697 698 //MR CHANGE 699 // Measure the time - Start the timer 700 // MPI_Barrier(MPI_COMM_WORLD); 701 // timer_start = rdtsc(); 702 //MR CHANGE END 703 704 phio_writeheader( &f_descriptor, "errors", (void*)iarray, &nitems, 705 &isize, "double", phasta_iotype); 706 707 //MR CHANGE 708 // Measure the time - End of timer 709 // MPI_Barrier(MPI_COMM_WORLD); 710 // timer_end = rdtsc(); 711 // time_span=(double)((timer_end-timer_start)/cpu_speed); 712 // if (*pid==0) { 713 // printf("Time: header for 'error': %f s\n",time_span); 714 // } 715 //MR CHANGE END 716 717 //MR CHANGE 718 // Measure the time - Start the timer 719 // MPI_Barrier(MPI_COMM_WORLD); 720 // timer_start = rdtsc(); 721 //MR CHANGE END 722 723 phio_writedatablock( &f_descriptor, "errors", (void*)array1, &isize, 724 "double", phasta_iotype ); 725 726 //MR CHANGE 727 // Measure the time - End of timer 728 // MPI_Barrier(MPI_COMM_WORLD); 729 // timer_end = rdtsc(); 730 // time_span=(double)((timer_end-timer_start)/cpu_speed); 731 732 // int isizemin,isizemax,isizetot; 733 // double sizemin,sizemax,sizeavg,sizetot,rate; 734 735 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 736 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 737 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 738 739 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 740 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 741 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 742 // sizeavg=sizetot/workfc.numpe; 743 // rate=sizetot/time_span; 744 745 // if (*pid==0) { 746 // printf("Time: block for 'error': %f s\n",time_span); 747 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 748 // printf("*****************************\n"); 749 // } 750 //MR CHANGE END 751 752 } 753 754 // MPI_Barrier(MPI_COMM_WORLD); 755 // timer_end = rdtsc(); 756 // time_span=(double)(timer_end-timer_start)/cpu_speed; 757 758 // if (*pid==0) { 759 // printf("Field 'error' written in: %f s\n",time_span); 760 // printf("Write field '%s' finished! \n",fieldtag); 761 // } 762 763 if (field_flag==nfields){ 764 765 //MR CHANGE 766 // Measure the time - Start the timer 767 // MPI_Barrier(MPI_COMM_WORLD); 768 // timer_start = rdtsc(); 769 //MR CHANGE END 770 771 //MR CHANGE 772 // Measure the time - End of timer 773 // MPI_Barrier(MPI_COMM_WORLD); 774 // timer_end = rdtsc(); 775 // time_span=(double)((timer_end-timer_start)/cpu_speed); 776 // if (*pid==0) { 777 // printf("\n*****************************\n"); 778 // printf("Time: 'closefile' is: %f s\n",time_span); 779 // } 780 //MR CHANGE END 781 782 //MR CHANGE 783 // Measure the time - Start the timer 784 // MPI_Barrier(MPI_COMM_WORLD); 785 // timer_start = rdtsc(); 786 //MR CHANGE END 787 788 phio_closefile_write(&f_descriptor); 789 790 //MR CHANGE 791 // Measure the time - End of timer 792 // MPI_Barrier(MPI_COMM_WORLD); 793 // timer_end = rdtsc(); 794 // time_span=(double)((timer_end-timer_start)/cpu_speed); 795 if (*pid==0) { 796 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 797 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 798 printf("\n"); 799 // printf("*****************************\n"); 800 } 801 } 802 //MR CHANGE END 803 804 /////////////////////////////////////////////////////////////////////////////////////////// 805 806 807 } 808 809 810 void 811 Write_Displ( int* pid, 812 int* stepno, 813 int* nshg, 814 int* numVars, 815 double* array1 ) { 816 fprintf(stderr, "This function is dead...exiting\n"); 817 exit(1); 818 } 819 820 void 821 Write_Field( int *pid, 822 char* filemode, 823 char* fieldtag, 824 int* tagsize, 825 void* array, 826 char* arraytype, 827 int* nshg, 828 int* numvars, 829 int* stepno) { 830 831 //printf("Rank is %d, field is %s, tagsize is %d, nshg is %d, numvars is %d\n",*pid,fieldtag,*tagsize,*nshg,*numvars); 832 833 // char rfile[32]; 834 // assuming restart.sn.(pid+1) 835 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 836 837 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 838 strncpy(fieldlabel, fieldtag, *tagsize); 839 fieldlabel[*tagsize] = '\0'; 840 841 int irstou; 842 int magic_number = 362436; 843 int* mptr = &magic_number; 844 double version=0.0; 845 int isize, nitems; 846 int iarray[10]; 847 848 char fmode[10]; 849 if(!strncmp(filemode,"w",1)) 850 strcpy(fmode,"write"); 851 else // default is append 852 strcpy(fmode,"append"); 853 854 char datatype[10]; 855 if(!strncmp(arraytype,"i",1)) 856 strcpy(datatype,"int"); 857 else // default is double 858 strcpy(datatype,"double"); 859 860 /* openfile_(rfile, fmode, &irstou); 861 862 nitems = 3; // assuming field will write 3 items in iarray 863 iarray[ 0 ] = (*nshg); 864 iarray[ 1 ] = (*numvars); 865 iarray[ 2 ] = (*stepno); 866 867 isize = (*nshg)*(*numvars); 868 writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 869 870 nitems = (*nshg)*(*numvars); 871 writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 872 closefile_( &irstou, fmode); 873 */ 874 /////////////////////////////// Start of writing using new-lib //////////////////////////// 875 876 int nfiles; 877 int nfields; 878 int numparts; 879 int irank; 880 int nprocs; 881 882 // unsigned long long timer_start; 883 // unsigned long long timer_end; 884 // double time_span; 885 886 nfiles = outpar.nsynciofiles; 887 nfields = outpar.nsynciofieldswriterestart; 888 numparts = workfc.numpe; 889 irank = *pid; //workfc.myrank; 890 nprocs = workfc.numpe; 891 892 int nppf = numparts/nfiles; 893 int GPID; 894 895 // Calculate number of parts each proc deal with and where it start and end ... 896 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 897 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 898 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 899 900 char filename[255],path[255],fieldtag_s[255]; 901 bzero((void*)filename,255); 902 bzero((void*)fieldtag_s,255); 903 904 strncpy(fieldlabel, fieldtag, *tagsize); 905 906 field_flag++; 907 if(*pid==0) { 908 // printf("\n*****************************\n"); 909 printf("\n"); 910 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 911 } 912 913 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 914 915 // MPI_Barrier(MPI_COMM_WORLD); 916 // timer_start = rdtsc(); 917 918 int i; 919 for ( i = 0; i < nppp; i++ ) { 920 GPID = startpart + i; 921 922 // Write solution field ... 923 sprintf(fieldtag_s,"%s@%d",fieldlabel,GPID); 924 925 isize = (*nshg)*(*numvars); 926 nitems = 3; 927 iarray[ 0 ] = (*nshg); 928 iarray[ 1 ] = (*numvars); 929 iarray[ 2 ] = (*stepno); 930 931 //MR CHANGE 932 // Measure the time - Start the timer 933 // MPI_Barrier(MPI_COMM_WORLD); 934 // timer_start = rdtsc(); 935 //MR CHANGE END 936 937 phio_writeheader( &f_descriptor, fieldlabel, (void*)iarray, &nitems, 938 &isize, datatype, phasta_iotype); 939 940 //MR CHANGE 941 // Measure the time - End of timer 942 // MPI_Barrier(MPI_COMM_WORLD); 943 // timer_end = rdtsc(); 944 // time_span=(double)((timer_end-timer_start)/cpu_speed); 945 // if (*pid==0) { 946 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 947 // } 948 //MR CHANGE END 949 950 nitems = (*nshg)*(*numvars); 951 952 //MR CHANGE 953 // Measure the time - Start the timer 954 // MPI_Barrier(MPI_COMM_WORLD); 955 // timer_start = rdtsc(); 956 //MR CHANGE END 957 958 phio_writedatablock( &f_descriptor, fieldlabel, array, &isize, 959 datatype, phasta_iotype ); 960 961 //MR CHANGE 962 // Measure the time - End of timer 963 // MPI_Barrier(MPI_COMM_WORLD); 964 // timer_end = rdtsc(); 965 // time_span=(double)((timer_end-timer_start)/cpu_speed); 966 967 // int isizemin,isizemax,isizetot; 968 // double sizemin,sizemax,sizeavg,sizetot,rate; 969 970 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 971 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 972 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 973 974 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 975 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 976 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 977 // sizeavg=sizetot/workfc.numpe; 978 // rate=sizetot/time_span; 979 980 // if (*pid==0) { 981 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 982 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 983 // printf("*****************************\n"); 984 // } 985 //MR CHANGE END 986 987 } 988 989 // MPI_Barrier(MPI_COMM_WORLD); 990 // timer_end = rdtsc(); 991 // time_span=(double)(timer_end-timer_start)/cpu_speed; 992 993 // if (*pid==0) { 994 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 995 // printf("Write field '%s' finished! \n",fieldtag_s); 996 // } 997 998 // if (field_flag==nfields){ 999 // closefile_(&f_descriptor, "write"); 1000 // finalizephmpiio_(&f_descriptor); 1001 // if(*pid==0) { 1002 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1003 // printf("\n*****************************\n"); 1004 // } 1005 // } 1006 1007 if (field_flag==nfields){ 1008 1009 //MR CHANGE 1010 // Measure the time - Start the timer 1011 // MPI_Barrier(MPI_COMM_WORLD); 1012 // timer_start = rdtsc(); 1013 //MR CHANGE END 1014 1015 //MR CHANGE 1016 // Measure the time - End of timer 1017 // MPI_Barrier(MPI_COMM_WORLD); 1018 // timer_end = rdtsc(); 1019 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1020 // if (*pid==0) { 1021 // printf("\n*****************************\n"); 1022 // printf("Time: 'closefile' is: %f s\n",time_span); 1023 // } 1024 //MR CHANGE END 1025 1026 //MR CHANGE 1027 // Measure the time - Start the timer 1028 // MPI_Barrier(MPI_COMM_WORLD); 1029 // timer_start = rdtsc(); 1030 //MR CHANGE END 1031 1032 phio_closefile_write(&f_descriptor); 1033 1034 //MR CHANGE 1035 // Measure the time - End of timer 1036 // MPI_Barrier(MPI_COMM_WORLD); 1037 // timer_end = rdtsc(); 1038 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1039 if (*pid==0) { 1040 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1041 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1042 printf("\n"); 1043 // printf("*****************************\n"); 1044 } 1045 } 1046 //MR CHANGE END 1047 1048 /////////////////////////////////////////////////////////////////////////////////////////// 1049 1050 free(fieldlabel); 1051 } 1052 1053 //MR CHANGE 1054 1055 //MR CHANGE 1056 void 1057 Write_PhAvg2( int* pid, 1058 char* filemode, 1059 char* fieldtag, 1060 int* tagsize, 1061 int* iphase, 1062 int* nphasesincycle, 1063 void* array, 1064 char* arraytype, 1065 int* nshg, 1066 int* numvars, 1067 int* stepno) { 1068 1069 // char rfile[32]; 1070 // assuming restart.sn.(pid+1) 1071 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 1072 1073 int addtagsize=0; // phase number is added to the name of the field 1074 if(*iphase<10) 1075 addtagsize=1; 1076 else if(*iphase<100) 1077 addtagsize=2; 1078 else if(*iphase<1000) 1079 addtagsize=3; 1080 1081 int tagsize2; 1082 tagsize2=*tagsize+addtagsize; 1083 1084 // char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 1085 // strncpy(fieldlabel, fieldtag, *tagsize); 1086 // fieldlabel[*tagsize] = '\0'; 1087 1088 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 1089 strncpy(fieldlabel, fieldtag, *tagsize); 1090 fieldlabel[tagsize2] = '\0'; 1091 1092 char straddtagsize[10]; 1093 sprintf(straddtagsize,"%d",*iphase); 1094 1095 if(*iphase<10) { 1096 fieldlabel[tagsize2-1]=straddtagsize[0]; 1097 } 1098 else if(*iphase<100) { 1099 fieldlabel[tagsize2-2]=straddtagsize[0]; 1100 fieldlabel[tagsize2-1]=straddtagsize[1]; 1101 } 1102 else if(*iphase<1000) { 1103 fieldlabel[tagsize2-3]=straddtagsize[0]; 1104 fieldlabel[tagsize2-2]=straddtagsize[1]; 1105 fieldlabel[tagsize2-1]=straddtagsize[2]; 1106 } 1107 1108 int irstou; 1109 int magic_number = 362436; 1110 int* mptr = &magic_number; 1111 double version=0.0; 1112 int isize, nitems; 1113 int iarray[10]; 1114 1115 char fmode[10]; 1116 if(!strncmp(filemode,"w",1)) 1117 strcpy(fmode,"write"); 1118 else // default is append 1119 strcpy(fmode,"append"); 1120 1121 char datatype[10]; 1122 if(!strncmp(arraytype,"i",1)) 1123 strcpy(datatype,"int"); 1124 else // default is double 1125 strcpy(datatype,"double"); 1126 1127 // 1128 // // if(*iphase==1) //open the file but then keep it open for the remaining cycles 1129 // openfile_(rfile, fmode, &irstou); 1130 // 1131 // // printf("iphase: %d - pid: %d - irstou %d\n",*iphase,*pid,irstou); 1132 // 1133 // 1134 // nitems = 3; // assuming field will write 3 items in iarray 1135 // iarray[ 0 ] = (*nshg); 1136 // iarray[ 1 ] = (*numvars); 1137 // iarray[ 2 ] = (*stepno); 1138 // 1139 // isize = (*nshg)*(*numvars); 1140 // writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 1141 // 1142 // nitems = (*nshg)*(*numvars); 1143 // writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 1144 // 1145 // // if(*iphase==*nphasesincycle) //close the file after nphasesincycle 1146 // closefile_( &irstou, fmode); 1147 // 1148 1149 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1150 1151 int nfiles; 1152 int nfields; 1153 int numparts; 1154 int irank; 1155 int nprocs; 1156 // unsigned long long timer_start; 1157 // unsigned long long timer_end; 1158 // double time_span; 1159 1160 nfiles = outpar.nsynciofiles; 1161 nfields = outpar.nsynciofieldswriterestart; 1162 numparts = workfc.numpe; 1163 irank = *pid; //workfc.myrank; 1164 nprocs = workfc.numpe; 1165 1166 int nppf = numparts/nfiles; 1167 int GPID; 1168 1169 // Calculate number of parts each proc deal with and where it start and end ... 1170 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1171 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1172 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1173 1174 //int descriptor; 1175 char filename[255],path[255],fieldtag_s[255]; 1176 bzero((void*)filename,255); 1177 bzero((void*)fieldtag_s,255); 1178 1179 // char * namer; 1180 // namer = strtok(fieldlabel," "); 1181 // strncpy(fieldlabel, fieldtag, *tagsize); 1182 1183 field_flag++; 1184 if(*pid==0) { 1185 // printf("\n*****************************\n"); 1186 printf("\n"); 1187 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 1188 } 1189 1190 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 1191 1192 int i; 1193 for ( i = 0; i < nppp; i++ ) { 1194 GPID = startpart + i; 1195 1196 // Write solution field ... 1197 sprintf(fieldtag_s,"%s@%d",fieldlabel,GPID); 1198 1199 //printf("This is %d and fieldtag_s is %s \n",myrank,fieldtag_s); 1200 1201 isize = (*nshg)*(*numvars); 1202 nitems = 3; 1203 iarray[ 0 ] = (*nshg); 1204 iarray[ 1 ] = (*numvars); 1205 iarray[ 2 ] = (*stepno); 1206 1207 //MR CHANGE 1208 // Measure the time - Start the timer 1209 // MPI_Barrier(MPI_COMM_WORLD); 1210 // timer_start = rdtsc(); 1211 //MR CHANGE END 1212 1213 phio_writeheader( &f_descriptor, fieldlabel, (void*)iarray, &nitems, 1214 &isize, "double", phasta_iotype); 1215 1216 //MR CHANGE 1217 // Measure the time - End of timer 1218 // MPI_Barrier(MPI_COMM_WORLD); 1219 // timer_end = rdtsc(); 1220 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1221 // if (*pid==0) { 1222 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 1223 // } 1224 //MR CHANGE END 1225 1226 nitems = (*nshg)*(*numvars); 1227 1228 //MR CHANGE 1229 // Measure the time - Start the timer 1230 // MPI_Barrier(MPI_COMM_WORLD); 1231 // timer_start = rdtsc(); 1232 //MR CHANGE END 1233 1234 phio_writedatablock( &f_descriptor, fieldlabel, array, &isize, 1235 "double", phasta_iotype ); 1236 1237 //MR CHANGE 1238 // Measure the time - End of timer 1239 // MPI_Barrier(MPI_COMM_WORLD); 1240 // timer_end = rdtsc(); 1241 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1242 1243 // int isizemin,isizemax,isizetot; 1244 // double sizemin,sizemax,sizeavg,sizetot,rate; 1245 1246 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 1247 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 1248 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 1249 1250 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 1251 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 1252 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 1253 // sizeavg=sizetot/workfc.numpe; 1254 // rate=sizetot/time_span; 1255 1256 // if (*pid==0) { 1257 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 1258 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 1259 // printf("*****************************\n"); 1260 // } 1261 //MR CHANGE END 1262 1263 } 1264 1265 // if (*pid==0) { 1266 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 1267 // printf("Write field '%s' finished! \n",fieldtag_s); 1268 // } 1269 1270 // 1271 // if (field_flag==nfields){ 1272 // closefile_(&f_descriptor, "write"); 1273 // finalizephmpiio_(&f_descriptor); 1274 // if(*pid==0) { 1275 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1276 // printf("\n*****************************\n"); 1277 // } 1278 // } 1279 1280 if (field_flag==nfields){ 1281 1282 //MR CHANGE 1283 // Measure the time - Start the timer 1284 // MPI_Barrier(MPI_COMM_WORLD); 1285 // timer_start = rdtsc(); 1286 //MR CHANGE END 1287 1288 //MR CHANGE 1289 // Measure the time - End of timer 1290 // MPI_Barrier(MPI_COMM_WORLD); 1291 // timer_end = rdtsc(); 1292 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1293 // if (*pid==0) { 1294 // printf("\n*****************************\n"); 1295 // printf("Time: 'closefile' is: %f s\n",time_span); 1296 // } 1297 //MR CHANGE END 1298 1299 //MR CHANGE 1300 // Measure the time - Start the timer 1301 // MPI_Barrier(MPI_COMM_WORLD); 1302 // timer_start = rdtsc(); 1303 //MR CHANGE END 1304 1305 phio_closefile_write(&f_descriptor); 1306 1307 //MR CHANGE 1308 // Measure the time - End of timer 1309 // MPI_Barrier(MPI_COMM_WORLD); 1310 // timer_end = rdtsc(); 1311 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1312 if (*pid==0) { 1313 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1314 // printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1315 printf("\n"); 1316 // printf("*****************************\n"); 1317 } 1318 } 1319 //MR CHANGE END 1320 1321 /////////////////////////////////////////////////////////////////////////////////////////// 1322 1323 free(fieldlabel); 1324 } 1325 1326 1327 void 1328 Write_d2wall( int* pid, 1329 int* numnp, 1330 double* array1 ) { 1331 1332 // time_t timenow = time ( &timenow); 1333 int isize, nitems; 1334 int iarray[10]; 1335 1336 // MPI_Barrier(MPI_COMM_WORLD); 1337 1338 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1339 1340 int nfiles; 1341 int nfields; 1342 int numparts; 1343 int irank; 1344 int nprocs; 1345 1346 // First, count the number of fields to write and store the result in 1347 //countfieldstowriterestart(); 1348 1349 // Retrieve and compute the parameters required for SyncIO 1350 nfiles = outpar.nsynciofiles; 1351 nfields = 1; //outpar.nsynciofieldswriterestart; // Only the distance to the walls in d2wall 1352 numparts = workfc.numpe; 1353 irank = *pid; //workfc.myrank; 1354 nprocs = workfc.numpe; 1355 int nppf = numparts/nfiles; 1356 int GPID; 1357 1358 // Calculate number of parts each proc deal with and where it start and end ... 1359 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1360 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1361 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1362 1363 int descriptor; 1364 char filename[255],path[255],fieldtag_s[255]; 1365 bzero((void*)filename,255); 1366 bzero((void*)fieldtag_s,255); 1367 1368 phio_openfile_write("d2wall.", &nfiles, &nfields, &nppf, &f_descriptor); 1369 1370 field_flag=0; 1371 1372 int i; 1373 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 1374 // GPID : global part id, corresponds to rank ... 1375 // e.g : (in this example) 1376 // proc 0 : 1--4 1377 // proc 1 : 5--8 ... 1378 GPID = startpart + i; 1379 1380 // Write solution field ... 1381 sprintf(fieldtag_s,"d2wall@%d",GPID); 1382 1383 isize = (*numnp); 1384 nitems = 2; 1385 iarray[ 0 ] = (*numnp); 1386 iarray[ 1 ] = 1; //numVars = 1 1387 1388 phio_writeheader( &f_descriptor, "d2wall", (void*)iarray, &nitems, 1389 &isize, "double", phasta_iotype); 1390 1391 //nitems = (*nshg)*(*numVars); 1392 //nitems = (*numnp); 1393 1394 phio_writedatablock( &f_descriptor, "d2wall", (void*)(array1), &isize, 1395 "double", phasta_iotype ); 1396 } 1397 field_flag++; 1398 1399 if (field_flag==nfields){ 1400 phio_closefile_write(&f_descriptor); 1401 if (irank==0) { 1402 printf("\n"); 1403 } 1404 } 1405 } 1406 1407