1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdio.h> 7 #include <string.h> 8 #include <ctype.h> 9 #include <stdlib.h> 10 #include <time.h> 11 #include <math.h> 12 #include "mpi.h" 13 #include "phastaIO.h" 14 #include "rdtsc.h" 15 #include <FCMangle.h> 16 #include "new_interface.h" 17 #include "phIO.h" 18 19 //MR CHANGE 20 #include "common_c.h" 21 //MR CHANGE END 22 23 #ifdef intel 24 #include <winsock2.h> 25 #else 26 #include <unistd.h> 27 #include <strings.h> 28 #endif 29 30 //extern double cpu_speed = 2600000000.0; //for Jaguar XT5 31 //extern double cpu_speed = 2100000000.0; //for Jaguar xt4 32 //extern double cpu_speed = 850000000.0; //for Intrepid 33 34 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 35 int isThisRank; 36 37 double *value = (double*)malloc(sizeof(double)); 38 *value = 1.0*(*ivalue); 39 40 rgetMinMaxAvg(value,stats,statRanks); 41 42 /* MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 43 isThisRank=workfc.numpe+1; 44 if(*value==stats[0]) 45 isThisRank=workfc.myrank; 46 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 47 48 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 49 isThisRank=workfc.numpe+1; 50 if(*value==stats[1]) 51 isThisRank=workfc.myrank; 52 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 53 54 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 55 stats[2] /= workfc.numpe; */ 56 57 free(value); 58 } 59 60 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 61 int isThisRank; 62 63 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 64 isThisRank=workfc.numpe+1; 65 if(*value==stats[0]) 66 isThisRank=workfc.myrank; 67 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 68 69 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 70 isThisRank=workfc.numpe+1; 71 if(*value==stats[1]) 72 isThisRank=workfc.myrank; 73 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 74 75 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 76 stats[2] /= workfc.numpe; 77 78 double sqValue = (*value)*(*value), sqValueAvg = 0.; 79 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 80 sqValueAvg /= workfc.numpe; 81 // stats[3] = sqValueAvg; 82 83 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 84 } 85 86 void print_mesh_stats(void) { 87 int statRanks[2]; 88 double iStats[4], rStats[4]; 89 90 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 91 if(workfc.myrank==workfc.master) 92 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 93 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 94 if(workfc.myrank==workfc.master) 95 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 96 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 97 if(workfc.myrank==workfc.master) 98 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 99 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 100 if(workfc.myrank==workfc.master) { 101 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 102 printf("\n"); 103 } 104 } 105 106 void print_mpi_stats(void) { 107 int statRanks[2]; 108 double iStats[4], rStats[4]; 109 110 // NS equations 111 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 112 if(workfc.myrank==workfc.master) 113 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 114 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 115 if(workfc.myrank==workfc.master) 116 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 117 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 118 if(workfc.myrank==workfc.master) 119 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 120 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 121 if(workfc.myrank==workfc.master) 122 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 123 124 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 125 if(workfc.myrank==workfc.master) 126 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 127 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 128 if(workfc.myrank==workfc.master) 129 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 130 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 131 if(workfc.myrank==workfc.master) 132 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 133 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 134 if(workfc.myrank==workfc.master) 135 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 136 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 137 if(workfc.myrank==workfc.master) { 138 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 139 printf("\n"); 140 } 141 // Scalars 142 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 143 if(workfc.myrank==workfc.master) 144 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 145 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 146 if(workfc.myrank==workfc.master) 147 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 148 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 149 if(workfc.myrank==workfc.master) 150 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 151 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 152 if(workfc.myrank==workfc.master) 153 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 154 155 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 156 if(workfc.myrank==workfc.master) 157 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 158 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 159 if(workfc.myrank==workfc.master) 160 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 161 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 162 if(workfc.myrank==workfc.master) 163 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 164 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 165 if(workfc.myrank==workfc.master) 166 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 167 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 168 if(workfc.myrank==workfc.master) 169 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 170 171 172 } 173 174 //void print_system_stats(double tcorecp[2]) { 175 void print_system_stats(double *tcorecp, double *tcorecpscal) { 176 int statRanks[2]; 177 double iStats[4], rStats[4]; 178 double syst_assembly, syst_solve; 179 180 // NS equations 181 syst_assembly = tcorecp[0]; 182 syst_solve = tcorecp[1]; 183 184 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 185 if(workfc.myrank==workfc.master) 186 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 187 188 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 189 if(workfc.myrank==workfc.master) 190 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 191 192 // Scalars 193 syst_assembly = tcorecpscal[0]; 194 syst_solve = tcorecpscal[1]; 195 196 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 197 if(workfc.myrank==workfc.master) 198 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 199 200 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 201 if(workfc.myrank==workfc.master) { 202 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 203 printf("\n"); 204 } 205 //printf("rank %d - syst_assembly %f - syst_solve %f\n",workfc.myrank,syst_assembly,syst_solve); 206 } 207 208 209 210 void countfieldstowriterestart() 211 { 212 int nfields; 213 214 // printf("TEST: %d %d %d %d %d\n",timdat.istep,timdat.itseq,inpdat.nstep[0],inpdat.nstep[1],timdat.lstep); 215 216 nfields = 2; //solution, time derivatives 217 218 if(outpar.ivort == 1){ 219 nfields++; //vorticity 220 } 221 222 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 223 nfields++; //instantaneous wss in bflux.f 224 } 225 226 // if(ideformwall.eq.1) not handled yet 227 228 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 229 230 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 231 nfields = nfields +2; 232 233 //if Print Error Indicators = true (call write_error in itrdrv) 234 if(turbvar.ierrcalc == 1){ 235 nfields++; 236 } 237 238 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 239 if(outpar.ioybar == 1){ 240 nfields++; //ybar 241 242 //phase average fields 243 if(outpar.nphasesincycle >0) { 244 nfields = nfields + outpar.nphasesincycle; 245 } 246 247 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 248 nfields++; //wssbar 249 } 250 251 } 252 253 if(turbvari.irans < 0) { 254 nfields++; //dwal 255 } 256 257 } 258 259 outpar.nsynciofieldswriterestart = nfields; 260 261 if(workfc.myrank == 0) { 262 printf("Number of fields to write in restart files: %d\n", nfields); 263 } 264 } 265 266 267 void 268 Write_Restart( int* pid, 269 int* stepno, 270 int* nshg, 271 int* numVars, 272 double* array1, 273 double* array2 ) { 274 275 char fname[255]; 276 char rfile[60]; 277 char existingfile[30], linkfile[30]; 278 int irstou; 279 int magic_number = 362436; 280 int* mptr = &magic_number; 281 // time_t timenow = time ( &timenow); 282 double version=0.0; 283 int isize, nitems; 284 int iarray[10]; 285 286 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 287 openfile_(rfile,"write", &irstou); 288 289 // writing the top ascii header for the restart file 290 291 writestring_( &irstou,"# PHASTA Input File Version 2.0\n"); 292 writestring_( &irstou, 293 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 294 295 bzero( (void*)fname, 255 ); 296 sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); 297 writestring_( &irstou, fname ); 298 299 bzero( (void*)fname, 255 ); 300 gethostname(fname,255); 301 writestring_( &irstou,"# This result was produced on: "); 302 writestring_( &irstou, fname ); 303 writestring_( &irstou,"\n"); 304 305 bzero( (void*)fname, 255 ); 306 sprintf(fname,"# %s\n", ctime( &timenow )); 307 writestring_( &irstou, fname ); 308 309 isize = 1; 310 nitems = 1; 311 iarray[ 0 ] = 1; 312 writeheader_( &irstou, "byteorder magic number ", 313 (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); 314 315 nitems = 1; 316 writedatablock_( &irstou, "byteorder magic number ", 317 (void*)mptr, &nitems, "integer", phasta_iotype ); 318 319 320 bzero( (void*)fname, 255 ); 321 sprintf(fname,"number of modes : < 0 > %d\n", *nshg); 322 writestring_( &irstou, fname ); 323 324 bzero( (void*)fname, 255 ); 325 sprintf(fname,"number of variables : < 0 > %d\n", *numVars); 326 writestring_( &irstou, fname ); 327 328 329 isize = (*nshg)*(*numVars); 330 nitems = 3; 331 iarray[ 0 ] = (*nshg); 332 iarray[ 1 ] = (*numVars); 333 iarray[ 2 ] = (*stepno); 334 writeheader_( &irstou, "solution ", 335 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 336 337 338 nitems = (*nshg)*(*numVars); 339 writedatablock_( &irstou, "solution ", 340 (void*)(array1), &nitems, "double", phasta_iotype ); 341 342 343 344 nitems = 3; 345 writeheader_( &irstou, "time derivative of solution ", 346 (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 347 348 349 nitems = (*nshg)*(*numVars); 350 writedatablock_( &irstou, "time derivative of solution ", 351 (void*)(array2), &nitems, "double", phasta_iotype ); 352 353 354 closefile_( &irstou, "write" ); 355 */ 356 //MPI_Barrier(MPI_COMM_WORLD); 357 358 /////////////////////////////// Start of writing using new-lib //////////////////////////// 359 360 //MR CHANGE 361 int nfiles; 362 int nfields; 363 int numparts; 364 int irank; 365 int nprocs; 366 367 // First, count the number of fields to write and store the result in 368 countfieldstowriterestart(); 369 370 // Retrieve and compute the parameters required for SyncIO 371 nfiles = outpar.nsynciofiles; 372 nfields = outpar.nsynciofieldswriterestart; 373 numparts = workfc.numpe; 374 irank = *pid; //workfc.myrank; 375 nprocs = workfc.numpe; 376 //MR CHANGE END 377 int nppf = numparts/nfiles; 378 int GPID; 379 380 // Calculate number of parts each proc deal with and where it start and end ... 381 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 382 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 383 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 384 385 int descriptor; 386 char filename[255],path[255],fieldtag_s[255]; 387 bzero((void*)filename,255); 388 bzero((void*)fieldtag_s,255); 389 390 phio_restartname(stepno, filename); 391 phio_openfile_write(filename, &nfiles, &nfields, &nppf, &f_descriptor); 392 393 //MR CHANGE 394 // Measure the time - End of timer 395 // MPI_Barrier(MPI_COMM_WORLD); 396 // timer_end = rdtsc(); 397 // time_span=(double)((timer_end-timer_start)/cpu_speed); 398 // if (*pid==0) { 399 // printf("Time: 'openfile' of %s with %d fields and %d files is: %f s\n",filename,nfields,nfiles,time_span); 400 // printf("*****************************\n"); 401 // } 402 //MR CHANGE END 403 404 field_flag=0; 405 406 int i; 407 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 408 // GPID : global part id, corresponds to rank ... 409 // e.g : (in this example) 410 // proc 0 : 1--4 411 // proc 1 : 5--8 ... 412 GPID = startpart + i; 413 414 // Write solution field ... 415 sprintf(fieldtag_s,"solution@%d",GPID); 416 417 isize = (*nshg)*(*numVars); 418 nitems = 3; 419 iarray[ 0 ] = (*nshg); 420 iarray[ 1 ] = (*numVars); 421 iarray[ 2 ] = (*stepno); 422 423 //MR CHANGE 424 // Measure the time - Start the timer 425 // MPI_Barrier(MPI_COMM_WORLD); 426 // timer_start = rdtsc(); 427 //MR CHANGE END 428 429 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 430 431 //MR CHANGE 432 // Measure the time - End of timer 433 // MPI_Barrier(MPI_COMM_WORLD); 434 // timer_end = rdtsc(); 435 // time_span=(double)((timer_end-timer_start)/cpu_speed); 436 // if (*pid==0) { 437 // printf("\n*****************************\n"); 438 // printf("Time: header for 'Solution': %f s\n",time_span); 439 // } 440 //MR CHANGE END 441 442 nitems = (*nshg)*(*numVars); 443 444 //MR CHANGE 445 // Measure the time - Start the timer 446 // MPI_Barrier(MPI_COMM_WORLD); 447 // timer_start = rdtsc(); 448 //MR CHANGE END 449 450 writedatablock( &f_descriptor, fieldtag_s, (void*)(array1), &isize, "double", phasta_iotype ); 451 452 //MR CHANGE 453 // Measure the time - End of timer 454 // MPI_Barrier(MPI_COMM_WORLD); 455 // timer_end = rdtsc(); 456 // time_span=(double)((timer_end-timer_start)/cpu_speed); 457 458 // int isizemin,isizemax,isizetot; 459 // double sizemin,sizemax,sizeavg,sizetot,rate; 460 461 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 462 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 463 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 464 465 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 466 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 467 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 468 // sizeavg=(double)(1.0*sizetot/workfc.numpe); 469 // rate=(double)(1.0*sizetot/time_span); 470 471 // if (*pid==0) { 472 // printf("Time: block for 'Solution': %f s\n",time_span); 473 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 474 475 // } 476 //MR CHANGE END 477 478 } 479 field_flag++; 480 481 for ( i = 0; i < nppp; i++) { 482 483 // GPID : global part id, corresponds to rank ... 484 // e.g : (in this example) 485 // proc 0 : 1--4 486 // proc 1 : 5--8 ... 487 GPID = startpart + i; 488 489 // Write solution field ... 490 sprintf(fieldtag_s,"time derivative of solution@%d",GPID); 491 492 isize = (*nshg)*(*numVars); 493 nitems = 3; 494 iarray[ 0 ] = (*nshg); 495 iarray[ 1 ] = (*numVars); 496 iarray[ 2 ] = (*stepno); 497 498 //MR CHANGE 499 // Measure the time - Start the timer 500 // MPI_Barrier(MPI_COMM_WORLD); 501 // timer_start = rdtsc(); 502 //MR CHANGE END 503 504 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 505 506 //MR CHANGE 507 // Measure the time - End of timer 508 // MPI_Barrier(MPI_COMM_WORLD); 509 // timer_end = rdtsc(); 510 // time_span=(double)((timer_end-timer_start)/cpu_speed); 511 // if (*pid==0) { 512 // printf("Time: header for 'Time Derivative of solution': %f s\n",time_span); 513 // } 514 //MR CHANGE END 515 516 nitems = (*nshg)*(*numVars); 517 518 //MR CHANGE 519 // Measure the time - Start the timer 520 // MPI_Barrier(MPI_COMM_WORLD); 521 // timer_start = rdtsc(); 522 //MR CHANGE END 523 524 writedatablock( &f_descriptor, fieldtag_s, (void*)(array2), &isize, "double", phasta_iotype ); 525 526 //MR CHANGE 527 // Measure the time - End of timer 528 // MPI_Barrier(MPI_COMM_WORLD); 529 // timer_end = rdtsc(); 530 // time_span=(double)((timer_end-timer_start)/cpu_speed); 531 532 // int isizemin,isizemax,isizetot; 533 // double sizemin,sizemax,sizeavg,sizetot,rate; 534 535 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 536 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 537 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 538 539 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 540 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 541 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 542 // sizeavg=sizetot/workfc.numpe; 543 // rate=sizetot/time_span; 544 545 // if (*pid==0) { 546 // printf("Time: block for 'Time Derivative of Solution': %f s\n",time_span); 547 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 548 // printf("*****************************\n"); 549 550 // } 551 //MR CHANGE END 552 553 } 554 field_flag++; 555 556 if (field_flag==nfields){ 557 558 //MR CHANGE 559 // Measure the time - Start the timer 560 // MPI_Barrier(MPI_COMM_WORLD); 561 // timer_start = rdtsc(); 562 //MR CHANGE END 563 564 closefile(&f_descriptor, "write"); 565 566 //MR CHANGE 567 // Measure the time - End of timer 568 // MPI_Barrier(MPI_COMM_WORLD); 569 // timer_end = rdtsc(); 570 // time_span=(double)((timer_end-timer_start)/cpu_speed); 571 // if (*pid==0) { 572 // printf("\n*****************************\n"); 573 // printf("Time: 'closefile' is: %f s\n",time_span); 574 // } 575 //MR CHANGE END 576 577 //MR CHANGE 578 // Measure the time - Start the timer 579 // MPI_Barrier(MPI_COMM_WORLD); 580 // timer_start = rdtsc(); 581 //MR CHANGE END 582 583 finalizephmpiio(&f_descriptor); 584 585 //MR CHANGE 586 // Measure the time - End of timer 587 // MPI_Barrier(MPI_COMM_WORLD); 588 // timer_end = rdtsc(); 589 // time_span=(double)((timer_end-timer_start)/cpu_speed); 590 if (*pid==0) { 591 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 592 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 593 printf("\n"); 594 // printf("*****************************\n"); 595 } 596 } 597 //MR CHANGE END 598 599 600 601 /////////////////////////////////////////////////////////////////////////////////////////// 602 603 /* create a soft link of the restart we just wrote to restart.latest 604 this is the file the next run will always try to start from */ 605 606 /* sprintf( linkfile, "restart.latest.%d", *pid+1 ); 607 unlink( linkfile ); 608 sprintf( existingfile, "restart.%d.%d", *stepno, *pid+1 ); 609 link( existingfile, linkfile ); 610 */ 611 } 612 613 void 614 Write_Error( int* pid, 615 int* stepno, 616 int* nshg, 617 int* numVars, 618 double* array1 ) { 619 620 621 char fname[255]; 622 char rfile[60]; 623 int irstou; 624 int magic_number = 362436; 625 int* mptr = &magic_number; 626 //printf("Time is commented\n"); 627 //time_t timenow = time ( &timenow); 628 //printf("Yes\n"); 629 double version=0.0; 630 int isize, nitems; 631 int iarray[10]; 632 633 /*sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 634 openfile_(rfile,"append", &irstou); 635 636 isize = (*nshg)*(*numVars); 637 nitems = 3; 638 iarray[ 0 ] = (*nshg); 639 iarray[ 1 ] = (*numVars); 640 iarray[ 2 ] = (*stepno); 641 writeheader_( &irstou, "errors", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 642 643 644 nitems = (*nshg)*(*numVars); 645 writedatablock_( &irstou, "errors ", (void*)(array1), &nitems, "double", phasta_iotype ); 646 647 closefile_( &irstou, "append" );*/ 648 649 /////////////////////////////// Start of writing using new-lib //////////////////////////// 650 651 int nfiles; 652 int nfields; 653 int numparts; 654 int irank; 655 int nprocs; 656 657 // unsigned long long timer_start; 658 // unsigned long long timer_end; 659 // double time_span; 660 661 nfiles = outpar.nsynciofiles; 662 nfields = outpar.nsynciofieldswriterestart; 663 numparts = workfc.numpe; 664 irank = *pid; //workfc.myrank; 665 nprocs = workfc.numpe; 666 667 int nppf = numparts/nfiles; 668 int GPID; 669 670 // Calculate number of parts each proc deal with and where it start and end ... 671 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 672 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 673 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 674 675 field_flag++; 676 677 char fieldtag[255]; 678 679 int i; 680 for ( i = 0; i < nppp; i++ ) { 681 GPID = startpart + i; 682 sprintf(fieldtag,"errors@%d",GPID); 683 684 if(*pid==0) { 685 // printf("\n*****************************\n"); 686 printf("\n"); 687 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldtag); 688 } 689 690 isize = (*nshg)*(*numVars); 691 nitems = 3; 692 iarray[ 0 ] = (*nshg); 693 iarray[ 1 ] = (*numVars); 694 iarray[ 2 ] = (*stepno); 695 696 //MR CHANGE 697 // Measure the time - Start the timer 698 // MPI_Barrier(MPI_COMM_WORLD); 699 // timer_start = rdtsc(); 700 //MR CHANGE END 701 702 writeheader( &f_descriptor, fieldtag, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 703 704 //MR CHANGE 705 // Measure the time - End of timer 706 // MPI_Barrier(MPI_COMM_WORLD); 707 // timer_end = rdtsc(); 708 // time_span=(double)((timer_end-timer_start)/cpu_speed); 709 // if (*pid==0) { 710 // printf("Time: header for 'error': %f s\n",time_span); 711 // } 712 //MR CHANGE END 713 714 //MR CHANGE 715 // Measure the time - Start the timer 716 // MPI_Barrier(MPI_COMM_WORLD); 717 // timer_start = rdtsc(); 718 //MR CHANGE END 719 720 writedatablock( &f_descriptor, fieldtag, (void*)array1, &isize, "double", phasta_iotype ); 721 722 //MR CHANGE 723 // Measure the time - End of timer 724 // MPI_Barrier(MPI_COMM_WORLD); 725 // timer_end = rdtsc(); 726 // time_span=(double)((timer_end-timer_start)/cpu_speed); 727 728 // int isizemin,isizemax,isizetot; 729 // double sizemin,sizemax,sizeavg,sizetot,rate; 730 731 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 732 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 733 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 734 735 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 736 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 737 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 738 // sizeavg=sizetot/workfc.numpe; 739 // rate=sizetot/time_span; 740 741 // if (*pid==0) { 742 // printf("Time: block for 'error': %f s\n",time_span); 743 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 744 // printf("*****************************\n"); 745 // } 746 //MR CHANGE END 747 748 } 749 750 // MPI_Barrier(MPI_COMM_WORLD); 751 // timer_end = rdtsc(); 752 // time_span=(double)(timer_end-timer_start)/cpu_speed; 753 754 // if (*pid==0) { 755 // printf("Field 'error' written in: %f s\n",time_span); 756 // printf("Write field '%s' finished! \n",fieldtag); 757 // } 758 759 if (field_flag==nfields){ 760 761 //MR CHANGE 762 // Measure the time - Start the timer 763 // MPI_Barrier(MPI_COMM_WORLD); 764 // timer_start = rdtsc(); 765 //MR CHANGE END 766 767 closefile(&f_descriptor, "write"); 768 769 //MR CHANGE 770 // Measure the time - End of timer 771 // MPI_Barrier(MPI_COMM_WORLD); 772 // timer_end = rdtsc(); 773 // time_span=(double)((timer_end-timer_start)/cpu_speed); 774 // if (*pid==0) { 775 // printf("\n*****************************\n"); 776 // printf("Time: 'closefile' is: %f s\n",time_span); 777 // } 778 //MR CHANGE END 779 780 //MR CHANGE 781 // Measure the time - Start the timer 782 // MPI_Barrier(MPI_COMM_WORLD); 783 // timer_start = rdtsc(); 784 //MR CHANGE END 785 786 finalizephmpiio(&f_descriptor); 787 788 //MR CHANGE 789 // Measure the time - End of timer 790 // MPI_Barrier(MPI_COMM_WORLD); 791 // timer_end = rdtsc(); 792 // time_span=(double)((timer_end-timer_start)/cpu_speed); 793 if (*pid==0) { 794 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 795 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 796 printf("\n"); 797 // printf("*****************************\n"); 798 } 799 } 800 //MR CHANGE END 801 802 /////////////////////////////////////////////////////////////////////////////////////////// 803 804 805 } 806 807 808 void 809 Write_Displ( int* pid, 810 int* stepno, 811 int* nshg, 812 int* numVars, 813 double* array1 ) { 814 815 816 char fname[255]; 817 char rfile[60]; 818 int irstou; 819 int magic_number = 362436; 820 int* mptr = &magic_number; 821 time_t timenow = time ( &timenow); 822 double version=0.0; 823 int isize, nitems; 824 int iarray[10]; 825 826 sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 827 openfile(rfile,"append", &irstou); 828 829 isize = (*nshg)*(*numVars); 830 nitems = 3; 831 iarray[ 0 ] = (*nshg); 832 iarray[ 1 ] = (*numVars); 833 iarray[ 2 ] = (*stepno); 834 writeheader( &irstou, "displacement", (void*)iarray, &nitems, &isize, "double", phasta_iotype ); 835 836 nitems = (*nshg)*(*numVars); 837 writedatablock( &irstou, "displacement", (void*)(array1), &nitems, "double", phasta_iotype ); 838 839 closefile( &irstou, "append" ); 840 } 841 842 void 843 Write_Field( int *pid, 844 char* filemode, 845 char* fieldtag, 846 int* tagsize, 847 void* array, 848 char* arraytype, 849 int* nshg, 850 int* numvars, 851 int* stepno) { 852 853 //printf("Rank is %d, field is %s, tagsize is %d, nshg is %d, numvars is %d\n",*pid,fieldtag,*tagsize,*nshg,*numvars); 854 855 // char rfile[32]; 856 // assuming restart.sn.(pid+1) 857 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 858 859 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 860 strncpy(fieldlabel, fieldtag, *tagsize); 861 fieldlabel[*tagsize] = '\0'; 862 863 int irstou; 864 int magic_number = 362436; 865 int* mptr = &magic_number; 866 double version=0.0; 867 int isize, nitems; 868 int iarray[10]; 869 870 char fmode[10]; 871 if(!strncmp(filemode,"w",1)) 872 strcpy(fmode,"write"); 873 else // default is append 874 strcpy(fmode,"append"); 875 876 char datatype[10]; 877 if(!strncmp(arraytype,"i",1)) 878 strcpy(datatype,"int"); 879 else // default is double 880 strcpy(datatype,"double"); 881 882 /* openfile_(rfile, fmode, &irstou); 883 884 nitems = 3; // assuming field will write 3 items in iarray 885 iarray[ 0 ] = (*nshg); 886 iarray[ 1 ] = (*numvars); 887 iarray[ 2 ] = (*stepno); 888 889 isize = (*nshg)*(*numvars); 890 writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 891 892 nitems = (*nshg)*(*numvars); 893 writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 894 closefile_( &irstou, fmode); 895 */ 896 /////////////////////////////// Start of writing using new-lib //////////////////////////// 897 898 int nfiles; 899 int nfields; 900 int numparts; 901 int irank; 902 int nprocs; 903 904 // unsigned long long timer_start; 905 // unsigned long long timer_end; 906 // double time_span; 907 908 nfiles = outpar.nsynciofiles; 909 nfields = outpar.nsynciofieldswriterestart; 910 numparts = workfc.numpe; 911 irank = *pid; //workfc.myrank; 912 nprocs = workfc.numpe; 913 914 int nppf = numparts/nfiles; 915 int GPID; 916 917 // Calculate number of parts each proc deal with and where it start and end ... 918 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 919 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 920 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 921 922 char filename[255],path[255],fieldtag_s[255]; 923 bzero((void*)filename,255); 924 bzero((void*)fieldtag_s,255); 925 926 strncpy(fieldlabel, fieldtag, *tagsize); 927 928 field_flag++; 929 if(*pid==0) { 930 // printf("\n*****************************\n"); 931 printf("\n"); 932 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 933 } 934 935 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 936 937 // MPI_Barrier(MPI_COMM_WORLD); 938 // timer_start = rdtsc(); 939 940 int i; 941 for ( i = 0; i < nppp; i++ ) { 942 GPID = startpart + i; 943 944 // Write solution field ... 945 sprintf(fieldtag_s,"%s@%d",fieldlabel,GPID); 946 947 isize = (*nshg)*(*numvars); 948 nitems = 3; 949 iarray[ 0 ] = (*nshg); 950 iarray[ 1 ] = (*numvars); 951 iarray[ 2 ] = (*stepno); 952 953 //MR CHANGE 954 // Measure the time - Start the timer 955 // MPI_Barrier(MPI_COMM_WORLD); 956 // timer_start = rdtsc(); 957 //MR CHANGE END 958 959 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, datatype, phasta_iotype); 960 961 //MR CHANGE 962 // Measure the time - End of timer 963 // MPI_Barrier(MPI_COMM_WORLD); 964 // timer_end = rdtsc(); 965 // time_span=(double)((timer_end-timer_start)/cpu_speed); 966 // if (*pid==0) { 967 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 968 // } 969 //MR CHANGE END 970 971 nitems = (*nshg)*(*numvars); 972 973 //MR CHANGE 974 // Measure the time - Start the timer 975 // MPI_Barrier(MPI_COMM_WORLD); 976 // timer_start = rdtsc(); 977 //MR CHANGE END 978 979 writedatablock( &f_descriptor, fieldtag_s, array, &isize, datatype, phasta_iotype ); 980 981 //MR CHANGE 982 // Measure the time - End of timer 983 // MPI_Barrier(MPI_COMM_WORLD); 984 // timer_end = rdtsc(); 985 // time_span=(double)((timer_end-timer_start)/cpu_speed); 986 987 // int isizemin,isizemax,isizetot; 988 // double sizemin,sizemax,sizeavg,sizetot,rate; 989 990 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 991 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 992 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 993 994 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 995 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 996 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 997 // sizeavg=sizetot/workfc.numpe; 998 // rate=sizetot/time_span; 999 1000 // if (*pid==0) { 1001 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 1002 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 1003 // printf("*****************************\n"); 1004 // } 1005 //MR CHANGE END 1006 1007 } 1008 1009 // MPI_Barrier(MPI_COMM_WORLD); 1010 // timer_end = rdtsc(); 1011 // time_span=(double)(timer_end-timer_start)/cpu_speed; 1012 1013 // if (*pid==0) { 1014 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 1015 // printf("Write field '%s' finished! \n",fieldtag_s); 1016 // } 1017 1018 // if (field_flag==nfields){ 1019 // closefile_(&f_descriptor, "write"); 1020 // finalizephmpiio_(&f_descriptor); 1021 // if(*pid==0) { 1022 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1023 // printf("\n*****************************\n"); 1024 // } 1025 // } 1026 1027 if (field_flag==nfields){ 1028 1029 //MR CHANGE 1030 // Measure the time - Start the timer 1031 // MPI_Barrier(MPI_COMM_WORLD); 1032 // timer_start = rdtsc(); 1033 //MR CHANGE END 1034 1035 closefile(&f_descriptor, "write"); 1036 1037 //MR CHANGE 1038 // Measure the time - End of timer 1039 // MPI_Barrier(MPI_COMM_WORLD); 1040 // timer_end = rdtsc(); 1041 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1042 // if (*pid==0) { 1043 // printf("\n*****************************\n"); 1044 // printf("Time: 'closefile' is: %f s\n",time_span); 1045 // } 1046 //MR CHANGE END 1047 1048 //MR CHANGE 1049 // Measure the time - Start the timer 1050 // MPI_Barrier(MPI_COMM_WORLD); 1051 // timer_start = rdtsc(); 1052 //MR CHANGE END 1053 1054 finalizephmpiio(&f_descriptor); 1055 1056 //MR CHANGE 1057 // Measure the time - End of timer 1058 // MPI_Barrier(MPI_COMM_WORLD); 1059 // timer_end = rdtsc(); 1060 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1061 if (*pid==0) { 1062 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1063 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1064 printf("\n"); 1065 // printf("*****************************\n"); 1066 } 1067 } 1068 //MR CHANGE END 1069 1070 /////////////////////////////////////////////////////////////////////////////////////////// 1071 1072 free(fieldlabel); 1073 } 1074 1075 //MR CHANGE 1076 void 1077 Write_PhAvg( int* pid, 1078 char* filemode, 1079 char* fieldtag, 1080 int* tagsize, 1081 int* iphase, 1082 void* array, 1083 char* arraytype, 1084 int* nshg, 1085 int* numvars, 1086 int* stepno) { 1087 1088 char rfile[32]; 1089 // assuming restart_phase_avg_<sn>.<iphase>.<pid+1> 1090 sprintf(rfile,"restart_phase_avg_%d.%d.%d",*stepno,*iphase,*pid+1); 1091 1092 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 1093 strncpy(fieldlabel, fieldtag, *tagsize); 1094 fieldlabel[*tagsize] = '\0'; 1095 1096 int irstou; 1097 int isize, nitems; 1098 int iarray[10]; 1099 1100 char fmode[10]; 1101 if(!strncmp(filemode,"w",1)) 1102 strcpy(fmode,"write"); 1103 else // default is append 1104 strcpy(fmode,"append"); 1105 1106 char datatype[10]; 1107 if(!strncmp(arraytype,"i",1)) 1108 strcpy(datatype,"int"); 1109 else // default is double 1110 strcpy(datatype,"double"); 1111 1112 openfile(rfile, fmode, &irstou); 1113 1114 if(!strcmp(fmode,"write")) { 1115 // may be create a routine for 'top' portion under write mode 1116 int magic_number = 362436; 1117 int* mptr = &magic_number; 1118 time_t timenow = time ( &timenow); 1119 double version=0.0; 1120 1121 /* writing the top ascii header for the restart file */ 1122 1123 writestring( &irstou,"# PHASTA Input File Version 2.0\n"); 1124 writestring( &irstou, 1125 "# format \"keyphrase : sizeofnextblock usual headers\"\n"); 1126 1127 char fname[255]; 1128 bzero( (void*)fname, 255 ); 1129 sprintf(fname,"# Output generated by phasta version (NOT YET CURRENT): %lf \n", version); 1130 writestring( &irstou, fname ); 1131 1132 bzero( (void*)fname, 255 ); 1133 gethostname(fname,255); 1134 writestring( &irstou,"# This result was produced on: "); 1135 writestring( &irstou, fname ); 1136 writestring( &irstou,"\n"); 1137 1138 bzero( (void*)fname, 255 ); 1139 sprintf(fname,"# %s\n", ctime( &timenow )); 1140 writestring( &irstou, fname ); 1141 1142 isize = 1; 1143 nitems = 1; 1144 iarray[ 0 ] = 1; 1145 writeheader( &irstou, "byteorder magic number ", 1146 (void*)iarray, &nitems, &isize, "integer", phasta_iotype ); 1147 nitems = 1; 1148 writedatablock( &irstou, "byteorder magic number ", 1149 (void*)mptr, &nitems, "integer", phasta_iotype ); 1150 } 1151 1152 nitems = 3; // assuming field will write 3 items in iarray 1153 iarray[ 0 ] = (*nshg); 1154 iarray[ 1 ] = (*numvars); 1155 iarray[ 2 ] = (*stepno); 1156 1157 isize = (*nshg)*(*numvars); 1158 writeheader( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 1159 1160 nitems = (*nshg)*(*numvars); 1161 writedatablock( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 1162 1163 closefile( &irstou, fmode); 1164 1165 free(fieldlabel); 1166 } 1167 1168 //MR CHANGE 1169 void 1170 Write_PhAvg2( int* pid, 1171 char* filemode, 1172 char* fieldtag, 1173 int* tagsize, 1174 int* iphase, 1175 int* nphasesincycle, 1176 void* array, 1177 char* arraytype, 1178 int* nshg, 1179 int* numvars, 1180 int* stepno) { 1181 1182 // char rfile[32]; 1183 // assuming restart.sn.(pid+1) 1184 // sprintf(rfile,"restart.%d.%d",*stepno,*pid+1); 1185 1186 int addtagsize=0; // phase number is added to the name of the field 1187 if(*iphase<10) 1188 addtagsize=1; 1189 else if(*iphase<100) 1190 addtagsize=2; 1191 else if(*iphase<1000) 1192 addtagsize=3; 1193 1194 int tagsize2; 1195 tagsize2=*tagsize+addtagsize; 1196 1197 // char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 1198 // strncpy(fieldlabel, fieldtag, *tagsize); 1199 // fieldlabel[*tagsize] = '\0'; 1200 1201 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 1202 strncpy(fieldlabel, fieldtag, *tagsize); 1203 fieldlabel[tagsize2] = '\0'; 1204 1205 char straddtagsize[10]; 1206 sprintf(straddtagsize,"%d",*iphase); 1207 1208 if(*iphase<10) { 1209 fieldlabel[tagsize2-1]=straddtagsize[0]; 1210 } 1211 else if(*iphase<100) { 1212 fieldlabel[tagsize2-2]=straddtagsize[0]; 1213 fieldlabel[tagsize2-1]=straddtagsize[1]; 1214 } 1215 else if(*iphase<1000) { 1216 fieldlabel[tagsize2-3]=straddtagsize[0]; 1217 fieldlabel[tagsize2-2]=straddtagsize[1]; 1218 fieldlabel[tagsize2-1]=straddtagsize[2]; 1219 } 1220 1221 int irstou; 1222 int magic_number = 362436; 1223 int* mptr = &magic_number; 1224 double version=0.0; 1225 int isize, nitems; 1226 int iarray[10]; 1227 1228 char fmode[10]; 1229 if(!strncmp(filemode,"w",1)) 1230 strcpy(fmode,"write"); 1231 else // default is append 1232 strcpy(fmode,"append"); 1233 1234 char datatype[10]; 1235 if(!strncmp(arraytype,"i",1)) 1236 strcpy(datatype,"int"); 1237 else // default is double 1238 strcpy(datatype,"double"); 1239 1240 // 1241 // // if(*iphase==1) //open the file but then keep it open for the remaining cycles 1242 // openfile_(rfile, fmode, &irstou); 1243 // 1244 // // printf("iphase: %d - pid: %d - irstou %d\n",*iphase,*pid,irstou); 1245 // 1246 // 1247 // nitems = 3; // assuming field will write 3 items in iarray 1248 // iarray[ 0 ] = (*nshg); 1249 // iarray[ 1 ] = (*numvars); 1250 // iarray[ 2 ] = (*stepno); 1251 // 1252 // isize = (*nshg)*(*numvars); 1253 // writeheader_( &irstou, fieldlabel, (void*)iarray, &nitems, &isize, datatype, phasta_iotype ); 1254 // 1255 // nitems = (*nshg)*(*numvars); 1256 // writedatablock_( &irstou, fieldlabel, array, &nitems, datatype, phasta_iotype ); 1257 // 1258 // // if(*iphase==*nphasesincycle) //close the file after nphasesincycle 1259 // closefile_( &irstou, fmode); 1260 // 1261 1262 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1263 1264 int nfiles; 1265 int nfields; 1266 int numparts; 1267 int irank; 1268 int nprocs; 1269 // unsigned long long timer_start; 1270 // unsigned long long timer_end; 1271 // double time_span; 1272 1273 nfiles = outpar.nsynciofiles; 1274 nfields = outpar.nsynciofieldswriterestart; 1275 numparts = workfc.numpe; 1276 irank = *pid; //workfc.myrank; 1277 nprocs = workfc.numpe; 1278 1279 int nppf = numparts/nfiles; 1280 int GPID; 1281 1282 // Calculate number of parts each proc deal with and where it start and end ... 1283 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1284 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1285 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1286 1287 //int descriptor; 1288 char filename[255],path[255],fieldtag_s[255]; 1289 bzero((void*)filename,255); 1290 bzero((void*)fieldtag_s,255); 1291 1292 // char * namer; 1293 // namer = strtok(fieldlabel," "); 1294 // strncpy(fieldlabel, fieldtag, *tagsize); 1295 1296 field_flag++; 1297 if(*pid==0) { 1298 // printf("\n*****************************\n"); 1299 printf("\n"); 1300 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 1301 } 1302 1303 sprintf(filename,"restart-dat.%d.%d",*stepno,((int)(irank/(nprocs/nfiles))+1)); 1304 1305 int i; 1306 for ( i = 0; i < nppp; i++ ) { 1307 GPID = startpart + i; 1308 1309 // Write solution field ... 1310 sprintf(fieldtag_s,"%s@%d",fieldlabel,GPID); 1311 1312 //printf("This is %d and fieldtag_s is %s \n",myrank,fieldtag_s); 1313 1314 isize = (*nshg)*(*numvars); 1315 nitems = 3; 1316 iarray[ 0 ] = (*nshg); 1317 iarray[ 1 ] = (*numvars); 1318 iarray[ 2 ] = (*stepno); 1319 1320 //MR CHANGE 1321 // Measure the time - Start the timer 1322 // MPI_Barrier(MPI_COMM_WORLD); 1323 // timer_start = rdtsc(); 1324 //MR CHANGE END 1325 1326 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 1327 1328 //MR CHANGE 1329 // Measure the time - End of timer 1330 // MPI_Barrier(MPI_COMM_WORLD); 1331 // timer_end = rdtsc(); 1332 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1333 // if (*pid==0) { 1334 // printf("Time: header for '%s': %f s\n",fieldtag_s,time_span); 1335 // } 1336 //MR CHANGE END 1337 1338 nitems = (*nshg)*(*numvars); 1339 1340 //MR CHANGE 1341 // Measure the time - Start the timer 1342 // MPI_Barrier(MPI_COMM_WORLD); 1343 // timer_start = rdtsc(); 1344 //MR CHANGE END 1345 1346 writedatablock( &f_descriptor, fieldtag_s, array, &isize, "double", phasta_iotype ); 1347 1348 //MR CHANGE 1349 // Measure the time - End of timer 1350 // MPI_Barrier(MPI_COMM_WORLD); 1351 // timer_end = rdtsc(); 1352 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1353 1354 // int isizemin,isizemax,isizetot; 1355 // double sizemin,sizemax,sizeavg,sizetot,rate; 1356 1357 // MPI_Allreduce(&isize,&isizemin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 1358 // MPI_Allreduce(&isize,&isizemax,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); 1359 // MPI_Allreduce(&isize,&isizetot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD); 1360 1361 // sizemin=(double)(8.0*isizemin/1024.0/1024.0); 1362 // sizemax=(double)(8.0*isizemax/1024.0/1024.0); 1363 // sizetot=(double)(8.0*isizetot/1024.0/1024.0); 1364 // sizeavg=sizetot/workfc.numpe; 1365 // rate=sizetot/time_span; 1366 1367 // if (*pid==0) { 1368 // printf("Time: block for '%s': %f s\n",fieldtag_s,time_span); 1369 // printf("Time: block: Min= %f MB; Max= %f MB; Avg= %f MB; Tot= %f MB; Rate= %f MB/s; \n",sizemin,sizemax,sizeavg,sizetot,rate); 1370 // printf("*****************************\n"); 1371 // } 1372 //MR CHANGE END 1373 1374 } 1375 1376 // if (*pid==0) { 1377 // printf("Field '%s' written in: %f s\n",fieldtag,time_span); 1378 // printf("Write field '%s' finished! \n",fieldtag_s); 1379 // } 1380 1381 // 1382 // if (field_flag==nfields){ 1383 // closefile_(&f_descriptor, "write"); 1384 // finalizephmpiio_(&f_descriptor); 1385 // if(*pid==0) { 1386 // printf("Last field %d '%s' finished! \n",nfields, fieldtag_s); 1387 // printf("\n*****************************\n"); 1388 // } 1389 // } 1390 1391 if (field_flag==nfields){ 1392 1393 //MR CHANGE 1394 // Measure the time - Start the timer 1395 // MPI_Barrier(MPI_COMM_WORLD); 1396 // timer_start = rdtsc(); 1397 //MR CHANGE END 1398 1399 closefile(&f_descriptor, "write"); 1400 1401 //MR CHANGE 1402 // Measure the time - End of timer 1403 // MPI_Barrier(MPI_COMM_WORLD); 1404 // timer_end = rdtsc(); 1405 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1406 // if (*pid==0) { 1407 // printf("\n*****************************\n"); 1408 // printf("Time: 'closefile' is: %f s\n",time_span); 1409 // } 1410 //MR CHANGE END 1411 1412 //MR CHANGE 1413 // Measure the time - Start the timer 1414 // MPI_Barrier(MPI_COMM_WORLD); 1415 // timer_start = rdtsc(); 1416 //MR CHANGE END 1417 1418 finalizephmpiio(&f_descriptor); 1419 1420 //MR CHANGE 1421 // Measure the time - End of timer 1422 // MPI_Barrier(MPI_COMM_WORLD); 1423 // timer_end = rdtsc(); 1424 // time_span=(double)((timer_end-timer_start)/cpu_speed); 1425 if (*pid==0) { 1426 // printf("Time: 'finalizephmpiio' is: %f s\n",time_span); 1427 // printf("Last field %d '%s' finished! \n",nfields, fieldtag); 1428 printf("\n"); 1429 // printf("*****************************\n"); 1430 } 1431 } 1432 //MR CHANGE END 1433 1434 /////////////////////////////////////////////////////////////////////////////////////////// 1435 1436 free(fieldlabel); 1437 } 1438 1439 1440 void 1441 Write_d2wall( int* pid, 1442 int* numnp, 1443 double* array1 ) { 1444 1445 // time_t timenow = time ( &timenow); 1446 int isize, nitems; 1447 int iarray[10]; 1448 1449 // MPI_Barrier(MPI_COMM_WORLD); 1450 1451 /////////////////////////////// Start of writing using new-lib //////////////////////////// 1452 1453 int nfiles; 1454 int nfields; 1455 int numparts; 1456 int irank; 1457 int nprocs; 1458 1459 // First, count the number of fields to write and store the result in 1460 //countfieldstowriterestart(); 1461 1462 // Retrieve and compute the parameters required for SyncIO 1463 nfiles = outpar.nsynciofiles; 1464 nfields = 1; //outpar.nsynciofieldswriterestart; // Only the distance to the walls in d2wall 1465 numparts = workfc.numpe; 1466 irank = *pid; //workfc.myrank; 1467 nprocs = workfc.numpe; 1468 int nppf = numparts/nfiles; 1469 int GPID; 1470 1471 // Calculate number of parts each proc deal with and where it start and end ... 1472 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 1473 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 1474 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 1475 1476 int descriptor; 1477 char filename[255],path[255],fieldtag_s[255]; 1478 bzero((void*)filename,255); 1479 bzero((void*)fieldtag_s,255); 1480 1481 phio_openfile_write("d2wall.", &nfiles, &nfields, &nppf, &f_descriptor); 1482 1483 field_flag=0; 1484 1485 int i; 1486 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 1487 // GPID : global part id, corresponds to rank ... 1488 // e.g : (in this example) 1489 // proc 0 : 1--4 1490 // proc 1 : 5--8 ... 1491 GPID = startpart + i; 1492 1493 // Write solution field ... 1494 sprintf(fieldtag_s,"d2wall@%d",GPID); 1495 1496 isize = (*numnp); 1497 nitems = 2; 1498 iarray[ 0 ] = (*numnp); 1499 iarray[ 1 ] = 1; //numVars = 1 1500 1501 writeheader( &f_descriptor, fieldtag_s, (void*)iarray, &nitems, &isize, "double", phasta_iotype); 1502 1503 //nitems = (*nshg)*(*numVars); 1504 //nitems = (*numnp); 1505 1506 writedatablock( &f_descriptor, fieldtag_s, (void*)(array1), &isize, "double", phasta_iotype ); 1507 1508 1509 } 1510 field_flag++; 1511 1512 if (field_flag==nfields){ 1513 1514 closefile(&f_descriptor, "write"); 1515 1516 finalizephmpiio(&f_descriptor); 1517 1518 if (irank==0) { 1519 printf("\n"); 1520 } 1521 } 1522 } 1523 1524