1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdlib.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <ctype.h> 10 #include <stdlib.h> 11 #include <time.h> 12 #include <math.h> 13 #include <assert.h> 14 #include "mpi.h" 15 #include "phastaIO.h" 16 #include "rdtsc.h" 17 #include <FCMangle.h> 18 #include "new_interface.h" 19 #include "phIO.h" 20 #include "syncio.h" 21 #include "posixio.h" 22 #include "streamio.h" 23 #include "common_c.h" 24 25 #ifdef intel 26 #include <winsock2.h> 27 #else 28 #include <unistd.h> 29 #include <strings.h> 30 #endif 31 32 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 33 int isThisRank; 34 double *value = (double*)malloc(sizeof(double)); 35 *value = 1.0*(*ivalue); 36 rgetMinMaxAvg(value,stats,statRanks); 37 free(value); 38 } 39 40 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 41 int isThisRank; 42 43 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 44 isThisRank=workfc.numpe+1; 45 if(*value==stats[0]) 46 isThisRank=workfc.myrank; 47 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 48 49 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 50 isThisRank=workfc.numpe+1; 51 if(*value==stats[1]) 52 isThisRank=workfc.myrank; 53 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 54 55 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 56 stats[2] /= workfc.numpe; 57 58 double sqValue = (*value)*(*value), sqValueAvg = 0.; 59 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 60 sqValueAvg /= workfc.numpe; 61 62 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 63 } 64 65 void print_mesh_stats(void) { 66 int statRanks[2]; 67 double iStats[4], rStats[4]; 68 69 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 70 if(workfc.myrank==workfc.master) 71 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 72 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 73 if(workfc.myrank==workfc.master) 74 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 75 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 76 if(workfc.myrank==workfc.master) 77 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 78 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 79 if(workfc.myrank==workfc.master) { 80 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 81 printf("\n"); 82 } 83 } 84 85 void print_mpi_stats(void) { 86 int statRanks[2]; 87 double iStats[4], rStats[4]; 88 89 // NS equations 90 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 91 if(workfc.myrank==workfc.master) 92 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 93 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 94 if(workfc.myrank==workfc.master) 95 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 96 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 97 if(workfc.myrank==workfc.master) 98 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 99 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 100 if(workfc.myrank==workfc.master) 101 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 102 103 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 104 if(workfc.myrank==workfc.master) 105 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 106 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 107 if(workfc.myrank==workfc.master) 108 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 109 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 110 if(workfc.myrank==workfc.master) 111 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 112 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 113 if(workfc.myrank==workfc.master) 114 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 115 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 116 if(workfc.myrank==workfc.master) { 117 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 118 printf("\n"); 119 } 120 // Scalars 121 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 122 if(workfc.myrank==workfc.master) 123 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 124 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 125 if(workfc.myrank==workfc.master) 126 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 127 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 128 if(workfc.myrank==workfc.master) 129 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 130 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 131 if(workfc.myrank==workfc.master) 132 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 133 134 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 135 if(workfc.myrank==workfc.master) 136 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 137 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 138 if(workfc.myrank==workfc.master) 139 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 140 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 141 if(workfc.myrank==workfc.master) 142 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 143 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 144 if(workfc.myrank==workfc.master) 145 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 146 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 147 if(workfc.myrank==workfc.master) 148 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 149 150 151 } 152 153 void print_system_stats(double *tcorecp, double *tcorecpscal) { 154 int statRanks[2]; 155 double iStats[4], rStats[4]; 156 double syst_assembly, syst_solve; 157 158 // NS equations 159 syst_assembly = tcorecp[0]; 160 syst_solve = tcorecp[1]; 161 162 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 163 if(workfc.myrank==workfc.master) 164 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 165 166 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 167 if(workfc.myrank==workfc.master) 168 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 169 170 // Scalars 171 syst_assembly = tcorecpscal[0]; 172 syst_solve = tcorecpscal[1]; 173 174 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 175 if(workfc.myrank==workfc.master) 176 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 177 178 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 179 if(workfc.myrank==workfc.master) { 180 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 181 printf("\n"); 182 } 183 } 184 185 186 187 void countfieldstowriterestart() 188 { 189 int nfields = 2; //solution, time derivatives 190 191 if(outpar.ivort == 1){ 192 nfields++; //vorticity 193 } 194 195 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 196 nfields++; //instantaneous wss in bflux.f 197 } 198 199 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 200 201 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 202 if(matdat.matflg[0][0] ==-1) { 203 nfields = nfields +2; 204 } 205 206 //if Print Error Indicators = true (call write_error in itrdrv) 207 if(turbvar.ierrcalc == 1){ 208 nfields++; 209 } 210 211 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 212 if(outpar.ioybar == 1){ 213 nfields++; //ybar 214 215 //phase average fields 216 if(outpar.nphasesincycle >0) { 217 nfields = nfields + outpar.nphasesincycle; 218 } 219 220 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 221 nfields++; //wssbar 222 } 223 224 } 225 226 if(turbvari.irans < 0) { 227 nfields++; //dwal 228 } 229 230 } 231 232 outpar.nsynciofieldswriterestart = nfields; 233 234 if(workfc.myrank == 0) { 235 printf("Number of fields to write in restart files: %d\n", nfields); 236 } 237 } 238 239 240 void 241 Write_Restart( int* pid, 242 int* stepno, 243 int* nshg, 244 int* numVars, 245 double* array1, 246 double* array2 ) { 247 248 char fname[255]; 249 char rfile[60]; 250 char existingfile[30], linkfile[30]; 251 int irstou; 252 int magic_number = 362436; 253 int* mptr = &magic_number; 254 double version=0.0; 255 int isize, nitems; 256 int iarray[10]; 257 int nfiles; 258 int nfields; 259 int numparts; 260 int irank; 261 int nprocs; 262 263 // First, count the number of fields to write and store the result in 264 countfieldstowriterestart(); 265 266 // Retrieve and compute the parameters required for SyncIO 267 nfiles = outpar.nsynciofiles; 268 nfields = outpar.nsynciofieldswriterestart; 269 numparts = workfc.numpe; 270 irank = *pid; //workfc.myrank; 271 nprocs = workfc.numpe; 272 int nppp = numparts/nprocs; // always 1 for PHASTA 273 int descriptor; 274 char filename[255]; 275 bzero((void*)filename,255); 276 277 if(outpar.output_mode == -1 ) 278 streamio_setup_write(&f_descriptor, streamio_get_r()); 279 else if(outpar.output_mode == 0 ) 280 posixio_setup(&f_descriptor, 'w'); 281 else if(outpar.output_mode > 0 ) 282 syncio_setup_write(nfiles, nfields, numparts/nfiles, &f_descriptor); 283 else 284 exit(EXIT_FAILURE); 285 phio_constructName(f_descriptor,"restart",filename); 286 phstr_appendInt(filename, *stepno); 287 phstr_appendStr(filename, "."); 288 phio_openfile(filename, f_descriptor); 289 290 field_flag=0; 291 292 293 int i; 294 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 295 // Write solution field ... 296 isize = (*nshg)*(*numVars); 297 nitems = 3; 298 iarray[ 0 ] = (*nshg); 299 iarray[ 1 ] = (*numVars); 300 iarray[ 2 ] = (*stepno); 301 302 phio_writeheader(f_descriptor, "solution", (void*)iarray, &nitems, 303 &isize, "double", phasta_iotype); 304 nitems = (*nshg)*(*numVars); 305 phio_writedatablock(f_descriptor, "solution", (void*)(array1), 306 &isize, "double", phasta_iotype ); 307 } 308 field_flag++; 309 310 for ( i = 0; i < nppp; i++) { 311 // Write solution field ... 312 isize = (*nshg)*(*numVars); 313 nitems = 3; 314 iarray[ 0 ] = (*nshg); 315 iarray[ 1 ] = (*numVars); 316 iarray[ 2 ] = (*stepno); 317 phio_writeheader(f_descriptor, "time derivative of solution", 318 (void*)iarray, &nitems, &isize, "double", phasta_iotype); 319 nitems = (*nshg)*(*numVars); 320 phio_writedatablock(f_descriptor, "time derivative of solution", 321 (void*)(array2), &isize, "double", phasta_iotype ); 322 } 323 field_flag++; 324 325 if (field_flag==nfields){ 326 phio_closefile(f_descriptor); 327 if (*pid==0) { 328 printf("\n"); 329 } 330 } 331 } 332 333 void 334 Write_Error( int* pid, 335 int* stepno, 336 int* nshg, 337 int* numVars, 338 double* array1 ) { 339 char fname[255]; 340 char rfile[60]; 341 int irstou; 342 int magic_number = 362436; 343 int* mptr = &magic_number; 344 double version=0.0; 345 int isize, nitems; 346 int iarray[10]; 347 int nfiles; 348 int nfields; 349 int numparts; 350 int irank; 351 int nprocs; 352 353 nfiles = outpar.nsynciofiles; 354 nfields = outpar.nsynciofieldswriterestart; 355 numparts = workfc.numpe; 356 irank = *pid; //workfc.myrank; 357 nprocs = workfc.numpe; 358 359 // Calculate number of parts each proc deal with and where it start and end ... 360 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 361 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 362 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 363 364 field_flag++; 365 366 int i; 367 for ( i = 0; i < nppp; i++ ) { 368 369 if(*pid==0) { 370 printf("\n"); 371 printf("The %d/%d th field to be written is 'errors'\n",field_flag,nfields); 372 } 373 374 isize = (*nshg)*(*numVars); 375 nitems = 3; 376 iarray[ 0 ] = (*nshg); 377 iarray[ 1 ] = (*numVars); 378 iarray[ 2 ] = (*stepno); 379 380 phio_writeheader(f_descriptor, "errors", (void*)iarray, &nitems, 381 &isize, "double", phasta_iotype); 382 383 phio_writedatablock(f_descriptor, "errors", (void*)array1, &isize, 384 "double", phasta_iotype ); 385 } 386 if (field_flag==nfields){ 387 phio_closefile(f_descriptor); 388 if (*pid==0) { 389 printf("Last field %d 'errors' finished! \n",nfields); 390 printf("\n"); 391 } 392 } 393 } 394 395 void 396 Write_Displ( int* pid, 397 int* stepno, 398 int* nshg, 399 int* numVars, 400 double* array1 ) { 401 fprintf(stderr, "This function is dead...exiting\n"); 402 exit(1); 403 } 404 405 void 406 Write_Field( int *pid, 407 char* filemode, 408 char* fieldtag, 409 int* tagsize, 410 void* array, 411 char* arraytype, 412 int* nshg, 413 int* numvars, 414 int* stepno) { 415 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 416 strncpy(fieldlabel, fieldtag, *tagsize); 417 fieldlabel[*tagsize] = '\0'; 418 419 int irstou; 420 int magic_number = 362436; 421 int* mptr = &magic_number; 422 double version=0.0; 423 int isize, nitems; 424 int iarray[10]; 425 426 char fmode[10]; 427 if(!strncmp(filemode,"w",1)) 428 strcpy(fmode,"write"); 429 else // default is append 430 strcpy(fmode,"append"); 431 432 char datatype[10]; 433 if(!strncmp(arraytype,"i",1)) 434 strcpy(datatype,"int"); 435 else // default is double 436 strcpy(datatype,"double"); 437 438 int nfiles; 439 int nfields; 440 int numparts; 441 int irank; 442 int nprocs; 443 444 nfiles = outpar.nsynciofiles; 445 nfields = outpar.nsynciofieldswriterestart; 446 numparts = workfc.numpe; 447 irank = *pid; //workfc.myrank; 448 nprocs = workfc.numpe; 449 450 // Calculate number of parts each proc deal with and where it start and end ... 451 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 452 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 453 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 454 455 strncpy(fieldlabel, fieldtag, *tagsize); 456 457 field_flag++; 458 if(*pid==0) { 459 printf("\n"); 460 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 461 } 462 463 int i; 464 for ( i = 0; i < nppp; i++ ) { 465 // Write solution field ... 466 isize = (*nshg)*(*numvars); 467 nitems = 3; 468 iarray[ 0 ] = (*nshg); 469 iarray[ 1 ] = (*numvars); 470 iarray[ 2 ] = (*stepno); 471 472 phio_writeheader(f_descriptor, fieldlabel, (void*)iarray, &nitems, 473 &isize, datatype, phasta_iotype); 474 nitems = (*nshg)*(*numvars); 475 phio_writedatablock(f_descriptor, fieldlabel, array, &isize, 476 datatype, phasta_iotype ); 477 } 478 if (field_flag==nfields){ 479 phio_closefile(f_descriptor); 480 if (*pid==0) { 481 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 482 printf("\n"); 483 } 484 } 485 free(fieldlabel); 486 } 487 488 void 489 Write_PhAvg2( int* pid, 490 char* filemode, 491 char* fieldtag, 492 int* tagsize, 493 int* iphase, 494 int* nphasesincycle, 495 void* array, 496 char* arraytype, 497 int* nshg, 498 int* numvars, 499 int* stepno) { 500 int addtagsize=0; // phase number is added to the name of the field 501 if(*iphase<10) 502 addtagsize=1; 503 else if(*iphase<100) 504 addtagsize=2; 505 else if(*iphase<1000) 506 addtagsize=3; 507 508 int tagsize2; 509 tagsize2=*tagsize+addtagsize; 510 511 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 512 strncpy(fieldlabel, fieldtag, *tagsize); 513 fieldlabel[tagsize2] = '\0'; 514 515 char straddtagsize[10]; 516 sprintf(straddtagsize,"%d",*iphase); 517 518 if(*iphase<10) { 519 fieldlabel[tagsize2-1]=straddtagsize[0]; 520 } 521 else if(*iphase<100) { 522 fieldlabel[tagsize2-2]=straddtagsize[0]; 523 fieldlabel[tagsize2-1]=straddtagsize[1]; 524 } 525 else if(*iphase<1000) { 526 fieldlabel[tagsize2-3]=straddtagsize[0]; 527 fieldlabel[tagsize2-2]=straddtagsize[1]; 528 fieldlabel[tagsize2-1]=straddtagsize[2]; 529 } 530 531 int irstou; 532 int magic_number = 362436; 533 int* mptr = &magic_number; 534 double version=0.0; 535 int isize, nitems; 536 int iarray[10]; 537 538 char fmode[10]; 539 if(!strncmp(filemode,"w",1)) 540 strcpy(fmode,"write"); 541 else // default is append 542 strcpy(fmode,"append"); 543 544 char datatype[10]; 545 if(!strncmp(arraytype,"i",1)) 546 strcpy(datatype,"int"); 547 else // default is double 548 strcpy(datatype,"double"); 549 550 int nfiles; 551 int nfields; 552 int numparts; 553 int irank; 554 int nprocs; 555 556 nfiles = outpar.nsynciofiles; 557 nfields = outpar.nsynciofieldswriterestart; 558 numparts = workfc.numpe; 559 irank = *pid; //workfc.myrank; 560 nprocs = workfc.numpe; 561 562 // Calculate number of parts each proc deal with and where it start and end ... 563 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 564 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 565 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 566 567 field_flag++; 568 if(*pid==0) { 569 printf("\n"); 570 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 571 } 572 573 int i; 574 for ( i = 0; i < nppp; i++ ) { 575 // Write solution field ... 576 isize = (*nshg)*(*numvars); 577 nitems = 3; 578 iarray[ 0 ] = (*nshg); 579 iarray[ 1 ] = (*numvars); 580 iarray[ 2 ] = (*stepno); 581 phio_writeheader(f_descriptor, fieldlabel, (void*)iarray, &nitems, 582 &isize, "double", phasta_iotype); 583 nitems = (*nshg)*(*numvars); 584 phio_writedatablock(f_descriptor, fieldlabel, array, &isize, 585 "double", phasta_iotype ); 586 } 587 if (field_flag==nfields){ 588 phio_closefile(f_descriptor); 589 if (*pid==0) { 590 printf("\n"); 591 } 592 } 593 free(fieldlabel); 594 } 595