1 /* This file provides interface functions for 'partial ' random 2 access into the PHASTA input files 3 4 Anil Karanam March 2001 */ 5 6 #include <stdlib.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <ctype.h> 10 #include <stdlib.h> 11 #include <time.h> 12 #include <math.h> 13 #include <assert.h> 14 #include "mpi.h" 15 #include "phastaIO.h" 16 #include "rdtsc.h" 17 #include <FCMangle.h> 18 #include "new_interface.h" 19 #include "phIO.h" 20 #include "syncio.h" 21 #include "posixio.h" 22 #include "common_c.h" 23 24 #ifdef intel 25 #include <winsock2.h> 26 #else 27 #include <unistd.h> 28 #include <strings.h> 29 #endif 30 31 void igetMinMaxAvg(int *ivalue, double *stats, int *statRanks) { 32 int isThisRank; 33 double *value = (double*)malloc(sizeof(double)); 34 *value = 1.0*(*ivalue); 35 rgetMinMaxAvg(value,stats,statRanks); 36 free(value); 37 } 38 39 void rgetMinMaxAvg(double *value, double *stats, int *statRanks) { 40 int isThisRank; 41 42 MPI_Allreduce(value,&stats[0],1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); 43 isThisRank=workfc.numpe+1; 44 if(*value==stats[0]) 45 isThisRank=workfc.myrank; 46 MPI_Allreduce(&isThisRank,&statRanks[0],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 47 48 MPI_Allreduce(value,&stats[1],1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD); 49 isThisRank=workfc.numpe+1; 50 if(*value==stats[1]) 51 isThisRank=workfc.myrank; 52 MPI_Allreduce(&isThisRank,&statRanks[1],1,MPI_INT,MPI_MIN,MPI_COMM_WORLD); 53 54 MPI_Allreduce(value,&stats[2],1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 55 stats[2] /= workfc.numpe; 56 57 double sqValue = (*value)*(*value), sqValueAvg = 0.; 58 MPI_Allreduce(&sqValue,&sqValueAvg,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); 59 sqValueAvg /= workfc.numpe; 60 61 stats[3] = sqrt(sqValueAvg-stats[2]*stats[2]); 62 } 63 64 void print_mesh_stats(void) { 65 int statRanks[2]; 66 double iStats[4], rStats[4]; 67 68 igetMinMaxAvg(&conpar.nshg,iStats,statRanks); 69 if(workfc.myrank==workfc.master) 70 printf("nshg : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 71 igetMinMaxAvg(&conpar.numel,iStats,statRanks); 72 if(workfc.myrank==workfc.master) 73 printf("numel : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 74 igetMinMaxAvg(&conpar.numelb,iStats,statRanks); 75 if(workfc.myrank==workfc.master) 76 printf("numelb : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 77 igetMinMaxAvg(&conpar.nnz_tot,iStats,statRanks); 78 if(workfc.myrank==workfc.master) { 79 printf("nnz_tot : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 80 printf("\n"); 81 } 82 } 83 84 void print_mpi_stats(void) { 85 int statRanks[2]; 86 double iStats[4], rStats[4]; 87 88 // NS equations 89 igetMinMaxAvg(&mpistats.iISend,iStats,statRanks); 90 if(workfc.myrank==workfc.master) 91 printf("iISend : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 92 igetMinMaxAvg(&mpistats.iIRecv,iStats,statRanks); 93 if(workfc.myrank==workfc.master) 94 printf("iIRecv : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 95 igetMinMaxAvg(&mpistats.iWaitAll,iStats,statRanks); 96 if(workfc.myrank==workfc.master) 97 printf("iWtAll : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 98 igetMinMaxAvg(&mpistats.iAllR,iStats,statRanks); 99 if(workfc.myrank==workfc.master) 100 printf("iAllR : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 101 102 rgetMinMaxAvg(&mpistats.rISend,rStats,statRanks); 103 if(workfc.myrank==workfc.master) 104 printf("rISend : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 105 rgetMinMaxAvg(&mpistats.rIRecv,rStats,statRanks); 106 if(workfc.myrank==workfc.master) 107 printf("rIRecv : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 108 rgetMinMaxAvg(&mpistats.rWaitAll,rStats,statRanks); 109 if(workfc.myrank==workfc.master) 110 printf("rWtAll : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 111 rgetMinMaxAvg(&mpistats.rCommu,rStats,statRanks); 112 if(workfc.myrank==workfc.master) 113 printf("rCommu : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 114 rgetMinMaxAvg(&mpistats.rAllR,rStats,statRanks); 115 if(workfc.myrank==workfc.master) { 116 printf("rAllR : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 117 printf("\n"); 118 } 119 // Scalars 120 igetMinMaxAvg(&mpistats.iISendScal,iStats,statRanks); 121 if(workfc.myrank==workfc.master) 122 printf("iISendScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 123 igetMinMaxAvg(&mpistats.iIRecvScal,iStats,statRanks); 124 if(workfc.myrank==workfc.master) 125 printf("iIRecvScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 126 igetMinMaxAvg(&mpistats.iWaitAllScal,iStats,statRanks); 127 if(workfc.myrank==workfc.master) 128 printf("iWtAllScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 129 igetMinMaxAvg(&mpistats.iAllRScal,iStats,statRanks); 130 if(workfc.myrank==workfc.master) 131 printf("iAllRScal : min [%d,%d], max[%d,%d] and avg[.,%d] (rms=%d)\n",statRanks[0],(int)iStats[0],statRanks[1],(int)iStats[1],(int)iStats[2],(int)iStats[3]); 132 133 rgetMinMaxAvg(&mpistats.rISendScal,rStats,statRanks); 134 if(workfc.myrank==workfc.master) 135 printf("rISendScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 136 rgetMinMaxAvg(&mpistats.rIRecvScal,rStats,statRanks); 137 if(workfc.myrank==workfc.master) 138 printf("rIRecvScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 139 rgetMinMaxAvg(&mpistats.rWaitAllScal,rStats,statRanks); 140 if(workfc.myrank==workfc.master) 141 printf("rWtAllScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 142 rgetMinMaxAvg(&mpistats.rCommuScal,rStats,statRanks); 143 if(workfc.myrank==workfc.master) 144 printf("rCommuScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 145 rgetMinMaxAvg(&mpistats.rAllRScal,rStats,statRanks); 146 if(workfc.myrank==workfc.master) 147 printf("rAllRScal : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 148 149 150 } 151 152 void print_system_stats(double *tcorecp, double *tcorecpscal) { 153 int statRanks[2]; 154 double iStats[4], rStats[4]; 155 double syst_assembly, syst_solve; 156 157 // NS equations 158 syst_assembly = tcorecp[0]; 159 syst_solve = tcorecp[1]; 160 161 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 162 if(workfc.myrank==workfc.master) 163 printf("Elm. form. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 164 165 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 166 if(workfc.myrank==workfc.master) 167 printf("Lin. alg. sol : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 168 169 // Scalars 170 syst_assembly = tcorecpscal[0]; 171 syst_solve = tcorecpscal[1]; 172 173 rgetMinMaxAvg(&syst_assembly,rStats,statRanks); 174 if(workfc.myrank==workfc.master) 175 printf("Elm. form. Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 176 177 rgetMinMaxAvg(&syst_solve,rStats,statRanks); 178 if(workfc.myrank==workfc.master) { 179 printf("Lin. alg. sol Scal. : min [%d,%2.5f], max[%d,%2.5f] and avg[.,%2.5f] (rms=%2.5f)\n",statRanks[0],rStats[0],statRanks[1],rStats[1],rStats[2],rStats[3]); 180 printf("\n"); 181 } 182 } 183 184 185 186 void countfieldstowriterestart() 187 { 188 int nfields = 2; //solution, time derivatives 189 190 if(outpar.ivort == 1){ 191 nfields++; //vorticity 192 } 193 194 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 195 nfields++; //instantaneous wss in bflux.f 196 } 197 198 if(timdat.istep == inpdat.nstep[timdat.itseq-1]){ //Last time step of the computation 199 200 //projection vectors and pressure projection vectors (call saveLesRestart in itrdrv) 201 if(matdat.matflg[0][0] ==-1) { 202 nfields = nfields +2; 203 } 204 205 //if Print Error Indicators = true (call write_error in itrdrv) 206 if(turbvar.ierrcalc == 1){ 207 nfields++; 208 } 209 210 //if Print ybar = True (call write_field(myrank,'a','ybar',4,... in itrdrv) 211 if(outpar.ioybar == 1){ 212 nfields++; //ybar 213 214 //phase average fields 215 if(outpar.nphasesincycle >0) { 216 nfields = nfields + outpar.nphasesincycle; 217 } 218 219 if(abs(turbvar.itwmod) != 1 && outpar.iowflux == 1) { 220 nfields++; //wssbar 221 } 222 223 } 224 225 if(turbvari.irans < 0) { 226 nfields++; //dwal 227 } 228 229 } 230 231 outpar.nsynciofieldswriterestart = nfields; 232 233 if(workfc.myrank == 0) { 234 printf("Number of fields to write in restart files: %d\n", nfields); 235 } 236 } 237 238 239 void 240 Write_Restart( int* pid, 241 int* stepno, 242 int* nshg, 243 int* numVars, 244 double* array1, 245 double* array2 ) { 246 247 char fname[255]; 248 char rfile[60]; 249 char existingfile[30], linkfile[30]; 250 int irstou; 251 int magic_number = 362436; 252 int* mptr = &magic_number; 253 double version=0.0; 254 int isize, nitems; 255 int iarray[10]; 256 int nfiles; 257 int nfields; 258 int numparts; 259 int irank; 260 int nprocs; 261 int nppf; 262 263 // First, count the number of fields to write and store the result in 264 countfieldstowriterestart(); 265 266 // Retrieve and compute the parameters required for SyncIO 267 nfiles = outpar.nsynciofiles; 268 nfields = outpar.nsynciofieldswriterestart; 269 numparts = workfc.numpe; 270 irank = *pid; //workfc.myrank; 271 nprocs = workfc.numpe; 272 int nppp = numparts/nprocs; // always 1 for PHASTA 273 int descriptor; 274 char filename[255]; 275 bzero((void*)filename,255); 276 277 if(nfiles == 0 ){ 278 sprintf(filename,"restart.%d.", *stepno); 279 posixio_setup(&f_descriptor, 'w'); 280 } else { 281 nppf=numparts/nfiles; 282 sprintf(filename,"restart-dat.%d.", *stepno); 283 syncio_setup_write(nfiles, nfields, nppf, &f_descriptor); 284 } 285 phio_openfile(filename, f_descriptor); 286 287 field_flag=0; 288 289 290 int i; 291 for ( i = 0; i < nppp; i++) { //This loop is useful only if several parts per processor 292 // Write solution field ... 293 isize = (*nshg)*(*numVars); 294 nitems = 3; 295 iarray[ 0 ] = (*nshg); 296 iarray[ 1 ] = (*numVars); 297 iarray[ 2 ] = (*stepno); 298 299 phio_writeheader(f_descriptor, "solution", (void*)iarray, &nitems, 300 &isize, "double", phasta_iotype); 301 nitems = (*nshg)*(*numVars); 302 phio_writedatablock(f_descriptor, "solution", (void*)(array1), 303 &isize, "double", phasta_iotype ); 304 } 305 field_flag++; 306 307 for ( i = 0; i < nppp; i++) { 308 // Write solution field ... 309 isize = (*nshg)*(*numVars); 310 nitems = 3; 311 iarray[ 0 ] = (*nshg); 312 iarray[ 1 ] = (*numVars); 313 iarray[ 2 ] = (*stepno); 314 phio_writeheader(f_descriptor, "time derivative of solution", 315 (void*)iarray, &nitems, &isize, "double", phasta_iotype); 316 nitems = (*nshg)*(*numVars); 317 phio_writedatablock(f_descriptor, "time derivative of solution", 318 (void*)(array2), &isize, "double", phasta_iotype ); 319 } 320 field_flag++; 321 322 if (field_flag==nfields){ 323 phio_closefile(f_descriptor); 324 if (*pid==0) { 325 printf("\n"); 326 } 327 } 328 } 329 330 void 331 Write_Error( int* pid, 332 int* stepno, 333 int* nshg, 334 int* numVars, 335 double* array1 ) { 336 char fname[255]; 337 char rfile[60]; 338 int irstou; 339 int magic_number = 362436; 340 int* mptr = &magic_number; 341 double version=0.0; 342 int isize, nitems; 343 int iarray[10]; 344 int nfiles; 345 int nfields; 346 int numparts; 347 int irank; 348 int nprocs; 349 350 nfiles = outpar.nsynciofiles; 351 nfields = outpar.nsynciofieldswriterestart; 352 numparts = workfc.numpe; 353 irank = *pid; //workfc.myrank; 354 nprocs = workfc.numpe; 355 356 // Calculate number of parts each proc deal with and where it start and end ... 357 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 358 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 359 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 360 361 field_flag++; 362 363 int i; 364 for ( i = 0; i < nppp; i++ ) { 365 366 if(*pid==0) { 367 printf("\n"); 368 printf("The %d/%d th field to be written is 'errors'\n",field_flag,nfields); 369 } 370 371 isize = (*nshg)*(*numVars); 372 nitems = 3; 373 iarray[ 0 ] = (*nshg); 374 iarray[ 1 ] = (*numVars); 375 iarray[ 2 ] = (*stepno); 376 377 phio_writeheader(f_descriptor, "errors", (void*)iarray, &nitems, 378 &isize, "double", phasta_iotype); 379 380 phio_writedatablock(f_descriptor, "errors", (void*)array1, &isize, 381 "double", phasta_iotype ); 382 } 383 if (field_flag==nfields){ 384 phio_closefile(f_descriptor); 385 if (*pid==0) { 386 printf("Last field %d 'errors' finished! \n",nfields); 387 printf("\n"); 388 } 389 } 390 } 391 392 void 393 Write_Displ( int* pid, 394 int* stepno, 395 int* nshg, 396 int* numVars, 397 double* array1 ) { 398 fprintf(stderr, "This function is dead...exiting\n"); 399 exit(1); 400 } 401 402 void 403 Write_Field( int *pid, 404 char* filemode, 405 char* fieldtag, 406 int* tagsize, 407 void* array, 408 char* arraytype, 409 int* nshg, 410 int* numvars, 411 int* stepno) { 412 char *fieldlabel = (char *)malloc((*tagsize+1)*sizeof(char)); 413 strncpy(fieldlabel, fieldtag, *tagsize); 414 fieldlabel[*tagsize] = '\0'; 415 416 int irstou; 417 int magic_number = 362436; 418 int* mptr = &magic_number; 419 double version=0.0; 420 int isize, nitems; 421 int iarray[10]; 422 423 char fmode[10]; 424 if(!strncmp(filemode,"w",1)) 425 strcpy(fmode,"write"); 426 else // default is append 427 strcpy(fmode,"append"); 428 429 char datatype[10]; 430 if(!strncmp(arraytype,"i",1)) 431 strcpy(datatype,"int"); 432 else // default is double 433 strcpy(datatype,"double"); 434 435 int nfiles; 436 int nfields; 437 int numparts; 438 int irank; 439 int nprocs; 440 441 nfiles = outpar.nsynciofiles; 442 nfields = outpar.nsynciofieldswriterestart; 443 numparts = workfc.numpe; 444 irank = *pid; //workfc.myrank; 445 nprocs = workfc.numpe; 446 447 // Calculate number of parts each proc deal with and where it start and end ... 448 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 449 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 450 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 451 452 strncpy(fieldlabel, fieldtag, *tagsize); 453 454 field_flag++; 455 if(*pid==0) { 456 printf("\n"); 457 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 458 } 459 460 int i; 461 for ( i = 0; i < nppp; i++ ) { 462 // Write solution field ... 463 isize = (*nshg)*(*numvars); 464 nitems = 3; 465 iarray[ 0 ] = (*nshg); 466 iarray[ 1 ] = (*numvars); 467 iarray[ 2 ] = (*stepno); 468 469 phio_writeheader(f_descriptor, fieldlabel, (void*)iarray, &nitems, 470 &isize, datatype, phasta_iotype); 471 nitems = (*nshg)*(*numvars); 472 phio_writedatablock(f_descriptor, fieldlabel, array, &isize, 473 datatype, phasta_iotype ); 474 } 475 if (field_flag==nfields){ 476 phio_closefile(f_descriptor); 477 if (*pid==0) { 478 printf("Last field %d '%s' finished! \n",nfields, fieldtag); 479 printf("\n"); 480 } 481 } 482 free(fieldlabel); 483 } 484 485 void 486 Write_PhAvg2( int* pid, 487 char* filemode, 488 char* fieldtag, 489 int* tagsize, 490 int* iphase, 491 int* nphasesincycle, 492 void* array, 493 char* arraytype, 494 int* nshg, 495 int* numvars, 496 int* stepno) { 497 int addtagsize=0; // phase number is added to the name of the field 498 if(*iphase<10) 499 addtagsize=1; 500 else if(*iphase<100) 501 addtagsize=2; 502 else if(*iphase<1000) 503 addtagsize=3; 504 505 int tagsize2; 506 tagsize2=*tagsize+addtagsize; 507 508 char *fieldlabel = (char *)malloc((tagsize2+1)*sizeof(char)); 509 strncpy(fieldlabel, fieldtag, *tagsize); 510 fieldlabel[tagsize2] = '\0'; 511 512 char straddtagsize[10]; 513 sprintf(straddtagsize,"%d",*iphase); 514 515 if(*iphase<10) { 516 fieldlabel[tagsize2-1]=straddtagsize[0]; 517 } 518 else if(*iphase<100) { 519 fieldlabel[tagsize2-2]=straddtagsize[0]; 520 fieldlabel[tagsize2-1]=straddtagsize[1]; 521 } 522 else if(*iphase<1000) { 523 fieldlabel[tagsize2-3]=straddtagsize[0]; 524 fieldlabel[tagsize2-2]=straddtagsize[1]; 525 fieldlabel[tagsize2-1]=straddtagsize[2]; 526 } 527 528 int irstou; 529 int magic_number = 362436; 530 int* mptr = &magic_number; 531 double version=0.0; 532 int isize, nitems; 533 int iarray[10]; 534 535 char fmode[10]; 536 if(!strncmp(filemode,"w",1)) 537 strcpy(fmode,"write"); 538 else // default is append 539 strcpy(fmode,"append"); 540 541 char datatype[10]; 542 if(!strncmp(arraytype,"i",1)) 543 strcpy(datatype,"int"); 544 else // default is double 545 strcpy(datatype,"double"); 546 547 int nfiles; 548 int nfields; 549 int numparts; 550 int irank; 551 int nprocs; 552 553 nfiles = outpar.nsynciofiles; 554 nfields = outpar.nsynciofieldswriterestart; 555 numparts = workfc.numpe; 556 irank = *pid; //workfc.myrank; 557 nprocs = workfc.numpe; 558 559 // Calculate number of parts each proc deal with and where it start and end ... 560 int nppp = numparts/nprocs;// nppp : Number of parts per proc ... 561 int startpart = irank * nppp +1;// Part id from which I (myrank) start ... 562 int endpart = startpart + nppp - 1;// Part id to which I (myrank) end ... 563 564 field_flag++; 565 if(*pid==0) { 566 printf("\n"); 567 printf("The %d/%d th field to be written is '%s'\n",field_flag,nfields,fieldlabel); 568 } 569 570 int i; 571 for ( i = 0; i < nppp; i++ ) { 572 // Write solution field ... 573 isize = (*nshg)*(*numvars); 574 nitems = 3; 575 iarray[ 0 ] = (*nshg); 576 iarray[ 1 ] = (*numvars); 577 iarray[ 2 ] = (*stepno); 578 phio_writeheader(f_descriptor, fieldlabel, (void*)iarray, &nitems, 579 &isize, "double", phasta_iotype); 580 nitems = (*nshg)*(*numvars); 581 phio_writedatablock(f_descriptor, fieldlabel, array, &isize, 582 "double", phasta_iotype ); 583 } 584 if (field_flag==nfields){ 585 phio_closefile(f_descriptor); 586 if (*pid==0) { 587 printf("\n"); 588 } 589 } 590 free(fieldlabel); 591 } 592