1 #include <stdio.h> 2 #include <iostream> 3 #include <string.h> 4 #include <stdlib.h> 5 //#define OMPI_SKIP_MPICXX 1 //Added in the CMakeList.txt file 6 #include <mpi.h> 7 #include <math.h> 8 #include "phastaIO.h" 9 #include <sys/stat.h> 10 #include <sys/types.h> 11 12 enum { 13 DIR_FANOUT = 2048 14 }; 15 16 inline int 17 cscompare( const char teststring[], 18 const char targetstring[] ) 19 { 20 char* s1 = const_cast<char*>(teststring); 21 char* s2 = const_cast<char*>(targetstring); 22 23 while( *s1 == ' ') s1++; 24 while( *s2 == ' ') s2++; 25 while( ( *s1 ) 26 && ( *s2 ) 27 && ( *s2 != '?') 28 && ( tolower( *s1 )==tolower( *s2 ) ) ) { 29 s1++; 30 s2++; 31 while( *s1 == ' ') s1++; 32 while( *s2 == ' ') s2++; 33 } 34 if ( !( *s1 ) || ( *s1 == '?') ) return 1; 35 else return 0; 36 } 37 38 inline int 39 computenitems(const int localpartid, const int fieldid, const int myrank, const char *fieldName, int ***para, const int intHeader, const int numVariables) { 40 // This routine computes the number of items in the data block based on 41 // - the name of the fields 42 // - the integers read from the header 43 44 int nItems = -1; 45 46 if (cscompare("nbc values",fieldName)) 47 nItems = para[localpartid][fieldid][0] * (numVariables+1); 48 else if (cscompare("nbc codes",fieldName)) 49 nItems = para[localpartid][fieldid][0] * 2; 50 else if ( intHeader==1) 51 nItems = para[localpartid][fieldid][0]; 52 else 53 nItems = para[localpartid][fieldid][0] * para[localpartid][fieldid][1]; 54 return nItems; 55 } 56 57 58 int main(int argc, char *argv[]) { 59 60 MPI_Init(&argc,&argv); 61 62 int myrank, N_procs; 63 MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 64 MPI_Comm_size(MPI_COMM_WORLD, &N_procs); 65 66 FILE * pFile,* AccessoryFileHandle; 67 char target[1024], pool[256], tempString[128],numpe[8],numstart[8]; 68 char * temp, * token; 69 int fieldCompareMark; 70 bool readField; 71 int i, j, k, N_geombc_double, N_geombc_integer, N_restart_double; 72 int N_restart_integer, N_steps, N_parts, N_files; 73 74 pFile = fopen("./IO.O2N.input","r"); 75 if (pFile == NULL) 76 printf("Error openning\n"); 77 78 fgets( target, 1024, pFile ); 79 token = strtok ( target, ";" );strcpy(pool,token); 80 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 81 N_geombc_double = atoi(temp); 82 83 fgets( target, 1024, pFile ); 84 token = strtok ( target, ";" );strcpy(pool,token); 85 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 86 N_geombc_integer = atoi(temp); 87 88 fgets( target, 1024, pFile ); 89 token = strtok ( target, ";" );strcpy(pool,token); 90 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 91 N_restart_double = atoi(temp); 92 93 fgets( target, 1024, pFile ); 94 token = strtok ( target, ";" );strcpy(pool,token); 95 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 96 N_restart_integer = atoi(temp); 97 98 fgets( target, 1024, pFile ); 99 token = strtok ( target, ";" );strcpy(pool,token); 100 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 101 strncpy(numstart,temp,1); 102 N_steps = atoi(temp); 103 104 fgets( target, 1024, pFile ); 105 token = strtok ( target, ";" );strcpy(pool,token); 106 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 107 strncpy(numpe,temp,1); 108 N_parts = atoi(temp); 109 110 if(myrank==0){ 111 printf("numpe is %d and start is %d\n",N_parts,N_steps); 112 } 113 114 fgets( target, 1024, pFile ); 115 token = strtok ( target, ";" );strcpy(pool,token); 116 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" ); 117 N_files = atoi(temp); 118 119 double ***Dfield; int ***Ifield; 120 int ***paraD, ***paraI, *expectD, *expectI; 121 char **fieldNameD, **fileTypeD, **dataTypeD, **headerTypeD; 122 char **fieldNameI, **fileTypeI, **dataTypeI, **headerTypeI; 123 124 int* WriteLockD = new int[N_geombc_double]; 125 int* WriteLockI = new int[N_geombc_integer]; 126 127 int nppp = N_parts/N_procs; 128 int startpart = myrank * nppp +1; 129 int endpart = startpart + nppp - 1; 130 char gfname[64], numTemp[128]; 131 int iarray[10], igeom, isize,TempFileHandle; 132 133 //////////////////////////////////////// 134 // Test if the user has given the right parameters related to the number of parts, procs and SyncIO files 135 //////////////////////////////////////// 136 137 if (N_parts%N_files!=0) 138 { 139 printf("Input error: number of parts should be a multiple of number of files!\n"); 140 printf("Please modify the IO.O2N.input file!\n"); 141 return 0; 142 } 143 if (N_procs%N_files!=0) 144 { 145 printf("Input error: number of procs should be a multiple of number of files!\n"); 146 printf("Please modify the IO.O2N.input file!\n"); 147 return 0; 148 } 149 if (N_parts%N_procs!=0) 150 { 151 printf("Input error: number of parts should be a multiple of number of procs!\n"); 152 printf("Please modify the IO.O2N.input file!\n"); 153 return 0; 154 } 155 156 ///////////////////////////////////// 157 // Create numpe.in and numstart.dat 158 //////////////////////////////////////// 159 160 if (myrank==0) 161 { 162 163 //MR CHANGE 164 bzero((void*)gfname,64); 165 sprintf(gfname,"./%d-procs_case-SyncIO-%d",N_parts,N_files); 166 if(0<mkdir(gfname,0777)) { printf("ERROR - Could not create procs_case-SyncIO directory\n"); return 1; } 167 //MR CHANGE END 168 169 bzero((void*)gfname,64); 170 sprintf(gfname,"./%d-procs_case-SyncIO-%d/numstart.dat",N_parts,N_files); 171 AccessoryFileHandle=fopen(gfname,"w"); 172 fprintf(AccessoryFileHandle,"%d",N_steps); 173 fclose(AccessoryFileHandle); 174 175 bzero((void*)gfname,64); 176 sprintf(gfname,"./%d-procs_case-SyncIO-%d/numpe.in",N_parts,N_files); 177 AccessoryFileHandle=fopen(gfname,"w"); 178 fprintf(AccessoryFileHandle,"%d",N_parts); 179 fclose(AccessoryFileHandle); 180 } 181 182 /////////////////////////////////////// 183 Dfield = new double**[nppp]; 184 Ifield = new int**[nppp]; 185 186 paraD = new int**[nppp]; 187 paraI = new int**[nppp]; 188 ///////////////////////////////////// 189 190 int* interiorMark = new int[nppp]; 191 int* boundaryMark = new int[nppp]; 192 int* codesMark = new int[nppp]; 193 int* valuesMark = new int[nppp]; 194 int* numVariables = new int[nppp]; 195 // int numBoundaryFields[nppp], numInteriorFields[nppp]; 196 197 if(myrank==0){ 198 printf("Starting to read some headers in the restart.##.## and geombc.dat.## files\n"); 199 } 200 201 // The number of variables does not vary from one part to another. Do not read this info from every part. 202 // The ideal would be to ask only rank 0 to read only part 0 and broadcast the information to the other ranks. It saves a lot of time for very big meshes. 203 // Right now, every rank will read the number of variables from the solution field of one part only, which is already better than reading this info from every part. 204 int ithree = 3; 205 j = 0; 206 207 // Test if file exist in the procs_case directory 208 bzero((void*)gfname,64); 209 sprintf(gfname,"./%d-procs_case/restart.%d.%d", N_parts, N_steps, startpart+j); 210 int subdir = -1; 211 bool existsubdir = false; 212 FILE *pfiletest; 213 pfiletest = fopen(gfname, "r"); 214 if (pfiletest == NULL ) { 215 // Test if file exist in the procs_case/subdir directory 216 subdir = (startpart+j-1) / DIR_FANOUT; 217 bzero((void*)gfname,64); 218 sprintf(gfname,"./%d-procs_case/%d/restart.%d.%d",N_parts, subdir, N_steps, startpart+j); 219 pfiletest = fopen(gfname, "r"); 220 if (pfiletest == NULL ) { 221 printf("[%d] File %s does not exit - abort\n",myrank, gfname); 222 abort(); 223 } 224 else{ 225 existsubdir = true; 226 fclose(pfiletest); 227 } 228 } 229 else { 230 fclose(pfiletest); 231 } 232 233 // Debug 234 //printf("Rank: %d - subdir: %d - path: %s\n",myrank, subdir, gfname); 235 236 openfile(gfname,"read",&TempFileHandle); 237 readheader( &TempFileHandle, 238 "solution", 239 (void*)iarray, 240 &ithree, 241 "double", 242 "binary" ); 243 closefile(&TempFileHandle, "read"); 244 for ( j = 0; j < nppp; j++ ) 245 numVariables[j] = iarray[1]; //iarray[i] contains the number of variables from the header of the solution field 246 247 MPI_Barrier(MPI_COMM_WORLD); //added by MR 248 249 250 ///////////////////////// 251 for ( i = 0; i < nppp; i++ ) 252 { 253 Dfield[i] = new double*[N_geombc_double]; // Space for the datablock of double format 254 paraD[i] = new int*[N_geombc_double]; // Integers in the header of each field of double format 255 Ifield[i] = new int*[N_geombc_integer]; // Space for the datablock of integer format 256 paraI[i] = new int*[N_geombc_integer]; // Integers in the header of each field of integer format 257 258 } 259 260 expectD = new int[N_geombc_double]; // Expected number of integers in the header for each field of double format 261 expectI = new int[N_geombc_integer]; // Expected number of integers in the header for each field of integer format 262 263 fieldNameD = new char*[N_geombc_double]; // Name of the field in double format 264 fileTypeD = new char*[N_geombc_double]; // geombc or restart (useless if associated with geombc file but read) 265 dataTypeD = new char*[N_geombc_double]; // Integer or double (useless if associated with double data but read) 266 headerTypeD = new char*[N_geombc_double]; // block (means with data block) or header (just a header with no block) 267 268 fieldNameI = new char*[N_geombc_integer]; 269 fileTypeI = new char*[N_geombc_integer]; 270 dataTypeI = new char*[N_geombc_integer]; 271 headerTypeI = new char*[N_geombc_integer]; 272 273 ///////////////////////// 274 275 for ( i = 0; i < N_geombc_double; i++ ) 276 { 277 WriteLockD[i]=0; 278 279 fieldNameD[i] = new char[128]; 280 fileTypeD[i] = new char[128]; 281 dataTypeD[i] = new char[128]; 282 headerTypeD[i] = new char[128]; 283 } 284 285 for ( i = 0; i < N_geombc_integer; i++ ) 286 { 287 WriteLockI[i]=0; //This take value 1 if the field requested in IO.input is not found (typically for tpblocks) 288 289 fieldNameI[i] = new char[128]; 290 fileTypeI[i] = new char[128]; 291 dataTypeI[i] = new char[128]; 292 headerTypeI[i] = new char[128]; 293 } 294 295 //////////////////////////////////////////////////////////////// 296 // Reading IO.input 297 // temporary fix: in the new version the double and integer 298 // can mix and match to avoid the order confusion 299 //////////////////////////////////////////////////////////////// 300 301 char S1[128],S2[128],S3[128],S4[128],S5[128]; 302 int double_counter=0,int_counter=0; 303 304 for ( i = 0; i < N_geombc_double+N_geombc_integer; i++ ) 305 { 306 fgets( target, 1024, pFile ); 307 temp = strtok( target, ";" ); 308 token = strtok( temp, "," ); 309 strcpy( S1, token ); 310 token = strtok ( NULL, "," ); 311 strcpy( S2, token ); 312 token = strtok ( NULL, "," ); 313 strcpy( S3, token ); 314 token = strtok ( NULL, "," ); 315 strcpy( S4, token ); 316 token = strtok ( NULL, "," ); 317 strcpy( S5, token ); 318 319 //if (cscompare(S3,"double")) 320 if (cscompare("double",S3)) 321 { 322 strcpy( fileTypeD[double_counter], S1 ); 323 strcpy( fieldNameD[double_counter], S2 ); 324 strcpy( dataTypeD[double_counter], S3 ); 325 strcpy( headerTypeD[double_counter], S4 ); 326 strcpy( numTemp, S5 ); 327 expectD[double_counter] = atoi (numTemp); 328 double_counter++; 329 } 330 331 //if (cscompare(S3,"integer")) 332 if (cscompare("integer",S3)) 333 { 334 strcpy( fileTypeI[int_counter], S1 ); 335 strcpy( fieldNameI[int_counter], S2 ); 336 strcpy( dataTypeI[int_counter], S3 ); 337 strcpy( headerTypeI[int_counter], S4 ); 338 strcpy( numTemp, S5 ); 339 expectI[int_counter] = atoi (numTemp); 340 int_counter++; 341 } 342 } 343 344 ////////////////////////////////////////////////////////////// 345 346 //for ( i = 0; i < N_geombc_double; i++) { 347 //printf("%d %s %s %s %s %d\n", myrank, fileTypeD[i], fieldNameD[i], dataTypeD[i], headerTypeD[i], expectD[i]); 348 //} 349 350 351 // printf("rank %d is waiting\n",myrank); 352 MPI_Barrier(MPI_COMM_WORLD); //already there 353 354 if(myrank==0){ 355 printf("Starting to read some blocks (doubles) in the geombc.dat.## files\n"); 356 } 357 358 for ( i = 0; i < nppp; i++ ) 359 { 360 if(existsubdir) { 361 subdir = (startpart+i-1) / DIR_FANOUT; 362 sprintf(gfname,"./%d-procs_case/%d/geombc.dat.%d",N_parts, subdir, startpart+i); 363 } 364 else 365 sprintf(gfname,"./%d-procs_case/geombc.dat.%d",N_parts,startpart+i); 366 367 openfile(gfname,"read",&igeom); 368 369 // MPI_Barrier(MPI_COMM_WORLD); 370 371 for ( j = 0; j < N_geombc_double; j++ ) 372 { 373 374 // printf("rank: %d - read double: %s\n",myrank,fieldNameD[j]); 375 paraD[i][j] = new int[expectD[j]]; 376 377 for ( k = 0; k < 10; k++ ) 378 iarray[k]=0; 379 380 iarray[0]=-1; 381 WriteLockD[j]=0; 382 readheader( &igeom, 383 fieldNameD[j], 384 (void*)iarray, 385 &expectD[j], 386 "double", 387 "binary" ); 388 if ( iarray[0]==-1 ) // The field requested in IO.O2N.input has not been found (should be a tpblocks) 389 WriteLockD[j]=1; 390 391 // printf("rank: %d - part: %d - field: %s - iarray: %d\n",myrank,startpart+i,fieldNameD[j],iarray[0]); 392 393 // MPI_Barrier(MPI_COMM_WORLD); 394 395 // get the parameter list in data header ... 396 // different fields have different ways to get this list ... 397 for ( k = 0; k < expectD[j]; k++ ) 398 paraD[i][j][k] = iarray[k]; 399 400 if ( WriteLockD[j]==1) // Put the value of the expected integers in the header to 0 when field not present 401 for ( k = 0; k < expectD[j]; k++ ) 402 paraD[i][j][k] = 0; 403 404 /* int iproc; 405 for(iproc=0; iproc<N_procs; iproc++){ 406 MPI_Barrier(MPI_COMM_WORLD); 407 if(iproc == myrank){ 408 printf(" iproc: %d ", iproc); 409 printf("part: %d myrank: %d field: %s header: ",startpart+i,myrank,fieldNameD[j]); 410 for ( k = 0; k < expectD[j]; k++ ) 411 printf(" %d ",iarray[k]); 412 printf("\n"); 413 } 414 usleep(100); 415 MPI_Barrier(MPI_COMM_WORLD); 416 }*/ 417 418 419 if ( cscompare("block",headerTypeD[j]) ) 420 { 421 /* if ( expectD[j]==1) 422 isize = paraD[i][j][0]; 423 else 424 isize = paraD[i][j][0] * paraD[i][j][1]; 425 426 if (cscompare("nbc values",fieldNameD[j])) 427 isize = paraD[i][j][0] * (numVariables[i]+1); 428 */ 429 430 // int test; 431 // test = computenitems(i,j,myrank,fieldNameD[j],paraD,expectD[j],numVariables[i]); 432 // printf("irank: %d fieldname: %s ParaD: %d\n",myrank,fieldNameD[j], paraD[0][0][0], numVariables[i]); 433 // if(test != isize) 434 // printf("PROBLEM fieldname: %s part: %d isize: %d test: %d\n",fieldNameD[j],startpart+i,isize,test); 435 // else 436 // printf("fieldname: %s part: %d isize: %d test: %d\n",fieldNameD[j],startpart+i,isize,test); 437 438 isize = computenitems(i,j,myrank,fieldNameD[j],paraD,expectD[j],numVariables[i]); 439 //printf("fieldname: %s part: %d isize: %d\n",fieldNameD[j],startpart+i,isize); 440 441 Dfield[i][j] = new double[isize]; 442 readdatablock( &igeom, 443 fieldNameD[j], 444 (void*)Dfield[i][j], 445 &isize, 446 "double", 447 "binary" ); 448 } 449 } 450 // MPI_Barrier(MPI_COMM_WORLD); 451 closefile(&igeom, "read"); 452 } 453 454 MPI_Barrier(MPI_COMM_WORLD); 455 if(myrank==0){ 456 printf("Starting to read some blocks (integers) in the geombc.dat.## files\n"); 457 } 458 459 // Count the number of interior and boundary tpblocks for 2 new headers named 460 // 'total number of different interior tpblocks' and 461 // 'total number of different boundary tpblocks' 462 int interiorCounter, boundaryCounter; 463 interiorCounter=0; 464 boundaryCounter=0; 465 for ( j = 0; j < N_geombc_integer; j++ ) 466 { 467 if (cscompare("connectivity interior",fieldNameI[j])) 468 { 469 //printf("part: %d, fieldNameI[j]: %s\n",GPID,fieldNameI[j]); 470 interiorCounter++; 471 } 472 else if (cscompare("connectivity boundary",fieldNameI[j])) 473 { 474 //printf("part: %d, fieldNameI[j]: %s\n",GPID,fieldNameI[j]); 475 boundaryCounter++; 476 } 477 } 478 479 MPI_Barrier(MPI_COMM_WORLD); 480 if(myrank==0){ 481 printf("There are %d total connectivity interior and %d total connectivity boundary\n", interiorCounter, boundaryCounter); 482 } 483 484 485 486 // Now, start to read the integer fields 487 for ( i = 0; i < nppp; i++ ) 488 { 489 if(existsubdir) { 490 subdir = (startpart+i-1) / DIR_FANOUT; 491 sprintf(gfname,"./%d-procs_case/%d/geombc.dat.%d",N_parts, subdir, startpart+i); 492 } 493 else 494 sprintf(gfname,"./%d-procs_case/geombc.dat.%d", N_parts, startpart+i); 495 496 openfile(gfname,"read",&igeom); 497 498 // MPI_Barrier(MPI_COMM_WORLD); 499 500 // printf("gfname is %s and nppp is %d myrank %d\n",gfname,i,myrank); 501 502 for ( j = 0; j < N_geombc_integer; j++ ) 503 { 504 505 // printf("Writing integer ... %s\n",fieldNameI[j]); 506 paraI[i][j] = new int[expectI[j]]; 507 508 for ( k = 0; k < 10; k++ ) 509 iarray[k]=0; 510 511 // printf("myrank %d and i %d j %d numBou is %d\n",myrank,i,j,numBoundaryFields[i]); 512 513 WriteLockI[j]=0; 514 iarray[0]=-1; 515 516 if ( cscompare("total number of interior tpblocks",fieldNameI[j] ) ) 517 { 518 iarray[0] = interiorCounter; //New header that does not exist in the posix file 519 } 520 else if ( cscompare("total number of boundary tpblocks",fieldNameI[j] ) ) 521 { 522 iarray[0] = boundaryCounter; //New header that does not exist in the posix file 523 } 524 else 525 { 526 readheader( &igeom, 527 fieldNameI[j], 528 (void*)iarray, 529 &expectI[j], 530 "integer", 531 "binary" ); 532 if ( iarray[0]==-1) 533 WriteLockI[j]=1; // The field was not found in the posix geombc file 534 } 535 536 //MPI_Barrier(MPI_COMM_WORLD); 537 538 /* int iproc; 539 for(iproc=0; iproc<N_procs; iproc++){ 540 MPI_Barrier(MPI_COMM_WORLD); 541 if(iproc == myrank){ 542 printf(" iproc: %d ", iproc); 543 printf("part: %d myrank: %d field: %s header: ",startpart+i,myrank,fieldNameI[j]); 544 for ( k = 0; k < expectI[j]; k++ ) 545 printf(" %d ",iarray[k]); 546 printf("\n"); 547 } 548 usleep(100); 549 MPI_Barrier(MPI_COMM_WORLD); 550 }*/ 551 552 for ( k = 0; k < expectI[j]; k++ ) 553 paraI[i][j][k] = iarray[k]; 554 555 if ( WriteLockI[j]==1) //The field is not present but SyncIO needs it to read collectively. Put 0. 556 for ( k = 0; k < expectI[j]; k++ ) 557 paraI[i][j][k] = 0; 558 559 if ( cscompare("block",headerTypeI[j]) ) 560 { 561 /* if ( expectI[j]==1) 562 isize = paraI[i][j][0]; 563 else 564 isize = paraI[i][j][0] * paraI[i][j][1]; 565 566 if (cscompare("nbc codes",fieldNameI[j])) 567 isize = paraI[i][j][0] * 2; 568 */ 569 // int test; 570 // test = computenitems(i,j,myrank,fieldNameI[j],paraI,expectI[j],numVariables[i]); 571 // printf("irank: %d fieldname: %s ParaI: %d\n",myrank,fieldNameI[j], parapI[0][0][0], numVariables[i]); 572 // if(test != isize) 573 // printf("PROBLEM fieldname: %s part: %d isize: %d test: %d\n",fieldNameI[j],startpart+i,isize,test); 574 // else 575 // printf("fieldname: %s part: %d isize: %d test: %d\n",fieldNameI[j],startpart+i,isize,test); 576 577 isize = computenitems(i,j,myrank,fieldNameI[j],paraI,expectI[j],numVariables[i]); 578 //printf("fieldname: %s part: %d isize: %d\n",fieldNameI[j],startpart+i,isize); 579 580 Ifield[i][j] = new int[isize]; 581 readdatablock( &igeom, 582 fieldNameI[j], 583 (void*)Ifield[i][j], 584 &isize, 585 "integer", 586 "binary" ); 587 } 588 } 589 // MPI_Barrier(MPI_COMM_WORLD); 590 closefile(&igeom, "read"); 591 } 592 593 MPI_Barrier(MPI_COMM_WORLD); //added by MR 594 595 ///////////////////// Writing /////////////////////////////// 596 597 int nppf = N_parts/N_files; 598 int N_geombc = N_geombc_double + N_geombc_integer; 599 int writeHandle, GPID; 600 char fname[255],fieldtag[255]; 601 602 bzero((void*)fname,255); 603 //MR CHANGE 604 // sprintf(fname,"./%d-procs_case-SyncIO",N_parts); 605 // if(0<mkdir(fname,0777)) { printf("ERROR - Could not create procs_case-SyncIO directory\n"); return 1; } 606 //MR CHANGE END 607 sprintf(fname,"./%d-procs_case-SyncIO-%d/geombc-dat.%d",N_parts,N_files,((int)(myrank/(N_procs/N_files))+1)); 608 609 MPI_Barrier(MPI_COMM_WORLD); 610 if(myrank==0){ 611 printf("Starting to write some blocks (doubles) in the geombc.dat-## files\n"); 612 } 613 614 initphmpiio(&N_geombc, &nppf, &N_files,&writeHandle, "write"); 615 // initphmpiio(&N_geombc, &nppf, &N_files,&writeHandle); 616 openfile(fname, "write", &writeHandle); 617 618 for ( i = 0; i < nppp; i++ ) 619 { 620 valuesMark[i]=0; 621 } 622 623 for ( j = 0; j < N_geombc_double; j++ ) 624 { 625 for ( i = 0; i < nppp; i++ ) 626 { 627 628 //if ( WriteLockD[i] == 0 ) 629 { 630 GPID = startpart + i; 631 bzero((void*)fieldtag,255); 632 633 fieldCompareMark=0; 634 if (cscompare("nbc values",fieldNameD[j])) 635 { 636 fieldCompareMark = 1; 637 valuesMark[i]++; 638 bzero((void*)fieldNameD[j],128); 639 sprintf(fieldNameD[j],"nbc values%d",valuesMark[i]); 640 641 // if ( valuesMark[i]>numBoundaryFields[i] ) 642 // for ( k = 0; k < expectD[j]; k++ ) 643 // paraD[i][j][k] = 0; 644 } 645 646 sprintf(fieldtag,"%s@%d",fieldNameD[j],GPID); 647 648 649 /* if ( expectD[j]==1 ) 650 isize = paraD[i][j][0]; 651 else 652 isize = paraD[i][j][0] * paraD[i][j][1]; 653 654 //specially designed for nbc values fields 655 //check the size in presolver codes 656 //Yeah, you have to open restart to get the size 657 if ( fieldCompareMark==1 ) 658 isize = paraD[i][j][0] * (numVariables[i]+1); 659 */ 660 if ( cscompare("header",headerTypeD[j]) ) 661 isize = 0; 662 else // block 663 isize = computenitems(i,j,myrank,fieldNameD[j],paraD,expectD[j],numVariables[i]); 664 665 for ( k = 0; k < expectD[j]; k++ ) 666 iarray[k] = paraD[i][j][k]; 667 668 //printf("write fieldname: %s part: %d isize: %d iarray: %d\n",fieldNameD[j],startpart+i,isize, iarray[0]); 669 670 writeheader( &writeHandle, 671 fieldtag, 672 (void*)iarray, 673 &expectD[j], 674 &isize, 675 "double", 676 "binary"); 677 writedatablock( &writeHandle, 678 fieldtag, 679 (void*)Dfield[i][j], 680 &isize, 681 "double", 682 "binary"); 683 684 if ( cscompare("block",headerTypeD[j]) ) 685 delete [] Dfield[i][j]; 686 } 687 delete [] paraD[i][j]; 688 } 689 } 690 691 for ( i = 0; i < nppp; i++ ) 692 { 693 interiorMark[i]=0; 694 boundaryMark[i]=0; 695 codesMark[i]=0; 696 } 697 698 MPI_Barrier(MPI_COMM_WORLD); 699 if(myrank==0){ 700 printf("Starting to write some blocks (integers) in the geombc.dat-## files\n"); 701 } 702 703 // Now the other fields listed in IO.O2N.input 704 for ( j = 0; j < N_geombc_integer; j++ ) 705 { 706 for ( i = 0; i < nppp; i++ ) 707 { 708 //if ( WriteLockI[i] == 0 ) 709 { 710 GPID = startpart + i; 711 bzero((void*)fieldtag,255); 712 713 if (cscompare("connectivity interior",fieldNameI[j])) 714 { 715 interiorMark[i]++; 716 bzero((void*)fieldNameI[j],128); 717 sprintf(fieldNameI[j],"connectivity interior%d",interiorMark[i]); 718 719 // if ( interiorMark[i]>numInteriorFields[i] ) 720 // for ( k = 0; k < expectI[j]; k++ ) 721 // paraI[i][j][k] = 0; 722 723 } 724 725 if (cscompare("connectivity boundary",fieldNameI[j])) 726 { 727 boundaryMark[i]++; 728 bzero((void*)fieldNameI[j],128); 729 sprintf(fieldNameI[j],"connectivity boundary%d",boundaryMark[i]); 730 731 // if ( boundaryMark[i]>numBoundaryFields[i] ) 732 // for ( k = 0; k < expectI[j]; k++ ) 733 // paraI[i][j][k] = 0; 734 } 735 736 fieldCompareMark=0; 737 if (cscompare("nbc codes",fieldNameI[j])) 738 { 739 fieldCompareMark=1; 740 codesMark[i]++; 741 bzero((void*)fieldNameI[j],128); 742 sprintf(fieldNameI[j],"nbc codes%d",codesMark[i]); 743 744 // if ( codesMark[i]>numBoundaryFields[i] ) 745 // for ( k = 0; k < expectI[j]; k++ ) 746 // paraI[i][j][k] = 0; 747 } 748 749 sprintf(fieldtag,"%s@%d",fieldNameI[j],GPID); 750 751 // if ( expectI[j]==1) 752 // isize = paraI[i][j][0]; 753 // else 754 // isize = paraI[i][j][0] * paraI[i][j][1]; 755 756 //MR CHANGE 757 // printf("rank,j,i,isize %d %d %d %d\n",myrank,j,i,isize); 758 759 760 //specially designed for nbc codes fields 761 //check the size in presolver codes 762 // if (fieldCompareMark==1) 763 // isize = paraI[i][j][0] * 2; 764 765 if ( cscompare("header",headerTypeI[j]) ) 766 isize = 0; 767 else 768 isize = computenitems(i,j,myrank,fieldNameI[j],paraI,expectI[j],numVariables[i]); 769 770 for ( k = 0; k < expectI[j]; k++ ) 771 iarray[k] = paraI[i][j][k]; 772 773 // printf("write fieldname: %s part: %d isize: %d iarray: %d\n",fieldNameI[j],startpart+i,isize, iarray[0]); 774 775 writeheader( &writeHandle, 776 fieldtag, 777 (void*)iarray, 778 &expectI[j], 779 &isize, 780 "integer", 781 "binary"); 782 writedatablock( &writeHandle, 783 fieldtag, 784 (void*)Ifield[i][j], 785 &isize, 786 "integer", 787 "binary" ); 788 789 if ( cscompare("block",headerTypeI[j]) ){ 790 // printf("rank %d - deleting Ifield %d %d\n",myrank,i,j); 791 delete [] Ifield[i][j]; 792 // printf("rank %d - Ifield deleted %d %d\n",myrank,i,j); 793 } 794 } 795 // printf("rank %d - deleting paraI %d %d\n",myrank,i,j); 796 delete [] paraI[i][j]; 797 // printf("rank %d - paraI deleted %d %d\n",myrank,i,j); 798 } 799 800 } 801 802 MPI_Barrier(MPI_COMM_WORLD); 803 if(myrank==0){ 804 printf("Closing geombc-dat.##.## files\n"); 805 } 806 807 closefile(&writeHandle, "write"); 808 finalizephmpiio(&writeHandle); 809 810 if(myrank==0){ 811 printf("Free memory related to geombc-dat.##.## files\n"); 812 } 813 814 for ( i = 0; i < nppp; i++ ) 815 { 816 delete [] Dfield[i]; 817 delete [] paraD[i]; 818 819 delete [] Ifield[i]; 820 delete [] paraI[i]; 821 822 } 823 824 for ( i = 0; i < N_geombc_double; i++ ) 825 { 826 delete [] fieldNameD[i]; 827 delete [] fileTypeD[i]; 828 delete [] dataTypeD[i]; 829 delete [] headerTypeD[i]; 830 } 831 832 for ( i = 0; i < N_geombc_integer; i++ ) 833 { 834 835 delete [] fieldNameI[i]; 836 delete [] fileTypeI[i]; 837 delete [] dataTypeI[i]; 838 delete [] headerTypeI[i]; 839 } 840 841 delete [] Dfield; 842 delete [] Ifield; 843 844 delete [] paraD; 845 delete [] paraI; 846 847 delete [] expectD; 848 delete [] expectI; 849 850 delete [] fieldNameD; 851 delete [] fileTypeD; 852 delete [] dataTypeD; 853 delete [] headerTypeD; 854 855 delete [] fieldNameI; 856 delete [] fileTypeI; 857 delete [] dataTypeI; 858 delete [] headerTypeI; 859 860 MPI_Barrier(MPI_COMM_WORLD); 861 if(myrank==0){ 862 printf("Done with geombc-dat.##.## files\n"); 863 } 864 865 /////////////////////// restart data //////////////////////////// 866 867 int irestart; 868 869 Dfield = new double**[N_restart_double]; 870 Ifield = new int**[N_restart_integer]; 871 872 paraD = new int**[N_restart_double]; 873 paraI = new int**[N_restart_integer]; 874 875 expectD = new int[N_restart_double]; 876 expectI = new int[N_restart_integer]; 877 878 fieldNameD = new char*[N_restart_double]; 879 fileTypeD = new char*[N_restart_double]; 880 dataTypeD = new char*[N_restart_double]; 881 headerTypeD = new char*[N_restart_double]; 882 883 fieldNameI = new char*[N_restart_integer]; 884 fileTypeI = new char*[N_restart_integer]; 885 dataTypeI = new char*[N_restart_integer]; 886 headerTypeI = new char*[N_restart_integer]; 887 888 if (N_restart_double>0) 889 for ( i = 0; i < N_restart_double; i++ ) 890 { 891 WriteLockD[i]=0; 892 Dfield[i] = new double*[nppp]; 893 894 paraD[i] = new int*[nppp]; 895 896 fieldNameD[i] = new char[128]; 897 fileTypeD[i] = new char[128]; 898 dataTypeD[i] = new char[128]; 899 headerTypeD[i] = new char[128]; 900 } 901 902 if (N_restart_integer>0) 903 for ( i = 0; i < N_restart_integer; i++ ) 904 { 905 WriteLockI[i]=0; 906 Ifield[i] = new int*[nppp]; 907 908 paraI[i] = new int*[nppp]; 909 910 fieldNameI[i] = new char[128]; 911 fileTypeI[i] = new char[128]; 912 dataTypeI[i] = new char[128]; 913 headerTypeI[i] = new char[128]; 914 } 915 916 //////////////////////////////////////////////////////////////// 917 // temporary fix: in the new version the double and integer 918 // can mix and match to avoid the order confusion 919 //////////////////////////////////////////////////////////////// 920 921 double_counter=0,int_counter=0; 922 923 for ( i = 0; i < N_restart_double+N_restart_integer; i++ ) 924 { 925 fgets( target, 1024, pFile ); 926 temp = strtok( target, ";" ); 927 token = strtok( temp, "," ); 928 strcpy( S1, token ); 929 token = strtok ( NULL, "," ); 930 strcpy( S2, token ); 931 token = strtok ( NULL, "," ); 932 strcpy( S3, token ); 933 token = strtok ( NULL, "," ); 934 strcpy( S4, token ); 935 token = strtok ( NULL, "," ); 936 strcpy( S5, token ); 937 938 if (cscompare(S3,"double")) 939 { 940 strcpy( fileTypeD[double_counter], S1 ); 941 strcpy( fieldNameD[double_counter], S2 ); 942 strcpy( dataTypeD[double_counter], S3 ); 943 strcpy( headerTypeD[double_counter], S4 ); 944 strcpy( numTemp, S5 ); 945 expectD[double_counter] = atoi (numTemp); 946 double_counter++; 947 } 948 949 if (cscompare(S3,"integer")) 950 { 951 strcpy( fileTypeI[int_counter], S1 ); 952 strcpy( fieldNameI[int_counter], S2 ); 953 strcpy( dataTypeI[int_counter], S3 ); 954 strcpy( headerTypeI[int_counter], S4 ); 955 strcpy( numTemp, S5 ); 956 expectI[int_counter] = atoi (numTemp); 957 int_counter++; 958 } 959 } 960 961 MPI_Barrier(MPI_COMM_WORLD); 962 if(myrank==0){ 963 printf("Starting to read some blocks (doubles) in the restart.dat.##.## files\n"); 964 } 965 966 for ( i = 0; i < N_restart_double; i++ ) 967 { 968 for ( j = 0; j < nppp; j++ ) 969 { 970 971 if(existsubdir) { 972 subdir = (startpart+j-1) / DIR_FANOUT; 973 sprintf(gfname,"./%d-procs_case/%d/restart.%d.%d",N_parts, subdir, N_steps,startpart+j); 974 } 975 else 976 sprintf(gfname,"./%d-procs_case/restart.%d.%d",N_parts, N_steps, startpart+j); 977 978 openfile(gfname,"read",&irestart); 979 980 for ( k = 0; k < 10; k++ ) 981 iarray[k]=0; 982 983 paraD[i][j] = new int[expectD[i]]; 984 985 iarray[0]=-1; 986 readheader( &irestart, 987 fieldNameD[i], 988 (void*)iarray, 989 &expectD[i], 990 "double", 991 "binary" ); 992 993 for ( k = 0; k < expectD[i]; k++ ) 994 paraD[i][j][k] = iarray[k]; 995 996 if ( iarray[0]==-1 ) 997 WriteLockD[i]=1; 998 if ( WriteLockD[i]==0 ) 999 { 1000 if ( cscompare("block",headerTypeD[i]) ) 1001 { 1002 if ( expectD[i]==1) 1003 isize = paraD[i][j][0]; 1004 else 1005 isize = paraD[i][j][0] * paraD[i][j][1]; 1006 1007 Dfield[i][j] = new double[isize]; 1008 readdatablock( &irestart, 1009 fieldNameD[i], 1010 (void*)Dfield[i][j], 1011 &isize, 1012 "double", 1013 "binary" ); 1014 1015 } 1016 } 1017 closefile(&irestart, "read"); 1018 } 1019 } 1020 1021 MPI_Barrier(MPI_COMM_WORLD); 1022 if(myrank==0){ 1023 printf("Starting to read some blocks (integers) in the restart.dat.##.## files\n"); 1024 } 1025 1026 1027 for ( i = 0; i < N_restart_integer; i++ ) 1028 { 1029 for ( j = 0; j < nppp; j++ ) 1030 { 1031 1032 if(existsubdir) { 1033 subdir = (startpart+j-1) / DIR_FANOUT; 1034 sprintf(gfname,"./%d-procs_case/%d/restart.%d.%d",N_parts, subdir, N_steps, startpart+j); 1035 } 1036 else 1037 sprintf(gfname,"./%d-procs_case/restart.%d.%d",N_parts, N_steps, startpart+j); 1038 1039 openfile(gfname,"read",&irestart); 1040 1041 for ( k = 0; k < 10; k++ ) 1042 iarray[k]=0; 1043 1044 paraI[i][j] = new int[expectI[i]]; 1045 1046 iarray[0]=-1; 1047 readheader( &irestart, 1048 fieldNameI[i], 1049 (void*)iarray, 1050 &expectI[i], 1051 "integer", 1052 "binary" ); 1053 1054 for ( k = 0; k < expectI[i]; k++ ) 1055 paraI[i][j][k] = iarray[k]; 1056 1057 if ( iarray[0]==-1 ) 1058 WriteLockI[i]=1; 1059 if ( WriteLockI[i]==0 ) 1060 { 1061 1062 if ( cscompare("block",headerTypeI[i]) ) 1063 { 1064 if ( expectI[i]==1) 1065 isize = paraI[i][j][0]; 1066 else 1067 isize = paraI[i][j][0] * paraI[i][j][1]; 1068 1069 Ifield[i][j] = new int[isize]; 1070 readdatablock( &irestart, 1071 fieldNameI[i], 1072 (void*)Ifield[i][j], 1073 &isize, 1074 "integer", 1075 "binary" ); 1076 } 1077 } 1078 closefile(&irestart, "read"); 1079 } 1080 } 1081 1082 fclose(pFile); 1083 1084 ///////////////////// Writing /////////////////////////////// 1085 1086 int N_restart = N_restart_double + N_restart_integer; 1087 1088 bzero((void*)fname,255); 1089 //MR CHANGE 1090 // sprintf(fname,"./%d-procs_case/restart-dat.%d.%d",N_parts,N_steps,((int)(myrank/(N_procs/N_files))+1)); 1091 sprintf(fname,"./%d-procs_case-SyncIO-%d/restart-dat.%d.%d",N_parts,N_files,N_steps,((int)(myrank/(N_procs/N_files))+1)); 1092 //MR CHANGE END 1093 initphmpiio(&N_restart, &nppf, &N_files,&writeHandle, "write"); 1094 // initphmpiio(&N_restart, &nppf, &N_files,&writeHandle); 1095 openfile(fname, "write", &writeHandle); 1096 1097 MPI_Barrier(MPI_COMM_WORLD); 1098 if(myrank==0){ 1099 printf("Starting to write some blocks (doubles) in the restart-dat.##.## files\n"); 1100 } 1101 1102 for ( i = 0; i < N_restart_double; i++ ) 1103 { 1104 for ( j = 0; j < nppp; j++ ) 1105 { 1106 1107 if (WriteLockD[i]==0) 1108 { 1109 GPID = startpart + j; 1110 bzero((void*)fieldtag,255); 1111 sprintf(fieldtag,"%s@%d",fieldNameD[i],GPID); 1112 1113 if ( expectD[i]==1) 1114 isize = paraD[i][j][0]; 1115 else 1116 isize = paraD[i][j][0] * paraD[i][j][1]; 1117 1118 for ( k = 0; k < expectD[i]; k++ ) 1119 iarray[k] = paraD[i][j][k]; 1120 1121 if ( cscompare("header",headerTypeD[i]) ) 1122 isize = 0; 1123 1124 writeheader( &writeHandle, 1125 fieldtag, 1126 (void*)iarray, 1127 &expectD[i], 1128 &isize, 1129 "double", 1130 "binary"); 1131 1132 writedatablock( &writeHandle, 1133 fieldtag, 1134 (void*)Dfield[i][j], 1135 &isize, 1136 "double", 1137 "binary" ); 1138 if ( cscompare("block",headerTypeD[i]) ) 1139 delete [] Dfield[i][j]; 1140 } 1141 delete [] paraD[i][j]; 1142 } 1143 } 1144 1145 MPI_Barrier(MPI_COMM_WORLD); 1146 if(myrank==0){ 1147 printf("Starting to write some blocks (integers) in the restart.dat.##.## files\n"); 1148 } 1149 1150 for ( i = 0; i < N_restart_integer; i++ ) 1151 { 1152 for ( j = 0; j < nppp; j++ ) 1153 { 1154 1155 if (WriteLockI[i]==0) 1156 { 1157 GPID = startpart + j; 1158 bzero((void*)fieldtag,255); 1159 sprintf(fieldtag,"%s@%d",fieldNameI[i],GPID); 1160 1161 if ( expectI[i]==1) 1162 isize = paraI[i][j][0]; 1163 else 1164 isize = paraI[i][j][0] * paraI[i][j][1]; 1165 1166 for ( k = 0; k < expectI[i]; k++ ) 1167 iarray[k] = paraI[i][j][k]; 1168 1169 if ( cscompare("header",headerTypeI[i]) ) 1170 isize = 0; 1171 1172 writeheader( &writeHandle, 1173 fieldtag, 1174 (void*)iarray, 1175 &expectI[i], 1176 &isize, 1177 "integer", 1178 "binary"); 1179 1180 writedatablock( &writeHandle, 1181 fieldtag, 1182 (void*)Ifield[i][j], 1183 &isize, 1184 "integer", 1185 "binary" ); 1186 1187 if ( cscompare("block",headerTypeI[i]) ) 1188 delete [] Ifield[i][j]; 1189 } 1190 delete [] paraI[i][j]; 1191 } 1192 1193 } 1194 1195 MPI_Barrier(MPI_COMM_WORLD); 1196 if(myrank==0){ 1197 printf("Closing restart-dat.##.## files\n"); 1198 } 1199 1200 closefile(&writeHandle, "write"); 1201 finalizephmpiio(&writeHandle); 1202 1203 MPI_Barrier(MPI_COMM_WORLD); 1204 if(myrank==0){ 1205 printf("Free memory related to restart-dat.##.## files\n"); 1206 } 1207 1208 1209 for ( i = 0; i < N_restart_double; i++ ) 1210 { 1211 delete [] Dfield[i]; 1212 delete [] paraD[i]; 1213 1214 delete [] fieldNameD[i]; 1215 delete [] fileTypeD[i]; 1216 delete [] dataTypeD[i]; 1217 delete [] headerTypeD[i]; 1218 } 1219 1220 for ( i = 0; i < N_restart_integer; i++ ) 1221 { 1222 delete [] Ifield[i]; 1223 delete [] paraI[i]; 1224 1225 delete [] fieldNameI[i]; 1226 delete [] fileTypeI[i]; 1227 delete [] dataTypeI[i]; 1228 delete [] headerTypeI[i]; 1229 } 1230 1231 delete [] Dfield; 1232 delete [] Ifield; 1233 1234 delete [] paraD; 1235 delete [] paraI; 1236 1237 delete [] expectD; 1238 delete [] expectI; 1239 1240 delete [] fieldNameD; 1241 delete [] fileTypeD; 1242 delete [] dataTypeD; 1243 delete [] headerTypeD; 1244 1245 delete [] fieldNameI; 1246 delete [] fileTypeI; 1247 delete [] dataTypeI; 1248 delete [] headerTypeI; 1249 1250 delete [] WriteLockD; 1251 delete [] WriteLockI; 1252 1253 delete [] interiorMark; 1254 delete [] boundaryMark; 1255 delete [] codesMark; 1256 delete [] valuesMark; 1257 delete [] numVariables; 1258 1259 if (myrank==0) 1260 { 1261 printf("\nFinished transfer, please check data using:\n"); 1262 printf(" grep -a ': <' filename \n\n"); 1263 printf("Note that the size of the fields is computed based on previous geombc files\n"); 1264 printf("Check the routine 'computenitems' if you have any reason to think it has changes for the fields you are interested in\n\n"); 1265 } 1266 1267 MPI_Finalize(); 1268 1269 } 1270 1271 1272