1 #include <stdio.h>
2 #include <iostream>
3 #include <string.h>
4 #include <stdlib.h>
5 //#define OMPI_SKIP_MPICXX 1 //Added in the CMakeList.txt file
6 #include <mpi.h>
7 #include <math.h>
8 #include "phastaIO.h"
9 #include <sys/stat.h>
10 #include <sys/types.h>
11
12 enum {
13 DIR_FANOUT = 2048
14 };
15
16 inline int
cscompare(const char teststring[],const char targetstring[])17 cscompare( const char teststring[],
18 const char targetstring[] )
19 {
20 char* s1 = const_cast<char*>(teststring);
21 char* s2 = const_cast<char*>(targetstring);
22
23 while( *s1 == ' ') s1++;
24 while( *s2 == ' ') s2++;
25 while( ( *s1 )
26 && ( *s2 )
27 && ( *s2 != '?')
28 && ( tolower( *s1 )==tolower( *s2 ) ) ) {
29 s1++;
30 s2++;
31 while( *s1 == ' ') s1++;
32 while( *s2 == ' ') s2++;
33 }
34 if ( !( *s1 ) || ( *s1 == '?') ) return 1;
35 else return 0;
36 }
37
38 inline int
computenitems(const int localpartid,const int fieldid,const int myrank,const char * fieldName,int *** para,const int intHeader,const int numVariables)39 computenitems(const int localpartid, const int fieldid, const int myrank, const char *fieldName, int ***para, const int intHeader, const int numVariables) {
40 // This routine computes the number of items in the data block based on
41 // - the name of the fields
42 // - the integers read from the header
43
44 int nItems = -1;
45
46 if (cscompare("nbc values",fieldName))
47 nItems = para[localpartid][fieldid][0] * (numVariables+1);
48 else if (cscompare("nbc codes",fieldName))
49 nItems = para[localpartid][fieldid][0] * 2;
50 else if ( intHeader==1)
51 nItems = para[localpartid][fieldid][0];
52 else
53 nItems = para[localpartid][fieldid][0] * para[localpartid][fieldid][1];
54 return nItems;
55 }
56
57
main(int argc,char * argv[])58 int main(int argc, char *argv[]) {
59
60 MPI_Init(&argc,&argv);
61
62 int myrank, N_procs;
63 MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
64 MPI_Comm_size(MPI_COMM_WORLD, &N_procs);
65
66 FILE * pFile,* AccessoryFileHandle;
67 char target[1024], pool[256], tempString[128],numpe[8],numstart[8];
68 char * temp, * token;
69 int fieldCompareMark;
70 bool readField;
71 int i, j, k, N_geombc_double, N_geombc_integer, N_restart_double;
72 int N_restart_integer, N_steps, N_parts, N_files;
73
74 pFile = fopen("./IO.O2N.input","r");
75 if (pFile == NULL)
76 printf("Error openning\n");
77
78 fgets( target, 1024, pFile );
79 token = strtok ( target, ";" );strcpy(pool,token);
80 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
81 N_geombc_double = atoi(temp);
82
83 fgets( target, 1024, pFile );
84 token = strtok ( target, ";" );strcpy(pool,token);
85 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
86 N_geombc_integer = atoi(temp);
87
88 fgets( target, 1024, pFile );
89 token = strtok ( target, ";" );strcpy(pool,token);
90 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
91 N_restart_double = atoi(temp);
92
93 fgets( target, 1024, pFile );
94 token = strtok ( target, ";" );strcpy(pool,token);
95 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
96 N_restart_integer = atoi(temp);
97
98 fgets( target, 1024, pFile );
99 token = strtok ( target, ";" );strcpy(pool,token);
100 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
101 strncpy(numstart,temp,1);
102 N_steps = atoi(temp);
103
104 fgets( target, 1024, pFile );
105 token = strtok ( target, ";" );strcpy(pool,token);
106 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
107 strncpy(numpe,temp,1);
108 N_parts = atoi(temp);
109
110 if(myrank==0){
111 printf("numpe is %d and start is %d\n",N_parts,N_steps);
112 }
113
114 fgets( target, 1024, pFile );
115 token = strtok ( target, ";" );strcpy(pool,token);
116 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
117 N_files = atoi(temp);
118
119 double ***Dfield; int ***Ifield;
120 int ***paraD, ***paraI, *expectD, *expectI;
121 char **fieldNameD, **fileTypeD, **dataTypeD, **headerTypeD;
122 char **fieldNameI, **fileTypeI, **dataTypeI, **headerTypeI;
123
124 int* WriteLockD = new int[N_geombc_double];
125 int* WriteLockI = new int[N_geombc_integer];
126
127 int nppp = N_parts/N_procs;
128 int startpart = myrank * nppp +1;
129 int endpart = startpart + nppp - 1;
130 char gfname[64], numTemp[128];
131 int iarray[10], igeom, isize,TempFileHandle;
132
133 ////////////////////////////////////////
134 // Test if the user has given the right parameters related to the number of parts, procs and SyncIO files
135 ////////////////////////////////////////
136
137 if (N_parts%N_files!=0)
138 {
139 printf("Input error: number of parts should be a multiple of number of files!\n");
140 printf("Please modify the IO.O2N.input file!\n");
141 return 0;
142 }
143 if (N_procs%N_files!=0)
144 {
145 printf("Input error: number of procs should be a multiple of number of files!\n");
146 printf("Please modify the IO.O2N.input file!\n");
147 return 0;
148 }
149 if (N_parts%N_procs!=0)
150 {
151 printf("Input error: number of parts should be a multiple of number of procs!\n");
152 printf("Please modify the IO.O2N.input file!\n");
153 return 0;
154 }
155
156 /////////////////////////////////////
157 // Create numpe.in and numstart.dat
158 ////////////////////////////////////////
159
160 if (myrank==0)
161 {
162
163 //MR CHANGE
164 bzero((void*)gfname,64);
165 sprintf(gfname,"./%d-procs_case-SyncIO-%d",N_parts,N_files);
166 if(0<mkdir(gfname,0777)) { printf("ERROR - Could not create procs_case-SyncIO directory\n"); return 1; }
167 //MR CHANGE END
168
169 bzero((void*)gfname,64);
170 sprintf(gfname,"./%d-procs_case-SyncIO-%d/numstart.dat",N_parts,N_files);
171 AccessoryFileHandle=fopen(gfname,"w");
172 fprintf(AccessoryFileHandle,"%d",N_steps);
173 fclose(AccessoryFileHandle);
174
175 bzero((void*)gfname,64);
176 sprintf(gfname,"./%d-procs_case-SyncIO-%d/numpe.in",N_parts,N_files);
177 AccessoryFileHandle=fopen(gfname,"w");
178 fprintf(AccessoryFileHandle,"%d",N_parts);
179 fclose(AccessoryFileHandle);
180 }
181
182 ///////////////////////////////////////
183 Dfield = new double**[nppp];
184 Ifield = new int**[nppp];
185
186 paraD = new int**[nppp];
187 paraI = new int**[nppp];
188 /////////////////////////////////////
189
190 int* interiorMark = new int[nppp];
191 int* boundaryMark = new int[nppp];
192 int* codesMark = new int[nppp];
193 int* valuesMark = new int[nppp];
194 int* numVariables = new int[nppp];
195 // int numBoundaryFields[nppp], numInteriorFields[nppp];
196
197 if(myrank==0){
198 printf("Starting to read some headers in the restart.##.## and geombc.dat.## files\n");
199 }
200
201 // The number of variables does not vary from one part to another. Do not read this info from every part.
202 // The ideal would be to ask only rank 0 to read only part 0 and broadcast the information to the other ranks. It saves a lot of time for very big meshes.
203 // Right now, every rank will read the number of variables from the solution field of one part only, which is already better than reading this info from every part.
204 int ithree = 3;
205 j = 0;
206
207 // Test if file exist in the procs_case directory
208 bzero((void*)gfname,64);
209 sprintf(gfname,"./%d-procs_case/restart.%d.%d", N_parts, N_steps, startpart+j);
210 int subdir = -1;
211 bool existsubdir = false;
212 FILE *pfiletest;
213 pfiletest = fopen(gfname, "r");
214 if (pfiletest == NULL ) {
215 // Test if file exist in the procs_case/subdir directory
216 subdir = (startpart+j-1) / DIR_FANOUT;
217 bzero((void*)gfname,64);
218 sprintf(gfname,"./%d-procs_case/%d/restart.%d.%d",N_parts, subdir, N_steps, startpart+j);
219 pfiletest = fopen(gfname, "r");
220 if (pfiletest == NULL ) {
221 printf("[%d] File %s does not exit - abort\n",myrank, gfname);
222 abort();
223 }
224 else{
225 existsubdir = true;
226 fclose(pfiletest);
227 }
228 }
229 else {
230 fclose(pfiletest);
231 }
232
233 // Debug
234 //printf("Rank: %d - subdir: %d - path: %s\n",myrank, subdir, gfname);
235
236 openfile(gfname,"read",&TempFileHandle);
237 readheader( &TempFileHandle,
238 "solution",
239 (void*)iarray,
240 &ithree,
241 "double",
242 "binary" );
243 closefile(&TempFileHandle, "read");
244 for ( j = 0; j < nppp; j++ )
245 numVariables[j] = iarray[1]; //iarray[i] contains the number of variables from the header of the solution field
246
247 MPI_Barrier(MPI_COMM_WORLD); //added by MR
248
249
250 /////////////////////////
251 for ( i = 0; i < nppp; i++ )
252 {
253 Dfield[i] = new double*[N_geombc_double]; // Space for the datablock of double format
254 paraD[i] = new int*[N_geombc_double]; // Integers in the header of each field of double format
255 Ifield[i] = new int*[N_geombc_integer]; // Space for the datablock of integer format
256 paraI[i] = new int*[N_geombc_integer]; // Integers in the header of each field of integer format
257
258 }
259
260 expectD = new int[N_geombc_double]; // Expected number of integers in the header for each field of double format
261 expectI = new int[N_geombc_integer]; // Expected number of integers in the header for each field of integer format
262
263 fieldNameD = new char*[N_geombc_double]; // Name of the field in double format
264 fileTypeD = new char*[N_geombc_double]; // geombc or restart (useless if associated with geombc file but read)
265 dataTypeD = new char*[N_geombc_double]; // Integer or double (useless if associated with double data but read)
266 headerTypeD = new char*[N_geombc_double]; // block (means with data block) or header (just a header with no block)
267
268 fieldNameI = new char*[N_geombc_integer];
269 fileTypeI = new char*[N_geombc_integer];
270 dataTypeI = new char*[N_geombc_integer];
271 headerTypeI = new char*[N_geombc_integer];
272
273 /////////////////////////
274
275 for ( i = 0; i < N_geombc_double; i++ )
276 {
277 WriteLockD[i]=0;
278
279 fieldNameD[i] = new char[128];
280 fileTypeD[i] = new char[128];
281 dataTypeD[i] = new char[128];
282 headerTypeD[i] = new char[128];
283 }
284
285 for ( i = 0; i < N_geombc_integer; i++ )
286 {
287 WriteLockI[i]=0; //This take value 1 if the field requested in IO.input is not found (typically for tpblocks)
288
289 fieldNameI[i] = new char[128];
290 fileTypeI[i] = new char[128];
291 dataTypeI[i] = new char[128];
292 headerTypeI[i] = new char[128];
293 }
294
295 ////////////////////////////////////////////////////////////////
296 // Reading IO.input
297 // temporary fix: in the new version the double and integer
298 // can mix and match to avoid the order confusion
299 ////////////////////////////////////////////////////////////////
300
301 char S1[128],S2[128],S3[128],S4[128],S5[128];
302 int double_counter=0,int_counter=0;
303
304 for ( i = 0; i < N_geombc_double+N_geombc_integer; i++ )
305 {
306 fgets( target, 1024, pFile );
307 temp = strtok( target, ";" );
308 token = strtok( temp, "," );
309 strcpy( S1, token );
310 token = strtok ( NULL, "," );
311 strcpy( S2, token );
312 token = strtok ( NULL, "," );
313 strcpy( S3, token );
314 token = strtok ( NULL, "," );
315 strcpy( S4, token );
316 token = strtok ( NULL, "," );
317 strcpy( S5, token );
318
319 //if (cscompare(S3,"double"))
320 if (cscompare("double",S3))
321 {
322 strcpy( fileTypeD[double_counter], S1 );
323 strcpy( fieldNameD[double_counter], S2 );
324 strcpy( dataTypeD[double_counter], S3 );
325 strcpy( headerTypeD[double_counter], S4 );
326 strcpy( numTemp, S5 );
327 expectD[double_counter] = atoi (numTemp);
328 double_counter++;
329 }
330
331 //if (cscompare(S3,"integer"))
332 if (cscompare("integer",S3))
333 {
334 strcpy( fileTypeI[int_counter], S1 );
335 strcpy( fieldNameI[int_counter], S2 );
336 strcpy( dataTypeI[int_counter], S3 );
337 strcpy( headerTypeI[int_counter], S4 );
338 strcpy( numTemp, S5 );
339 expectI[int_counter] = atoi (numTemp);
340 int_counter++;
341 }
342 }
343
344 //////////////////////////////////////////////////////////////
345
346 //for ( i = 0; i < N_geombc_double; i++) {
347 //printf("%d %s %s %s %s %d\n", myrank, fileTypeD[i], fieldNameD[i], dataTypeD[i], headerTypeD[i], expectD[i]);
348 //}
349
350
351 // printf("rank %d is waiting\n",myrank);
352 MPI_Barrier(MPI_COMM_WORLD); //already there
353
354 if(myrank==0){
355 printf("Starting to read some blocks (doubles) in the geombc.dat.## files\n");
356 }
357
358 for ( i = 0; i < nppp; i++ )
359 {
360 if(existsubdir) {
361 subdir = (startpart+i-1) / DIR_FANOUT;
362 sprintf(gfname,"./%d-procs_case/%d/geombc.dat.%d",N_parts, subdir, startpart+i);
363 }
364 else
365 sprintf(gfname,"./%d-procs_case/geombc.dat.%d",N_parts,startpart+i);
366
367 openfile(gfname,"read",&igeom);
368
369 // MPI_Barrier(MPI_COMM_WORLD);
370
371 for ( j = 0; j < N_geombc_double; j++ )
372 {
373
374 // printf("rank: %d - read double: %s\n",myrank,fieldNameD[j]);
375 paraD[i][j] = new int[expectD[j]];
376
377 for ( k = 0; k < 10; k++ )
378 iarray[k]=0;
379
380 iarray[0]=-1;
381 WriteLockD[j]=0;
382 readheader( &igeom,
383 fieldNameD[j],
384 (void*)iarray,
385 &expectD[j],
386 "double",
387 "binary" );
388 if ( iarray[0]==-1 ) // The field requested in IO.O2N.input has not been found (should be a tpblocks)
389 WriteLockD[j]=1;
390
391 // printf("rank: %d - part: %d - field: %s - iarray: %d\n",myrank,startpart+i,fieldNameD[j],iarray[0]);
392
393 // MPI_Barrier(MPI_COMM_WORLD);
394
395 // get the parameter list in data header ...
396 // different fields have different ways to get this list ...
397 for ( k = 0; k < expectD[j]; k++ )
398 paraD[i][j][k] = iarray[k];
399
400 if ( WriteLockD[j]==1) // Put the value of the expected integers in the header to 0 when field not present
401 for ( k = 0; k < expectD[j]; k++ )
402 paraD[i][j][k] = 0;
403
404 /* int iproc;
405 for(iproc=0; iproc<N_procs; iproc++){
406 MPI_Barrier(MPI_COMM_WORLD);
407 if(iproc == myrank){
408 printf(" iproc: %d ", iproc);
409 printf("part: %d myrank: %d field: %s header: ",startpart+i,myrank,fieldNameD[j]);
410 for ( k = 0; k < expectD[j]; k++ )
411 printf(" %d ",iarray[k]);
412 printf("\n");
413 }
414 usleep(100);
415 MPI_Barrier(MPI_COMM_WORLD);
416 }*/
417
418
419 if ( cscompare("block",headerTypeD[j]) )
420 {
421 /* if ( expectD[j]==1)
422 isize = paraD[i][j][0];
423 else
424 isize = paraD[i][j][0] * paraD[i][j][1];
425
426 if (cscompare("nbc values",fieldNameD[j]))
427 isize = paraD[i][j][0] * (numVariables[i]+1);
428 */
429
430 // int test;
431 // test = computenitems(i,j,myrank,fieldNameD[j],paraD,expectD[j],numVariables[i]);
432 // printf("irank: %d fieldname: %s ParaD: %d\n",myrank,fieldNameD[j], paraD[0][0][0], numVariables[i]);
433 // if(test != isize)
434 // printf("PROBLEM fieldname: %s part: %d isize: %d test: %d\n",fieldNameD[j],startpart+i,isize,test);
435 // else
436 // printf("fieldname: %s part: %d isize: %d test: %d\n",fieldNameD[j],startpart+i,isize,test);
437
438 isize = computenitems(i,j,myrank,fieldNameD[j],paraD,expectD[j],numVariables[i]);
439 //printf("fieldname: %s part: %d isize: %d\n",fieldNameD[j],startpart+i,isize);
440
441 Dfield[i][j] = new double[isize];
442 readdatablock( &igeom,
443 fieldNameD[j],
444 (void*)Dfield[i][j],
445 &isize,
446 "double",
447 "binary" );
448 }
449 }
450 // MPI_Barrier(MPI_COMM_WORLD);
451 closefile(&igeom, "read");
452 }
453
454 MPI_Barrier(MPI_COMM_WORLD);
455 if(myrank==0){
456 printf("Starting to read some blocks (integers) in the geombc.dat.## files\n");
457 }
458
459 // Count the number of interior and boundary tpblocks for 2 new headers named
460 // 'total number of different interior tpblocks' and
461 // 'total number of different boundary tpblocks'
462 int interiorCounter, boundaryCounter;
463 interiorCounter=0;
464 boundaryCounter=0;
465 for ( j = 0; j < N_geombc_integer; j++ )
466 {
467 if (cscompare("connectivity interior",fieldNameI[j]))
468 {
469 //printf("part: %d, fieldNameI[j]: %s\n",GPID,fieldNameI[j]);
470 interiorCounter++;
471 }
472 else if (cscompare("connectivity boundary",fieldNameI[j]))
473 {
474 //printf("part: %d, fieldNameI[j]: %s\n",GPID,fieldNameI[j]);
475 boundaryCounter++;
476 }
477 }
478
479 MPI_Barrier(MPI_COMM_WORLD);
480 if(myrank==0){
481 printf("There are %d total connectivity interior and %d total connectivity boundary\n", interiorCounter, boundaryCounter);
482 }
483
484
485
486 // Now, start to read the integer fields
487 for ( i = 0; i < nppp; i++ )
488 {
489 if(existsubdir) {
490 subdir = (startpart+i-1) / DIR_FANOUT;
491 sprintf(gfname,"./%d-procs_case/%d/geombc.dat.%d",N_parts, subdir, startpart+i);
492 }
493 else
494 sprintf(gfname,"./%d-procs_case/geombc.dat.%d", N_parts, startpart+i);
495
496 openfile(gfname,"read",&igeom);
497
498 // MPI_Barrier(MPI_COMM_WORLD);
499
500 // printf("gfname is %s and nppp is %d myrank %d\n",gfname,i,myrank);
501
502 for ( j = 0; j < N_geombc_integer; j++ )
503 {
504
505 // printf("Writing integer ... %s\n",fieldNameI[j]);
506 paraI[i][j] = new int[expectI[j]];
507
508 for ( k = 0; k < 10; k++ )
509 iarray[k]=0;
510
511 // printf("myrank %d and i %d j %d numBou is %d\n",myrank,i,j,numBoundaryFields[i]);
512
513 WriteLockI[j]=0;
514 iarray[0]=-1;
515
516 if ( cscompare("total number of interior tpblocks",fieldNameI[j] ) )
517 {
518 iarray[0] = interiorCounter; //New header that does not exist in the posix file
519 }
520 else if ( cscompare("total number of boundary tpblocks",fieldNameI[j] ) )
521 {
522 iarray[0] = boundaryCounter; //New header that does not exist in the posix file
523 }
524 else
525 {
526 readheader( &igeom,
527 fieldNameI[j],
528 (void*)iarray,
529 &expectI[j],
530 "integer",
531 "binary" );
532 if ( iarray[0]==-1)
533 WriteLockI[j]=1; // The field was not found in the posix geombc file
534 }
535
536 //MPI_Barrier(MPI_COMM_WORLD);
537
538 /* int iproc;
539 for(iproc=0; iproc<N_procs; iproc++){
540 MPI_Barrier(MPI_COMM_WORLD);
541 if(iproc == myrank){
542 printf(" iproc: %d ", iproc);
543 printf("part: %d myrank: %d field: %s header: ",startpart+i,myrank,fieldNameI[j]);
544 for ( k = 0; k < expectI[j]; k++ )
545 printf(" %d ",iarray[k]);
546 printf("\n");
547 }
548 usleep(100);
549 MPI_Barrier(MPI_COMM_WORLD);
550 }*/
551
552 for ( k = 0; k < expectI[j]; k++ )
553 paraI[i][j][k] = iarray[k];
554
555 if ( WriteLockI[j]==1) //The field is not present but SyncIO needs it to read collectively. Put 0.
556 for ( k = 0; k < expectI[j]; k++ )
557 paraI[i][j][k] = 0;
558
559 if ( cscompare("block",headerTypeI[j]) )
560 {
561 /* if ( expectI[j]==1)
562 isize = paraI[i][j][0];
563 else
564 isize = paraI[i][j][0] * paraI[i][j][1];
565
566 if (cscompare("nbc codes",fieldNameI[j]))
567 isize = paraI[i][j][0] * 2;
568 */
569 // int test;
570 // test = computenitems(i,j,myrank,fieldNameI[j],paraI,expectI[j],numVariables[i]);
571 // printf("irank: %d fieldname: %s ParaI: %d\n",myrank,fieldNameI[j], parapI[0][0][0], numVariables[i]);
572 // if(test != isize)
573 // printf("PROBLEM fieldname: %s part: %d isize: %d test: %d\n",fieldNameI[j],startpart+i,isize,test);
574 // else
575 // printf("fieldname: %s part: %d isize: %d test: %d\n",fieldNameI[j],startpart+i,isize,test);
576
577 isize = computenitems(i,j,myrank,fieldNameI[j],paraI,expectI[j],numVariables[i]);
578 //printf("fieldname: %s part: %d isize: %d\n",fieldNameI[j],startpart+i,isize);
579
580 Ifield[i][j] = new int[isize];
581 readdatablock( &igeom,
582 fieldNameI[j],
583 (void*)Ifield[i][j],
584 &isize,
585 "integer",
586 "binary" );
587 }
588 }
589 // MPI_Barrier(MPI_COMM_WORLD);
590 closefile(&igeom, "read");
591 }
592
593 MPI_Barrier(MPI_COMM_WORLD); //added by MR
594
595 ///////////////////// Writing ///////////////////////////////
596
597 int nppf = N_parts/N_files;
598 int N_geombc = N_geombc_double + N_geombc_integer;
599 int writeHandle, GPID;
600 char fname[255],fieldtag[255];
601
602 bzero((void*)fname,255);
603 //MR CHANGE
604 // sprintf(fname,"./%d-procs_case-SyncIO",N_parts);
605 // if(0<mkdir(fname,0777)) { printf("ERROR - Could not create procs_case-SyncIO directory\n"); return 1; }
606 //MR CHANGE END
607 sprintf(fname,"./%d-procs_case-SyncIO-%d/geombc-dat.%d",N_parts,N_files,((int)(myrank/(N_procs/N_files))+1));
608
609 MPI_Barrier(MPI_COMM_WORLD);
610 if(myrank==0){
611 printf("Starting to write some blocks (doubles) in the geombc.dat-## files\n");
612 }
613
614 initphmpiio(&N_geombc, &nppf, &N_files,&writeHandle, "write");
615 // initphmpiio(&N_geombc, &nppf, &N_files,&writeHandle);
616 openfile(fname, "write", &writeHandle);
617
618 for ( i = 0; i < nppp; i++ )
619 {
620 valuesMark[i]=0;
621 }
622
623 for ( j = 0; j < N_geombc_double; j++ )
624 {
625 for ( i = 0; i < nppp; i++ )
626 {
627
628 //if ( WriteLockD[i] == 0 )
629 {
630 GPID = startpart + i;
631 bzero((void*)fieldtag,255);
632
633 fieldCompareMark=0;
634 if (cscompare("nbc values",fieldNameD[j]))
635 {
636 fieldCompareMark = 1;
637 valuesMark[i]++;
638 bzero((void*)fieldNameD[j],128);
639 sprintf(fieldNameD[j],"nbc values%d",valuesMark[i]);
640
641 // if ( valuesMark[i]>numBoundaryFields[i] )
642 // for ( k = 0; k < expectD[j]; k++ )
643 // paraD[i][j][k] = 0;
644 }
645
646 sprintf(fieldtag,"%s@%d",fieldNameD[j],GPID);
647
648
649 /* if ( expectD[j]==1 )
650 isize = paraD[i][j][0];
651 else
652 isize = paraD[i][j][0] * paraD[i][j][1];
653
654 //specially designed for nbc values fields
655 //check the size in presolver codes
656 //Yeah, you have to open restart to get the size
657 if ( fieldCompareMark==1 )
658 isize = paraD[i][j][0] * (numVariables[i]+1);
659 */
660 if ( cscompare("header",headerTypeD[j]) )
661 isize = 0;
662 else // block
663 isize = computenitems(i,j,myrank,fieldNameD[j],paraD,expectD[j],numVariables[i]);
664
665 for ( k = 0; k < expectD[j]; k++ )
666 iarray[k] = paraD[i][j][k];
667
668 //printf("write fieldname: %s part: %d isize: %d iarray: %d\n",fieldNameD[j],startpart+i,isize, iarray[0]);
669
670 writeheader( &writeHandle,
671 fieldtag,
672 (void*)iarray,
673 &expectD[j],
674 &isize,
675 "double",
676 "binary");
677 writedatablock( &writeHandle,
678 fieldtag,
679 (void*)Dfield[i][j],
680 &isize,
681 "double",
682 "binary");
683
684 if ( cscompare("block",headerTypeD[j]) )
685 delete [] Dfield[i][j];
686 }
687 delete [] paraD[i][j];
688 }
689 }
690
691 for ( i = 0; i < nppp; i++ )
692 {
693 interiorMark[i]=0;
694 boundaryMark[i]=0;
695 codesMark[i]=0;
696 }
697
698 MPI_Barrier(MPI_COMM_WORLD);
699 if(myrank==0){
700 printf("Starting to write some blocks (integers) in the geombc.dat-## files\n");
701 }
702
703 // Now the other fields listed in IO.O2N.input
704 for ( j = 0; j < N_geombc_integer; j++ )
705 {
706 for ( i = 0; i < nppp; i++ )
707 {
708 //if ( WriteLockI[i] == 0 )
709 {
710 GPID = startpart + i;
711 bzero((void*)fieldtag,255);
712
713 if (cscompare("connectivity interior",fieldNameI[j]))
714 {
715 interiorMark[i]++;
716 bzero((void*)fieldNameI[j],128);
717 sprintf(fieldNameI[j],"connectivity interior%d",interiorMark[i]);
718
719 // if ( interiorMark[i]>numInteriorFields[i] )
720 // for ( k = 0; k < expectI[j]; k++ )
721 // paraI[i][j][k] = 0;
722
723 }
724
725 if (cscompare("connectivity boundary",fieldNameI[j]))
726 {
727 boundaryMark[i]++;
728 bzero((void*)fieldNameI[j],128);
729 sprintf(fieldNameI[j],"connectivity boundary%d",boundaryMark[i]);
730
731 // if ( boundaryMark[i]>numBoundaryFields[i] )
732 // for ( k = 0; k < expectI[j]; k++ )
733 // paraI[i][j][k] = 0;
734 }
735
736 fieldCompareMark=0;
737 if (cscompare("nbc codes",fieldNameI[j]))
738 {
739 fieldCompareMark=1;
740 codesMark[i]++;
741 bzero((void*)fieldNameI[j],128);
742 sprintf(fieldNameI[j],"nbc codes%d",codesMark[i]);
743
744 // if ( codesMark[i]>numBoundaryFields[i] )
745 // for ( k = 0; k < expectI[j]; k++ )
746 // paraI[i][j][k] = 0;
747 }
748
749 sprintf(fieldtag,"%s@%d",fieldNameI[j],GPID);
750
751 // if ( expectI[j]==1)
752 // isize = paraI[i][j][0];
753 // else
754 // isize = paraI[i][j][0] * paraI[i][j][1];
755
756 //MR CHANGE
757 // printf("rank,j,i,isize %d %d %d %d\n",myrank,j,i,isize);
758
759
760 //specially designed for nbc codes fields
761 //check the size in presolver codes
762 // if (fieldCompareMark==1)
763 // isize = paraI[i][j][0] * 2;
764
765 if ( cscompare("header",headerTypeI[j]) )
766 isize = 0;
767 else
768 isize = computenitems(i,j,myrank,fieldNameI[j],paraI,expectI[j],numVariables[i]);
769
770 for ( k = 0; k < expectI[j]; k++ )
771 iarray[k] = paraI[i][j][k];
772
773 // printf("write fieldname: %s part: %d isize: %d iarray: %d\n",fieldNameI[j],startpart+i,isize, iarray[0]);
774
775 writeheader( &writeHandle,
776 fieldtag,
777 (void*)iarray,
778 &expectI[j],
779 &isize,
780 "integer",
781 "binary");
782 writedatablock( &writeHandle,
783 fieldtag,
784 (void*)Ifield[i][j],
785 &isize,
786 "integer",
787 "binary" );
788
789 if ( cscompare("block",headerTypeI[j]) ){
790 // printf("rank %d - deleting Ifield %d %d\n",myrank,i,j);
791 delete [] Ifield[i][j];
792 // printf("rank %d - Ifield deleted %d %d\n",myrank,i,j);
793 }
794 }
795 // printf("rank %d - deleting paraI %d %d\n",myrank,i,j);
796 delete [] paraI[i][j];
797 // printf("rank %d - paraI deleted %d %d\n",myrank,i,j);
798 }
799
800 }
801
802 MPI_Barrier(MPI_COMM_WORLD);
803 if(myrank==0){
804 printf("Closing geombc-dat.##.## files\n");
805 }
806
807 closefile(&writeHandle, "write");
808 finalizephmpiio(&writeHandle);
809
810 if(myrank==0){
811 printf("Free memory related to geombc-dat.##.## files\n");
812 }
813
814 for ( i = 0; i < nppp; i++ )
815 {
816 delete [] Dfield[i];
817 delete [] paraD[i];
818
819 delete [] Ifield[i];
820 delete [] paraI[i];
821
822 }
823
824 for ( i = 0; i < N_geombc_double; i++ )
825 {
826 delete [] fieldNameD[i];
827 delete [] fileTypeD[i];
828 delete [] dataTypeD[i];
829 delete [] headerTypeD[i];
830 }
831
832 for ( i = 0; i < N_geombc_integer; i++ )
833 {
834
835 delete [] fieldNameI[i];
836 delete [] fileTypeI[i];
837 delete [] dataTypeI[i];
838 delete [] headerTypeI[i];
839 }
840
841 delete [] Dfield;
842 delete [] Ifield;
843
844 delete [] paraD;
845 delete [] paraI;
846
847 delete [] expectD;
848 delete [] expectI;
849
850 delete [] fieldNameD;
851 delete [] fileTypeD;
852 delete [] dataTypeD;
853 delete [] headerTypeD;
854
855 delete [] fieldNameI;
856 delete [] fileTypeI;
857 delete [] dataTypeI;
858 delete [] headerTypeI;
859
860 MPI_Barrier(MPI_COMM_WORLD);
861 if(myrank==0){
862 printf("Done with geombc-dat.##.## files\n");
863 }
864
865 /////////////////////// restart data ////////////////////////////
866
867 int irestart;
868
869 Dfield = new double**[N_restart_double];
870 Ifield = new int**[N_restart_integer];
871
872 paraD = new int**[N_restart_double];
873 paraI = new int**[N_restart_integer];
874
875 expectD = new int[N_restart_double];
876 expectI = new int[N_restart_integer];
877
878 fieldNameD = new char*[N_restart_double];
879 fileTypeD = new char*[N_restart_double];
880 dataTypeD = new char*[N_restart_double];
881 headerTypeD = new char*[N_restart_double];
882
883 fieldNameI = new char*[N_restart_integer];
884 fileTypeI = new char*[N_restart_integer];
885 dataTypeI = new char*[N_restart_integer];
886 headerTypeI = new char*[N_restart_integer];
887
888 if (N_restart_double>0)
889 for ( i = 0; i < N_restart_double; i++ )
890 {
891 WriteLockD[i]=0;
892 Dfield[i] = new double*[nppp];
893
894 paraD[i] = new int*[nppp];
895
896 fieldNameD[i] = new char[128];
897 fileTypeD[i] = new char[128];
898 dataTypeD[i] = new char[128];
899 headerTypeD[i] = new char[128];
900 }
901
902 if (N_restart_integer>0)
903 for ( i = 0; i < N_restart_integer; i++ )
904 {
905 WriteLockI[i]=0;
906 Ifield[i] = new int*[nppp];
907
908 paraI[i] = new int*[nppp];
909
910 fieldNameI[i] = new char[128];
911 fileTypeI[i] = new char[128];
912 dataTypeI[i] = new char[128];
913 headerTypeI[i] = new char[128];
914 }
915
916 ////////////////////////////////////////////////////////////////
917 // temporary fix: in the new version the double and integer
918 // can mix and match to avoid the order confusion
919 ////////////////////////////////////////////////////////////////
920
921 double_counter=0,int_counter=0;
922
923 for ( i = 0; i < N_restart_double+N_restart_integer; i++ )
924 {
925 fgets( target, 1024, pFile );
926 temp = strtok( target, ";" );
927 token = strtok( temp, "," );
928 strcpy( S1, token );
929 token = strtok ( NULL, "," );
930 strcpy( S2, token );
931 token = strtok ( NULL, "," );
932 strcpy( S3, token );
933 token = strtok ( NULL, "," );
934 strcpy( S4, token );
935 token = strtok ( NULL, "," );
936 strcpy( S5, token );
937
938 if (cscompare(S3,"double"))
939 {
940 strcpy( fileTypeD[double_counter], S1 );
941 strcpy( fieldNameD[double_counter], S2 );
942 strcpy( dataTypeD[double_counter], S3 );
943 strcpy( headerTypeD[double_counter], S4 );
944 strcpy( numTemp, S5 );
945 expectD[double_counter] = atoi (numTemp);
946 double_counter++;
947 }
948
949 if (cscompare(S3,"integer"))
950 {
951 strcpy( fileTypeI[int_counter], S1 );
952 strcpy( fieldNameI[int_counter], S2 );
953 strcpy( dataTypeI[int_counter], S3 );
954 strcpy( headerTypeI[int_counter], S4 );
955 strcpy( numTemp, S5 );
956 expectI[int_counter] = atoi (numTemp);
957 int_counter++;
958 }
959 }
960
961 MPI_Barrier(MPI_COMM_WORLD);
962 if(myrank==0){
963 printf("Starting to read some blocks (doubles) in the restart.dat.##.## files\n");
964 }
965
966 for ( i = 0; i < N_restart_double; i++ )
967 {
968 for ( j = 0; j < nppp; j++ )
969 {
970
971 if(existsubdir) {
972 subdir = (startpart+j-1) / DIR_FANOUT;
973 sprintf(gfname,"./%d-procs_case/%d/restart.%d.%d",N_parts, subdir, N_steps,startpart+j);
974 }
975 else
976 sprintf(gfname,"./%d-procs_case/restart.%d.%d",N_parts, N_steps, startpart+j);
977
978 openfile(gfname,"read",&irestart);
979
980 for ( k = 0; k < 10; k++ )
981 iarray[k]=0;
982
983 paraD[i][j] = new int[expectD[i]];
984
985 iarray[0]=-1;
986 readheader( &irestart,
987 fieldNameD[i],
988 (void*)iarray,
989 &expectD[i],
990 "double",
991 "binary" );
992
993 for ( k = 0; k < expectD[i]; k++ )
994 paraD[i][j][k] = iarray[k];
995
996 if ( iarray[0]==-1 )
997 WriteLockD[i]=1;
998 if ( WriteLockD[i]==0 )
999 {
1000 if ( cscompare("block",headerTypeD[i]) )
1001 {
1002 if ( expectD[i]==1)
1003 isize = paraD[i][j][0];
1004 else
1005 isize = paraD[i][j][0] * paraD[i][j][1];
1006
1007 Dfield[i][j] = new double[isize];
1008 readdatablock( &irestart,
1009 fieldNameD[i],
1010 (void*)Dfield[i][j],
1011 &isize,
1012 "double",
1013 "binary" );
1014
1015 }
1016 }
1017 closefile(&irestart, "read");
1018 }
1019 }
1020
1021 MPI_Barrier(MPI_COMM_WORLD);
1022 if(myrank==0){
1023 printf("Starting to read some blocks (integers) in the restart.dat.##.## files\n");
1024 }
1025
1026
1027 for ( i = 0; i < N_restart_integer; i++ )
1028 {
1029 for ( j = 0; j < nppp; j++ )
1030 {
1031
1032 if(existsubdir) {
1033 subdir = (startpart+j-1) / DIR_FANOUT;
1034 sprintf(gfname,"./%d-procs_case/%d/restart.%d.%d",N_parts, subdir, N_steps, startpart+j);
1035 }
1036 else
1037 sprintf(gfname,"./%d-procs_case/restart.%d.%d",N_parts, N_steps, startpart+j);
1038
1039 openfile(gfname,"read",&irestart);
1040
1041 for ( k = 0; k < 10; k++ )
1042 iarray[k]=0;
1043
1044 paraI[i][j] = new int[expectI[i]];
1045
1046 iarray[0]=-1;
1047 readheader( &irestart,
1048 fieldNameI[i],
1049 (void*)iarray,
1050 &expectI[i],
1051 "integer",
1052 "binary" );
1053
1054 for ( k = 0; k < expectI[i]; k++ )
1055 paraI[i][j][k] = iarray[k];
1056
1057 if ( iarray[0]==-1 )
1058 WriteLockI[i]=1;
1059 if ( WriteLockI[i]==0 )
1060 {
1061
1062 if ( cscompare("block",headerTypeI[i]) )
1063 {
1064 if ( expectI[i]==1)
1065 isize = paraI[i][j][0];
1066 else
1067 isize = paraI[i][j][0] * paraI[i][j][1];
1068
1069 Ifield[i][j] = new int[isize];
1070 readdatablock( &irestart,
1071 fieldNameI[i],
1072 (void*)Ifield[i][j],
1073 &isize,
1074 "integer",
1075 "binary" );
1076 }
1077 }
1078 closefile(&irestart, "read");
1079 }
1080 }
1081
1082 fclose(pFile);
1083
1084 ///////////////////// Writing ///////////////////////////////
1085
1086 int N_restart = N_restart_double + N_restart_integer;
1087
1088 bzero((void*)fname,255);
1089 //MR CHANGE
1090 // sprintf(fname,"./%d-procs_case/restart-dat.%d.%d",N_parts,N_steps,((int)(myrank/(N_procs/N_files))+1));
1091 sprintf(fname,"./%d-procs_case-SyncIO-%d/restart-dat.%d.%d",N_parts,N_files,N_steps,((int)(myrank/(N_procs/N_files))+1));
1092 //MR CHANGE END
1093 initphmpiio(&N_restart, &nppf, &N_files,&writeHandle, "write");
1094 // initphmpiio(&N_restart, &nppf, &N_files,&writeHandle);
1095 openfile(fname, "write", &writeHandle);
1096
1097 MPI_Barrier(MPI_COMM_WORLD);
1098 if(myrank==0){
1099 printf("Starting to write some blocks (doubles) in the restart-dat.##.## files\n");
1100 }
1101
1102 for ( i = 0; i < N_restart_double; i++ )
1103 {
1104 for ( j = 0; j < nppp; j++ )
1105 {
1106
1107 if (WriteLockD[i]==0)
1108 {
1109 GPID = startpart + j;
1110 bzero((void*)fieldtag,255);
1111 sprintf(fieldtag,"%s@%d",fieldNameD[i],GPID);
1112
1113 if ( expectD[i]==1)
1114 isize = paraD[i][j][0];
1115 else
1116 isize = paraD[i][j][0] * paraD[i][j][1];
1117
1118 for ( k = 0; k < expectD[i]; k++ )
1119 iarray[k] = paraD[i][j][k];
1120
1121 if ( cscompare("header",headerTypeD[i]) )
1122 isize = 0;
1123
1124 writeheader( &writeHandle,
1125 fieldtag,
1126 (void*)iarray,
1127 &expectD[i],
1128 &isize,
1129 "double",
1130 "binary");
1131
1132 writedatablock( &writeHandle,
1133 fieldtag,
1134 (void*)Dfield[i][j],
1135 &isize,
1136 "double",
1137 "binary" );
1138 if ( cscompare("block",headerTypeD[i]) )
1139 delete [] Dfield[i][j];
1140 }
1141 delete [] paraD[i][j];
1142 }
1143 }
1144
1145 MPI_Barrier(MPI_COMM_WORLD);
1146 if(myrank==0){
1147 printf("Starting to write some blocks (integers) in the restart.dat.##.## files\n");
1148 }
1149
1150 for ( i = 0; i < N_restart_integer; i++ )
1151 {
1152 for ( j = 0; j < nppp; j++ )
1153 {
1154
1155 if (WriteLockI[i]==0)
1156 {
1157 GPID = startpart + j;
1158 bzero((void*)fieldtag,255);
1159 sprintf(fieldtag,"%s@%d",fieldNameI[i],GPID);
1160
1161 if ( expectI[i]==1)
1162 isize = paraI[i][j][0];
1163 else
1164 isize = paraI[i][j][0] * paraI[i][j][1];
1165
1166 for ( k = 0; k < expectI[i]; k++ )
1167 iarray[k] = paraI[i][j][k];
1168
1169 if ( cscompare("header",headerTypeI[i]) )
1170 isize = 0;
1171
1172 writeheader( &writeHandle,
1173 fieldtag,
1174 (void*)iarray,
1175 &expectI[i],
1176 &isize,
1177 "integer",
1178 "binary");
1179
1180 writedatablock( &writeHandle,
1181 fieldtag,
1182 (void*)Ifield[i][j],
1183 &isize,
1184 "integer",
1185 "binary" );
1186
1187 if ( cscompare("block",headerTypeI[i]) )
1188 delete [] Ifield[i][j];
1189 }
1190 delete [] paraI[i][j];
1191 }
1192
1193 }
1194
1195 MPI_Barrier(MPI_COMM_WORLD);
1196 if(myrank==0){
1197 printf("Closing restart-dat.##.## files\n");
1198 }
1199
1200 closefile(&writeHandle, "write");
1201 finalizephmpiio(&writeHandle);
1202
1203 MPI_Barrier(MPI_COMM_WORLD);
1204 if(myrank==0){
1205 printf("Free memory related to restart-dat.##.## files\n");
1206 }
1207
1208
1209 for ( i = 0; i < N_restart_double; i++ )
1210 {
1211 delete [] Dfield[i];
1212 delete [] paraD[i];
1213
1214 delete [] fieldNameD[i];
1215 delete [] fileTypeD[i];
1216 delete [] dataTypeD[i];
1217 delete [] headerTypeD[i];
1218 }
1219
1220 for ( i = 0; i < N_restart_integer; i++ )
1221 {
1222 delete [] Ifield[i];
1223 delete [] paraI[i];
1224
1225 delete [] fieldNameI[i];
1226 delete [] fileTypeI[i];
1227 delete [] dataTypeI[i];
1228 delete [] headerTypeI[i];
1229 }
1230
1231 delete [] Dfield;
1232 delete [] Ifield;
1233
1234 delete [] paraD;
1235 delete [] paraI;
1236
1237 delete [] expectD;
1238 delete [] expectI;
1239
1240 delete [] fieldNameD;
1241 delete [] fileTypeD;
1242 delete [] dataTypeD;
1243 delete [] headerTypeD;
1244
1245 delete [] fieldNameI;
1246 delete [] fileTypeI;
1247 delete [] dataTypeI;
1248 delete [] headerTypeI;
1249
1250 delete [] WriteLockD;
1251 delete [] WriteLockI;
1252
1253 delete [] interiorMark;
1254 delete [] boundaryMark;
1255 delete [] codesMark;
1256 delete [] valuesMark;
1257 delete [] numVariables;
1258
1259 if (myrank==0)
1260 {
1261 printf("\nFinished transfer, please check data using:\n");
1262 printf(" grep -a ': <' filename \n\n");
1263 printf("Note that the size of the fields is computed based on previous geombc files\n");
1264 printf("Check the routine 'computenitems' if you have any reason to think it has changes for the fields you are interested in\n\n");
1265 }
1266
1267 MPI_Finalize();
1268
1269 }
1270
1271
1272