1 #include <stdio.h>
2 #include <iostream>
3 #include <string.h>
4 #include <stdlib.h>
5 //#define OMPI_SKIP_MPICXX 1 //Added in the CMakeList.txt file
6 #include <mpi.h>
7 #include <math.h>
8 #include <sys/stat.h>
9 #include <sys/types.h>
10 #include "phastaIO.h"
11
12 inline int
cscompare(const char teststring[],const char targetstring[])13 cscompare( const char teststring[],
14 const char targetstring[] )
15 {
16 char* s1 = const_cast<char*>(teststring);
17 char* s2 = const_cast<char*>(targetstring);
18
19 while( *s1 == ' ') s1++;
20 while( *s2 == ' ') s2++;
21 while( ( *s1 )
22 && ( *s2 )
23 && ( *s2 != '?')
24 && ( tolower( *s1 )==tolower( *s2 ) ) ) {
25 s1++;
26 s2++;
27 while( *s1 == ' ') s1++;
28 while( *s2 == ' ') s2++;
29 }
30 if ( !( *s1 ) || ( *s1 == '?') ) return 1;
31 else return 0;
32 }
33
main(int argc,char * argv[])34 int main(int argc, char *argv[]) {
35
36 MPI_Init(&argc,&argv);
37
38 int myrank, N_procs;
39 MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
40 MPI_Comm_size(MPI_COMM_WORLD, &N_procs);
41
42 FILE * pFile;
43 char target[1024], pool[256];
44 char * temp, * token;
45 int i, j, k, N_restart_integer, N_restart_double;
46 int N_geombc_double, N_geombc_integer;
47 int N_steps, N_parts, N_files;
48
49 pFile = fopen("./IO.N2O.input","r");
50 if (pFile == NULL)
51 printf("Error openning\n");
52
53 fgets( target, 1024, pFile );
54 token = strtok ( target, ";" );strcpy(pool,token);
55 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
56 N_geombc_double = atoi(temp);
57
58 fgets( target, 1024, pFile );
59 token = strtok ( target, ";" );strcpy(pool,token);
60 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
61 N_geombc_integer = atoi(temp);
62
63 fgets( target, 1024, pFile );
64 token = strtok ( target, ";" );strcpy(pool,token);
65 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
66 N_restart_double = atoi(temp);
67
68 fgets( target, 1024, pFile );
69 token = strtok ( target, ";" );strcpy(pool,token);
70 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
71 N_restart_integer = atoi(temp);
72
73 fgets( target, 1024, pFile );
74 token = strtok ( target, ";" );strcpy(pool,token);
75 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
76 N_steps = atoi(temp);
77
78 fgets( target, 1024, pFile );
79 token = strtok ( target, ";" );strcpy(pool,token);
80 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
81 N_parts = atoi(temp);
82
83 if(myrank==0){
84 printf("numpe is %d and start is %d\n",N_parts,N_steps);
85 }
86
87 fgets( target, 1024, pFile );
88 token = strtok ( target, ";" );strcpy(pool,token);
89 temp = strtok ( pool, ":" );temp = strtok ( NULL, ":" );
90 N_files = atoi(temp);
91
92 double ***Dfield; int ***Ifield;
93 int ***paraD, ***paraI, *expectD, *expectI;
94 char **fieldNameD, **fileTypeD, **dataTypeD, **headerTypeD;
95 char **fieldNameI, **fileTypeI, **dataTypeI, **headerTypeI;
96
97 int* WriteLockD = new int[N_restart_double];
98 int* WriteLockI = new int[N_restart_integer];
99
100 int nppp = N_parts/N_procs;
101 int startpart = myrank * nppp +1;
102 int endpart = startpart + nppp - 1;
103 char gfname[64], numTemp[128];
104 int iarray[10], igeom, isize;
105
106
107 if (N_parts != N_procs) {
108 printf("Input error: number of parts should be equal to the number of procs!\n");
109 printf("Please modify the IO.O2N.input file!\n");
110 return 0;
111 }
112
113
114
115 ///////////////////// reading ///////////////////////////////
116
117 int nppf = N_parts/N_files;
118 int N_geombc = N_geombc_double + N_geombc_integer;
119 int readHandle, GPID;
120 char fname[255],fieldtag[255];
121
122 int irestart;
123
124 Dfield = new double**[N_restart_double];
125 Ifield = new int**[N_restart_integer];
126
127 paraD = new int**[N_restart_double];
128 paraI = new int**[N_restart_integer];
129
130 expectD = new int[N_restart_double];
131 expectI = new int[N_restart_integer];
132
133 fieldNameD = new char*[N_restart_double];
134 fileTypeD = new char*[N_restart_double];
135 dataTypeD = new char*[N_restart_double];
136 headerTypeD = new char*[N_restart_double];
137
138 fieldNameI = new char*[N_restart_integer];
139 fileTypeI = new char*[N_restart_integer];
140 dataTypeI = new char*[N_restart_integer];
141 headerTypeI = new char*[N_restart_integer];
142
143 if (N_restart_double>0)
144 for ( i = 0; i < N_restart_double; i++ )
145 {
146 WriteLockD[i]=0;
147 Dfield[i] = new double*[nppp];
148
149 paraD[i] = new int*[nppp];
150
151 fieldNameD[i] = new char[128];
152 fileTypeD[i] = new char[128];
153 dataTypeD[i] = new char[128];
154 headerTypeD[i] = new char[128];
155 }
156
157 if (N_restart_integer>0)
158 for ( i = 0; i < N_restart_integer; i++ )
159 {
160 WriteLockI[i]=0;
161 Ifield[i] = new int*[nppp];
162
163 paraI[i] = new int*[nppp];
164
165 fieldNameI[i] = new char[128];
166 fileTypeI[i] = new char[128];
167 dataTypeI[i] = new char[128];
168 headerTypeI[i] = new char[128];
169 }
170
171
172 ///////////////////// reading ///////////////////////////////
173
174 int N_restart = N_restart_double + N_restart_integer;
175 int readHandle1;
176
177 bzero((void*)fname,255);
178 sprintf(fname,"./%d-procs_case/restart-dat.%d.%d",N_parts,N_steps,((int)(myrank/(N_procs/N_files))+1));
179
180 //if(myrank==0){
181 // printf("Myrank is %d - Filename is %s \n",myrank,fname);
182 //}
183
184 int nfields;
185 queryphmpiio(fname, &nfields, &nppf);
186 //initphmpiio(&N_restart, &nppf, &N_files,&readHandle1, "write") ;//WRONG
187 initphmpiio(&nfields, &nppf, &N_files, &readHandle1, "read");
188 openfile(fname, "read", &readHandle1);
189
190 for ( i = 0; i < N_restart_double; i++ )
191 {
192 fgets( target, 1024, pFile );
193 temp = strtok( target, ";" );
194 token = strtok( temp, "," );
195 strcpy( fileTypeD[i], token );
196 token = strtok ( NULL, "," );
197 strcpy( fieldNameD[i], token );
198 token = strtok ( NULL, "," );
199 strcpy( dataTypeD[i], token );
200 token = strtok ( NULL, "," );
201 strcpy( headerTypeD[i], token );
202 token = strtok ( NULL, "," );
203 strcpy( numTemp, token );
204 expectD[i] = atoi (numTemp);
205
206 for ( j = 0; j < nppp; j++ )
207 {
208 paraD[i][j] = new int[expectD[i]];
209
210 for ( k = 0; k < 10; k++ )
211 iarray[k]=0;
212
213 GPID = startpart + j;
214 bzero((void*)fieldtag,255);
215 sprintf(fieldtag,"%s@%d",fieldNameD[i],GPID);
216
217 //printf("myrank %d - filedtag %s\n",myrank,fieldtag);
218
219 iarray[0]=-1;
220 readheader( &readHandle1,
221 fieldtag,
222 (void*)iarray,
223 &expectD[i],
224 "double",
225 "binary" );
226
227 if ( iarray[0]==-1 )
228 WriteLockD[i]=1;
229 if ( WriteLockD[i]==0 )
230 {
231 for ( k = 0; k < expectD[i]; k++ )
232 paraD[i][j][k] = iarray[k];
233
234 if ( cscompare("block",headerTypeD[i]) )
235 {
236 if ( expectD[i]==1)
237 isize = paraD[i][j][0];
238 else
239 isize = paraD[i][j][0] * paraD[i][j][1];
240
241 Dfield[i][j] = new double[isize];
242 readdatablock( &readHandle1,
243 fieldtag,
244 (void*)Dfield[i][j],
245 &isize,
246 "double",
247 "binary" );
248 }
249
250 }
251 }
252 }
253
254 for ( i = 0; i < N_restart_integer; i++ )
255 {
256 fgets( target, 1024, pFile );
257 temp = strtok( target, ";" );
258 token = strtok( temp, "," );
259 strcpy( fileTypeI[i], token );
260 token = strtok ( NULL, "," );
261 strcpy( fieldNameI[i], token );
262 token = strtok ( NULL, "," );
263 strcpy( dataTypeI[i], token );
264 token = strtok ( NULL, "," );
265 strcpy( headerTypeI[i], token );
266 token = strtok ( NULL, "," );
267 strcpy( numTemp, token );
268 expectI[i] = atoi (numTemp);
269
270 for ( j = 0; j < nppp; j++ )
271 {
272 paraI[i][j] = new int[expectI[i]];
273
274 for ( k = 0; k < 10; k++ )
275 iarray[k]=0;
276
277 GPID = startpart + j;
278 bzero((void*)fieldtag,255);
279 sprintf(fieldtag,"%s@%d",fieldNameI[i],GPID);
280 iarray[0]=-1;
281
282 //printf("Rank %d, fieldname is %s \n",myrank,fieldtag);
283
284 readheader( &readHandle1,
285 fieldtag,
286 (void*)iarray,
287 &expectI[i],
288 "integer",
289 "binary" );
290
291 if ( iarray[0]==-1)
292 WriteLockI[i]=1;
293 if ( WriteLockI[i]==0 )
294 {
295 for ( k = 0; k < expectI[i]; k++ )
296 paraI[i][j][k] = iarray[k];
297
298 if ( cscompare("block",headerTypeI[i]) )
299 {
300 if ( expectI[i]==1)
301 isize = paraI[i][j][0];
302 else
303 isize = paraI[i][j][0] * paraI[i][j][1];
304
305 Ifield[i][j] = new int[isize];
306 readdatablock( &readHandle1,
307 fieldtag,
308 (void*)Ifield[i][j],
309 &isize,
310 "integer",
311 "binary" );
312 }
313 }
314 }
315
316 }
317
318 closefile(&readHandle1, "write");
319 finalizephmpiio(&readHandle1);
320
321 //////////////////////////writing////////////////////////////
322
323 int irstou;
324 int magic_number = 362436;
325 int* mptr = &magic_number;
326 int nitems = 1;
327
328 //MR CHANGE
329 bzero((void*)fname,255);
330 sprintf(fname,"./%d-procs_case-1PPP",N_parts);
331 if(0<mkdir(fname,0777)) { printf("ERROR - Could not create procs_case-1PPP directory\n"); return 1; }
332 //MR CHANGE END
333
334 bzero((void*)fname,255);
335 sprintf(fname,"./%d-procs_case-1PPP/restart.%d.%d",N_parts,N_steps,myrank+1);
336 openfile(fname,"write", &irstou);
337
338 /* writing the top ascii header for the restart file */
339
340 writestring( &irstou,"# PHASTA Input File Version 2.0\n");
341 writestring( &irstou,
342 "# format \"keyphrase : sizeofnextblock usual headers\"\n");
343
344 bzero( (void*)fname, 255 );
345 writestring( &irstou, fname );
346
347 writestring( &irstou, fname );
348 writestring( &irstou,"\n");
349
350
351 isize = 1;
352 nitems = 1;
353 iarray[ 0 ] = 1;
354 writeheader( &irstou, "byteorder magic number ",
355 (void*)iarray, &nitems, &isize, "integer", "binary" );
356
357 nitems = 1;
358 writedatablock( &irstou, "byteorder magic number ",
359 (void*)mptr, &nitems, "integer", "binary" );
360
361 for ( i = 0; i < N_restart_double; i++ )
362 {
363 for ( j = 0; j < nppp; j++ )
364 {
365 if ( WriteLockD[i] == 0 )
366 {
367 if ( cscompare("header",headerTypeD[i]) )
368 {
369 bzero( (void*)fname, 255 );
370 sprintf(fname,"%s : < 0 > %d\n", fieldNameD[i],paraD[i][j][0]);
371 writestring( &irstou, fname );
372 }
373
374 if ( cscompare("block",headerTypeD[i]) )
375 {
376 if ( expectD[i]==1 )
377 isize = paraD[i][j][0];
378 else
379 isize = paraD[i][j][0] * paraD[i][j][1];
380
381 for ( k = 0; k < expectD[i]; k++ )
382 iarray[k] = paraD[i][j][k];
383
384 if ( cscompare("header",headerTypeD[i]) )
385 isize = 0;
386
387 writeheader( &irstou,
388 fieldNameD[i],
389 (void*)iarray,
390 &expectD[i],
391 &isize,
392 "double",
393 "binary");
394 writedatablock( &irstou,
395 fieldNameD[i],
396 (void*)Dfield[i][j],
397 &isize,
398 "double",
399 "binary");
400 }
401
402
403 if ( cscompare("block",headerTypeD[i]) )
404 delete [] Dfield[i][j];
405 }
406 delete [] paraD[i][j];
407 }
408 }
409
410 for ( i = 0; i < N_restart_integer; i++ )
411 {
412 for ( j = 0; j < nppp; j++ )
413 {
414
415 if ( WriteLockI[i] == 0 )
416 {
417
418 if ( cscompare("header",headerTypeI[i]) )
419 {
420 bzero( (void*)fname, 255 );
421 sprintf(fname,"%s : < 0 > %d\n", fieldNameI[i],paraI[i][j][0]);
422 writestring( &irstou, fname );
423 }
424
425 if ( cscompare("block",headerTypeI[i]) )
426 {
427 if ( expectI[i]==1 )
428 isize = paraI[i][j][0];
429 else
430 isize = paraI[i][j][0] * paraI[i][j][1];
431
432 for ( k = 0; k < expectI[i]; k++ )
433 iarray[k] = paraI[i][j][k];
434
435 writeheader( &irstou,
436 fieldNameI[i],
437 (void*)iarray,
438 &expectI[i],
439 &isize,
440 "integer",
441 "binary");
442 writedatablock( &irstou,
443 fieldNameI[i],
444 (void*)Ifield[i][j],
445 &isize,
446 "integer",
447 "binary");
448 }
449
450 if ( cscompare("block",headerTypeI[i]) )
451 delete [] Ifield[i][j];
452 }
453 delete [] paraI[i][j];
454 }
455 }
456
457
458 closefile( &irstou, "write" );
459 MPI_Barrier(MPI_COMM_WORLD);
460
461
462 if (N_restart_double>0)
463 for ( i = 0; i < N_restart_double; i++ )
464 {
465 delete [] Dfield[i];
466 delete [] paraD[i];
467
468 delete [] fieldNameD[i];
469 delete [] fileTypeD[i];
470 delete [] dataTypeD[i];
471 delete [] headerTypeD[i];
472 }
473
474 if (N_restart_integer>0)
475 for ( i = 0; i < N_restart_integer; i++ )
476 {
477 delete [] Ifield[i];
478 delete [] paraI[i];
479
480 delete [] fieldNameI[i];
481 delete [] fileTypeI[i];
482 delete [] dataTypeI[i];
483 delete [] headerTypeI[i];
484 }
485
486 delete [] Dfield;
487 delete [] Ifield;
488
489 delete [] paraD;
490 delete [] paraI;
491
492 delete [] expectD;
493 delete [] expectI;
494
495 delete [] fieldNameD;
496 delete [] fileTypeD;
497 delete [] dataTypeD;
498 delete [] headerTypeD;
499
500 delete [] fieldNameI;
501 delete [] fileTypeI;
502 delete [] dataTypeI;
503 delete [] headerTypeI;
504
505 delete [] WriteLockD;
506 delete [] WriteLockI;
507
508 fclose(pFile);
509
510 if (myrank==0)
511 {
512 printf("\nFinished transfer, please check data using:\n");
513 printf(" grep -a ': <' filename \n\n");
514 }
515
516 MPI_Finalize();
517
518 }
519
520
521