xref: /petsc/src/sys/objects/pinit.c (revision bd53e248da711e076a5fe36f00f077ea0c544e34)
1 
2 /*
3    This file defines the initialization of PETSc, including PetscInitialize()
4 */
5 #define PETSC_DESIRE_COMPLEX
6 #include <petsc-private/petscimpl.h>        /*I  "petscsys.h"   I*/
7 #include <petscviewer.h>
8 
9 #if defined(PETSC_HAVE_CUDA)
10 #include <cublas.h>
11 #endif
12 
13 #include <petscthreadcomm.h>
14 
15 #if defined(PETSC_USE_LOG)
16 extern PetscErrorCode PetscLogBegin_Private(void);
17 #endif
18 extern PetscBool PetscHMPIWorker;
19 
20 #if defined(PETSC_SERIALIZE_FUNCTIONS)
21 PetscFPT PetscFPTData = 0;
22 #endif
23 
24 #if defined(PETSC_HAVE_SAWS)
25 #include <petscviewersaws.h>
26 #endif
27 /* -----------------------------------------------------------------------------------------*/
28 
29 extern FILE *petsc_history;
30 
31 extern PetscErrorCode PetscInitialize_DynamicLibraries(void);
32 extern PetscErrorCode PetscFinalize_DynamicLibraries(void);
33 extern PetscErrorCode PetscFunctionListPrintAll(void);
34 extern PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm,int);
35 extern PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm,int);
36 extern PetscErrorCode PetscCloseHistoryFile(FILE**);
37 
38 /* user may set this BEFORE calling PetscInitialize() */
39 MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL;
40 
41 PetscMPIInt Petsc_Counter_keyval   = MPI_KEYVAL_INVALID;
42 PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID;
43 PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID;
44 
45 /*
46      Declare and set all the string names of the PETSc enums
47 */
48 const char *const PetscBools[]     = {"FALSE","TRUE","PetscBool","PETSC_",0};
49 const char *const PetscCopyModes[] = {"COPY_VALUES","OWN_POINTER","USE_POINTER","PetscCopyMode","PETSC_",0};
50 const char *const PetscDataTypes[] = {"INT","DOUBLE","COMPLEX","LONG","SHORT","FLOAT",
51                                       "CHAR","LOGICAL","ENUM","BOOL","LONGDOUBLE","OBJECT","FUNCTION","PetscDataType","PETSC_",0};
52 
53 PetscBool PetscPreLoadingUsed = PETSC_FALSE;
54 PetscBool PetscPreLoadingOn   = PETSC_FALSE;
55 
56 /* pthread_key for PetscStack */
57 #if defined(PETSC_HAVE_PTHREADCLASSES) && !defined(PETSC_PTHREAD_LOCAL)
58 pthread_key_t petscstack;
59 #endif
60 
61 /*
62        Checks the options database for initializations related to the
63     PETSc components
64 */
65 #undef __FUNCT__
66 #define __FUNCT__ "PetscOptionsCheckInitial_Components"
67 PetscErrorCode  PetscOptionsCheckInitial_Components(void)
68 {
69   PetscBool      flg1;
70   PetscErrorCode ierr;
71 
72   PetscFunctionBegin;
73   ierr = PetscOptionsHasName(NULL,"-help",&flg1);CHKERRQ(ierr);
74   if (flg1) {
75 #if defined(PETSC_USE_LOG)
76     MPI_Comm comm = PETSC_COMM_WORLD;
77     ierr = (*PetscHelpPrintf)(comm,"------Additional PETSc component options--------\n");CHKERRQ(ierr);
78     ierr = (*PetscHelpPrintf)(comm," -log_summary_exclude: <vec,mat,pc.ksp,snes>\n");CHKERRQ(ierr);
79     ierr = (*PetscHelpPrintf)(comm," -info_exclude: <null,vec,mat,pc,ksp,snes,ts>\n");CHKERRQ(ierr);
80     ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr);
81 #endif
82   }
83   PetscFunctionReturn(0);
84 }
85 
86 #undef __FUNCT__
87 #define __FUNCT__ "PetscInitializeNoPointers"
88 /*
89       PetscInitializeNoPointers - Calls PetscInitialize() from C/C++ without the pointers to argc and args
90 
91    Collective
92 
93    Level: advanced
94 
95     Notes: this is called only by the PETSc MATLAB and Julia interface. Even though it might start MPI it sets the flag to
96      indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to
97      be called multiple times from MATLAB and Julia without the problem of trying to initialize MPI more than once.
98 
99      Turns off PETSc signal handling because that can interact with MATLAB's signal handling causing random crashes.
100 
101 .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments()
102 */
103 PetscErrorCode  PetscInitializeNoPointers(int argc,char **args,const char *filename,const char *help)
104 {
105   PetscErrorCode ierr;
106   int            myargc   = argc;
107   char           **myargs = args;
108 
109   PetscFunctionBegin;
110   ierr = PetscInitialize(&myargc,&myargs,filename,help);CHKERRQ(ierr);
111   ierr = PetscPopSignalHandler();CHKERRQ(ierr);
112   PetscBeganMPI = PETSC_FALSE;
113   PetscFunctionReturn(ierr);
114 }
115 
116 #undef __FUNCT__
117 #define __FUNCT__ "PetscGetPETSC_COMM_SELF"
118 /*
119       Used by MATLAB and Julia interface to get communicator
120 */
121 PetscErrorCode  PetscGetPETSC_COMM_SELF(MPI_Comm *comm)
122 {
123   PetscFunctionBegin;
124   *comm = PETSC_COMM_SELF;
125   PetscFunctionReturn(0);
126 }
127 
128 #undef __FUNCT__
129 #define __FUNCT__ "PetscInitializeNoArguments"
130 /*@C
131       PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without
132         the command line arguments.
133 
134    Collective
135 
136    Level: advanced
137 
138 .seealso: PetscInitialize(), PetscInitializeFortran()
139 @*/
140 PetscErrorCode  PetscInitializeNoArguments(void)
141 {
142   PetscErrorCode ierr;
143   int            argc   = 0;
144   char           **args = 0;
145 
146   PetscFunctionBegin;
147   ierr = PetscInitialize(&argc,&args,NULL,NULL);
148   PetscFunctionReturn(ierr);
149 }
150 
151 #undef __FUNCT__
152 #define __FUNCT__ "PetscInitialized"
153 /*@
154       PetscInitialized - Determine whether PETSc is initialized.
155 
156    Level: beginner
157 
158 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
159 @*/
160 PetscErrorCode PetscInitialized(PetscBool  *isInitialized)
161 {
162   *isInitialized = PetscInitializeCalled;
163   return 0;
164 }
165 
166 #undef __FUNCT__
167 #define __FUNCT__ "PetscFinalized"
168 /*@
169       PetscFinalized - Determine whether PetscFinalize() has been called yet
170 
171    Level: developer
172 
173 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
174 @*/
175 PetscErrorCode  PetscFinalized(PetscBool  *isFinalized)
176 {
177   *isFinalized = PetscFinalizeCalled;
178   return 0;
179 }
180 
181 extern PetscErrorCode PetscOptionsCheckInitial_Private(void);
182 
183 /*
184        This function is the MPI reduction operation used to compute the sum of the
185    first half of the datatype and the max of the second half.
186 */
187 MPI_Op PetscMaxSum_Op = 0;
188 
189 #undef __FUNCT__
190 #define __FUNCT__ "PetscMaxSum_Local"
191 PETSC_EXTERN void MPIAPI PetscMaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype)
192 {
193   PetscInt *xin = (PetscInt*)in,*xout = (PetscInt*)out,i,count = *cnt;
194 
195   PetscFunctionBegin;
196   if (*datatype != MPIU_2INT) {
197     (*PetscErrorPrintf)("Can only handle MPIU_2INT data types");
198     MPI_Abort(MPI_COMM_WORLD,1);
199   }
200 
201   for (i=0; i<count; i++) {
202     xout[2*i]    = PetscMax(xout[2*i],xin[2*i]);
203     xout[2*i+1] += xin[2*i+1];
204   }
205   PetscFunctionReturnVoid();
206 }
207 
208 /*
209     Returns the max of the first entry owned by this processor and the
210 sum of the second entry.
211 
212     The reason nprocs[2*i] contains lengths nprocs[2*i+1] contains flag of 1 if length is nonzero
213 is so that the PetscMaxSum_Op() can set TWO values, if we passed in only nprocs[i] with lengths
214 there would be no place to store the both needed results.
215 */
216 #undef __FUNCT__
217 #define __FUNCT__ "PetscMaxSum"
218 PetscErrorCode  PetscMaxSum(MPI_Comm comm,const PetscInt nprocs[],PetscInt *max,PetscInt *sum)
219 {
220   PetscMPIInt    size,rank;
221   struct {PetscInt max,sum;} *work;
222   PetscErrorCode ierr;
223 
224   PetscFunctionBegin;
225   ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
226   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
227   ierr = PetscMalloc(size*sizeof(*work),&work);CHKERRQ(ierr);
228   ierr = MPI_Allreduce((void*)nprocs,work,size,MPIU_2INT,PetscMaxSum_Op,comm);CHKERRQ(ierr);
229   *max = work[rank].max;
230   *sum = work[rank].sum;
231   ierr = PetscFree(work);CHKERRQ(ierr);
232   PetscFunctionReturn(0);
233 }
234 
235 /* ----------------------------------------------------------------------------*/
236 MPI_Op  PetscADMax_Op = 0;
237 
238 #undef __FUNCT__
239 #define __FUNCT__ "PetscADMax_Local"
240 PETSC_EXTERN void MPIAPI PetscADMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
241 {
242   PetscScalar *xin = (PetscScalar*)in,*xout = (PetscScalar*)out;
243   PetscInt    i,count = *cnt;
244 
245   PetscFunctionBegin;
246   if (*datatype != MPIU_2SCALAR) {
247     (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types");
248     MPI_Abort(MPI_COMM_WORLD,1);
249   }
250 
251   for (i=0; i<count; i++) {
252     if (PetscRealPart(xout[2*i]) < PetscRealPart(xin[2*i])) {
253       xout[2*i]   = xin[2*i];
254       xout[2*i+1] = xin[2*i+1];
255     }
256   }
257   PetscFunctionReturnVoid();
258 }
259 
260 MPI_Op PetscADMin_Op = 0;
261 
262 #undef __FUNCT__
263 #define __FUNCT__ "PetscADMin_Local"
264 PETSC_EXTERN void MPIAPI PetscADMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
265 {
266   PetscScalar *xin = (PetscScalar*)in,*xout = (PetscScalar*)out;
267   PetscInt    i,count = *cnt;
268 
269   PetscFunctionBegin;
270   if (*datatype != MPIU_2SCALAR) {
271     (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types");
272     MPI_Abort(MPI_COMM_WORLD,1);
273   }
274 
275   for (i=0; i<count; i++) {
276     if (PetscRealPart(xout[2*i]) > PetscRealPart(xin[2*i])) {
277       xout[2*i]   = xin[2*i];
278       xout[2*i+1] = xin[2*i+1];
279     }
280   }
281   PetscFunctionReturnVoid();
282 }
283 /* ---------------------------------------------------------------------------------------*/
284 
285 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
286 MPI_Op MPIU_SUM = 0;
287 
288 #undef __FUNCT__
289 #define __FUNCT__ "PetscSum_Local"
290 PETSC_EXTERN void PetscSum_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
291 {
292   PetscInt i,count = *cnt;
293 
294   PetscFunctionBegin;
295   if (*datatype == MPIU_REAL) {
296     PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out;
297     for (i=0; i<count; i++) xout[i] += xin[i];
298   }
299 #if defined(PETSC_HAVE_COMPLEX)
300   else if (*datatype == MPIU_COMPLEX) {
301     PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out;
302     for (i=0; i<count; i++) xout[i] += xin[i];
303   }
304 #endif
305   else {
306     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types");
307     MPI_Abort(MPI_COMM_WORLD,1);
308   }
309   PetscFunctionReturnVoid();
310 }
311 #endif
312 
313 #if defined(PETSC_USE_REAL___FLOAT128)
314 MPI_Op MPIU_MAX = 0;
315 MPI_Op MPIU_MIN = 0;
316 
317 #undef __FUNCT__
318 #define __FUNCT__ "PetscMax_Local"
319 PETSC_EXTERN void PetscMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
320 {
321   PetscInt i,count = *cnt;
322 
323   PetscFunctionBegin;
324   if (*datatype == MPIU_REAL) {
325     PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out;
326     for (i=0; i<count; i++) xout[i] = PetscMax(xout[i],xin[i]);
327   }
328 #if defined(PETSC_HAVE_COMPLEX)
329   else if (*datatype == MPIU_COMPLEX) {
330     PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out;
331     for (i=0; i<count; i++) {
332       xout[i] = PetscRealPartComplex(xout[i])<PetscRealPartComplex(xin[i]) ? xin[i] : xout[i];
333     }
334   }
335 #endif
336   else {
337     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types");
338     MPI_Abort(MPI_COMM_WORLD,1);
339   }
340   PetscFunctionReturnVoid();
341 }
342 
343 #undef __FUNCT__
344 #define __FUNCT__ "PetscMin_Local"
345 PETSC_EXTERN void PetscMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
346 {
347   PetscInt    i,count = *cnt;
348 
349   PetscFunctionBegin;
350   if (*datatype == MPIU_REAL) {
351     PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out;
352     for (i=0; i<count; i++) xout[i] = PetscMin(xout[i],xin[i]);
353   }
354 #if defined(PETSC_HAVE_COMPLEX)
355   else if (*datatype == MPIU_COMPLEX) {
356     PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out;
357     for (i=0; i<count; i++) {
358       xout[i] = PetscRealPartComplex(xout[i])>PetscRealPartComplex(xin[i]) ? xin[i] : xout[i];
359     }
360   }
361 #endif
362   else {
363     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types");
364     MPI_Abort(MPI_COMM_WORLD,1);
365   }
366   PetscFunctionReturnVoid();
367 }
368 #endif
369 
370 #undef __FUNCT__
371 #define __FUNCT__ "Petsc_DelCounter"
372 /*
373    Private routine to delete internal tag/name counter storage when a communicator is freed.
374 
375    This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this  data as an attribute is freed.
376 
377    Note: this is declared extern "C" because it is passed to MPI_Keyval_create()
378 
379 */
380 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelCounter(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
381 {
382   PetscErrorCode ierr;
383 
384   PetscFunctionBegin;
385   ierr = PetscInfo1(0,"Deleting counter data in an MPI_Comm %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
386   ierr = PetscFree(count_val);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
387   PetscFunctionReturn(MPI_SUCCESS);
388 }
389 
390 #undef __FUNCT__
391 #define __FUNCT__ "Petsc_DelComm_Outer"
392 /*
393   This is invoked on the outer comm as a result of either PetscCommDestroy() (via MPI_Attr_delete) or when the user
394   calls MPI_Comm_free().
395 
396   This is the only entry point for breaking the links between inner and outer comms.
397 
398   This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator.
399 
400   Note: this is declared extern "C" because it is passed to MPI_Keyval_create()
401 
402 */
403 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelComm_Outer(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state)
404 {
405   PetscErrorCode ierr;
406   PetscMPIInt    flg;
407   union {MPI_Comm comm; void *ptr;} icomm,ocomm;
408 
409   PetscFunctionBegin;
410   if (keyval != Petsc_InnerComm_keyval) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Unexpected keyval");
411   icomm.ptr = attr_val;
412 
413   ierr = MPI_Attr_get(icomm.comm,Petsc_OuterComm_keyval,&ocomm,&flg);CHKERRQ(ierr);
414   if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected reference to outer comm");
415   if (ocomm.comm != comm) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm has reference to non-matching outer comm");
416   ierr = MPI_Attr_delete(icomm.comm,Petsc_OuterComm_keyval);CHKERRQ(ierr); /* Calls Petsc_DelComm_Inner */
417   ierr = PetscInfo1(0,"User MPI_Comm %ld is being freed after removing reference from inner PETSc comm to this outer comm\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
418   PetscFunctionReturn(MPI_SUCCESS);
419 }
420 
421 #undef __FUNCT__
422 #define __FUNCT__ "Petsc_DelComm_Inner"
423 /*
424  * This is invoked on the inner comm when Petsc_DelComm_Outer calls MPI_Attr_delete.  It should not be reached any other way.
425  */
426 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelComm_Inner(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state)
427 {
428   PetscErrorCode ierr;
429 
430   PetscFunctionBegin;
431   ierr = PetscInfo1(0,"Removing reference to PETSc communicator embedded in a user MPI_Comm %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr);
432   PetscFunctionReturn(MPI_SUCCESS);
433 }
434 
435 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
436 #if !defined(PETSC_WORDS_BIGENDIAN)
437 PETSC_EXTERN PetscMPIInt PetscDataRep_extent_fn(MPI_Datatype,MPI_Aint*,void*);
438 PETSC_EXTERN PetscMPIInt PetscDataRep_read_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*);
439 PETSC_EXTERN PetscMPIInt PetscDataRep_write_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*);
440 #endif
441 #endif
442 
443 int  PetscGlobalArgc   = 0;
444 char **PetscGlobalArgs = 0;
445 PetscSegBuffer PetscCitationsList;
446 
447 #undef __FUNCT__
448 #define __FUNCT__ "PetscCitationsInitialize"
449 PetscErrorCode PetscCitationsInitialize()
450 {
451   PetscErrorCode ierr;
452 
453   PetscFunctionBegin;
454   ierr = PetscSegBufferCreate(1,10000,&PetscCitationsList);CHKERRQ(ierr);
455   ierr = PetscCitationsRegister("@TechReport{petsc-user-ref,\n  Author = {Satish Balay and Jed Brown and and Kris Buschelman and Victor Eijkhout\n            and William D.  Gropp and Dinesh Kaushik and Matthew G. Knepley\n            and Lois Curfman McInnes and Barry F. Smith and Hong Zhang},\n  Title = {{PETS}c Users Manual},\n  Number = {ANL-95/11 - Revision 3.4},\n  Institution = {Argonne National Laboratory},\n  Year = {2013}\n}\n",NULL);CHKERRQ(ierr);
456   ierr = PetscCitationsRegister("@InProceedings{petsc-efficient,\n  Author = {Satish Balay and William D. Gropp and Lois Curfman McInnes and Barry F. Smith},\n  Title = {Efficient Management of Parallelism in Object Oriented Numerical Software Libraries},\n  Booktitle = {Modern Software Tools in Scientific Computing},\n  Editor = {E. Arge and A. M. Bruaset and H. P. Langtangen},\n  Pages = {163--202},\n  Publisher = {Birkh{\\\"{a}}user Press},\n  Year = {1997}\n}\n",NULL);CHKERRQ(ierr);
457   PetscFunctionReturn(0);
458 }
459 
460 #undef __FUNCT__
461 #define __FUNCT__ "PetscGetArgs"
462 /*@C
463    PetscGetArgs - Allows you to access the raw command line arguments anywhere
464      after PetscInitialize() is called but before PetscFinalize().
465 
466    Not Collective
467 
468    Output Parameters:
469 +  argc - count of number of command line arguments
470 -  args - the command line arguments
471 
472    Level: intermediate
473 
474    Notes:
475       This is usually used to pass the command line arguments into other libraries
476    that are called internally deep in PETSc or the application.
477 
478       The first argument contains the program name as is normal for C arguments.
479 
480    Concepts: command line arguments
481 
482 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArguments()
483 
484 @*/
485 PetscErrorCode  PetscGetArgs(int *argc,char ***args)
486 {
487   PetscFunctionBegin;
488   if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
489   *argc = PetscGlobalArgc;
490   *args = PetscGlobalArgs;
491   PetscFunctionReturn(0);
492 }
493 
494 #undef __FUNCT__
495 #define __FUNCT__ "PetscGetArguments"
496 /*@C
497    PetscGetArguments - Allows you to access the  command line arguments anywhere
498      after PetscInitialize() is called but before PetscFinalize().
499 
500    Not Collective
501 
502    Output Parameters:
503 .  args - the command line arguments
504 
505    Level: intermediate
506 
507    Notes:
508       This does NOT start with the program name and IS null terminated (final arg is void)
509 
510    Concepts: command line arguments
511 
512 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscFreeArguments()
513 
514 @*/
515 PetscErrorCode  PetscGetArguments(char ***args)
516 {
517   PetscInt       i,argc = PetscGlobalArgc;
518   PetscErrorCode ierr;
519 
520   PetscFunctionBegin;
521   if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
522   if (!argc) {*args = 0; PetscFunctionReturn(0);}
523   ierr = PetscMalloc(argc*sizeof(char*),args);CHKERRQ(ierr);
524   for (i=0; i<argc-1; i++) {
525     ierr = PetscStrallocpy(PetscGlobalArgs[i+1],&(*args)[i]);CHKERRQ(ierr);
526   }
527   (*args)[argc-1] = 0;
528   PetscFunctionReturn(0);
529 }
530 
531 #undef __FUNCT__
532 #define __FUNCT__ "PetscFreeArguments"
533 /*@C
534    PetscFreeArguments - Frees the memory obtained with PetscGetArguments()
535 
536    Not Collective
537 
538    Output Parameters:
539 .  args - the command line arguments
540 
541    Level: intermediate
542 
543    Concepts: command line arguments
544 
545 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscGetArguments()
546 
547 @*/
548 PetscErrorCode  PetscFreeArguments(char **args)
549 {
550   PetscInt       i = 0;
551   PetscErrorCode ierr;
552 
553   PetscFunctionBegin;
554   if (!args) PetscFunctionReturn(0);
555   while (args[i]) {
556     ierr = PetscFree(args[i]);CHKERRQ(ierr);
557     i++;
558   }
559   ierr = PetscFree(args);CHKERRQ(ierr);
560   PetscFunctionReturn(0);
561 }
562 
563 #undef __FUNCT__
564 #define __FUNCT__ "PetscInitialize"
565 /*@C
566    PetscInitialize - Initializes the PETSc database and MPI.
567    PetscInitialize() calls MPI_Init() if that has yet to be called,
568    so this routine should always be called near the beginning of
569    your program -- usually the very first line!
570 
571    Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set
572 
573    Input Parameters:
574 +  argc - count of number of command line arguments
575 .  args - the command line arguments
576 .  file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL to not check for
577           code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
578 -  help - [optional] Help message to print, use NULL for no message
579 
580    If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that
581    communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a
582    four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not,
583    then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even
584    if different subcommunicators of the job are doing different things with PETSc.
585 
586    Options Database Keys:
587 +  -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
588 .  -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
589 .  -on_error_emacs <machinename> causes emacsclient to jump to error file
590 .  -on_error_abort calls abort() when error detected (no traceback)
591 .  -on_error_mpiabort calls MPI_abort() when error detected
592 .  -error_output_stderr prints error messages to stderr instead of the default stdout
593 .  -error_output_none does not print the error messages (but handles errors in the same way as if this was not called)
594 .  -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger
595 .  -debugger_pause [sleeptime] (in seconds) - Pauses debugger
596 .  -stop_for_debugger - Print message on how to attach debugger manually to
597                         process and wait (-debugger_pause) seconds for attachment
598 .  -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries)
599 .  -malloc no - Indicates not to use error-checking malloc
600 .  -malloc_debug - check for memory corruption at EVERY malloc or free
601 .  -malloc_dump - prints a list of all unfreed memory at the end of the run
602 .  -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds
603 .  -fp_trap - Stops on floating point exceptions (Note that on the
604               IBM RS6000 this slows code by at least a factor of 10.)
605 .  -no_signal_handler - Indicates not to trap error signals
606 .  -shared_tmp - indicates /tmp directory is shared by all processors
607 .  -not_shared_tmp - each processor has own /tmp
608 .  -tmp - alternative name of /tmp directory
609 .  -get_total_flops - returns total flops done by all processors
610 .  -memory_info - Print memory usage at end of run
611 -  -server <port> - start PETSc webserver (default port is 8080)
612 
613    Options Database Keys for Profiling:
614    See the <a href="../../docs/manual.pdf#nameddest=ch_profiling">profiling chapter of the users manual</a> for details.
615 +  -info <optional filename> - Prints verbose information to the screen
616 .  -info_exclude <null,vec,mat,pc,ksp,snes,ts> - Excludes some of the verbose messages
617 .  -log_sync - Log the synchronization in scatters, inner products and norms
618 .  -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program
619         hangs without running in the debugger).  See PetscLogTraceBegin().
620 .  -log_summary [filename] - Prints summary of flop and timing information to screen. If the filename is specified the
621         summary is written to the file.  See PetscLogView().
622 .  -log_summary_python [filename] - Prints data on of flop and timing usage to a file or screen. See PetscLogPrintSViewPython().
623 .  -log_all [filename] - Logs extensive profiling information  See PetscLogDump().
624 .  -log [filename] - Logs basic profiline information  See PetscLogDump().
625 -  -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution)
626 
627     Only one of -log_trace, -log_summary, -log_all, -log, or -log_mpe may be used at a time
628 
629    Environmental Variables:
630 +   PETSC_TMP - alternative tmp directory
631 .   PETSC_SHARED_TMP - tmp is shared by all processes
632 .   PETSC_NOT_SHARED_TMP - each process has its own private tmp
633 .   PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer
634 -   PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to
635 
636 
637    Level: beginner
638 
639    Notes:
640    If for some reason you must call MPI_Init() separately, call
641    it before PetscInitialize().
642 
643    Fortran Version:
644    In Fortran this routine has the format
645 $       call PetscInitialize(file,ierr)
646 
647 +   ierr - error return code
648 -  file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL_CHARACTER to not check for
649           code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
650 
651    Important Fortran Note:
652    In Fortran, you MUST use NULL_CHARACTER to indicate a
653    null character string; you CANNOT just use NULL as
654    in the C version. See the <a href="../../docs/manual.pdf">users manual</a> for details.
655 
656    If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after
657    calling PetscInitialize().
658 
659    Concepts: initializing PETSc
660 
661 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments()
662 
663 @*/
664 PetscErrorCode  PetscInitialize(int *argc,char ***args,const char file[],const char help[])
665 {
666   PetscErrorCode ierr;
667   PetscMPIInt    flag, size;
668   PetscInt       nodesize;
669   PetscBool      flg;
670   char           hostname[256];
671 
672   PetscFunctionBegin;
673   if (PetscInitializeCalled) PetscFunctionReturn(0);
674 
675   /* these must be initialized in a routine, not as a constant declaration*/
676   PETSC_STDOUT = stdout;
677   PETSC_STDERR = stderr;
678 
679   ierr = PetscOptionsCreate();CHKERRQ(ierr);
680 
681   /*
682      We initialize the program name here (before MPI_Init()) because MPICH has a bug in
683      it that it sets args[0] on all processors to be args[0] on the first processor.
684   */
685   if (argc && *argc) {
686     ierr = PetscSetProgramName(**args);CHKERRQ(ierr);
687   } else {
688     ierr = PetscSetProgramName("Unknown Name");CHKERRQ(ierr);
689   }
690 
691   ierr = MPI_Initialized(&flag);CHKERRQ(ierr);
692   if (!flag) {
693     if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first");
694 #if defined(PETSC_HAVE_MPI_INIT_THREAD)
695     {
696       PetscMPIInt provided;
697       ierr = MPI_Init_thread(argc,args,MPI_THREAD_FUNNELED,&provided);CHKERRQ(ierr);
698     }
699 #else
700     ierr = MPI_Init(argc,args);CHKERRQ(ierr);
701 #endif
702     PetscBeganMPI = PETSC_TRUE;
703   }
704   if (argc && args) {
705     PetscGlobalArgc = *argc;
706     PetscGlobalArgs = *args;
707   }
708   PetscFinalizeCalled = PETSC_FALSE;
709 
710   if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD;
711   ierr = MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);CHKERRQ(ierr);
712 
713   /* Done after init due to a bug in MPICH-GM? */
714   ierr = PetscErrorPrintfInitialize();CHKERRQ(ierr);
715 
716   ierr = MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);CHKERRQ(ierr);
717   ierr = MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);CHKERRQ(ierr);
718 
719   MPIU_BOOL = MPI_INT;
720   MPIU_ENUM = MPI_INT;
721 
722   /*
723      Initialized the global complex variable; this is because with
724      shared libraries the constructors for global variables
725      are not called; at least on IRIX.
726   */
727 #if defined(PETSC_HAVE_COMPLEX)
728   {
729 #if defined(PETSC_CLANGUAGE_CXX)
730     PetscComplex ic(0.0,1.0);
731     PETSC_i = ic;
732 #elif defined(PETSC_CLANGUAGE_C)
733     PETSC_i = _Complex_I;
734 #endif
735   }
736 
737 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
738   ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr);
739   ierr = MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr);
740   ierr = MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX);CHKERRQ(ierr);
741   ierr = MPI_Type_commit(&MPIU_C_COMPLEX);CHKERRQ(ierr);
742 #endif
743 #endif /* PETSC_HAVE_COMPLEX */
744 
745   /*
746      Create the PETSc MPI reduction operator that sums of the first
747      half of the entries and maxes the second half.
748   */
749   ierr = MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);CHKERRQ(ierr);
750 
751 #if defined(PETSC_USE_REAL___FLOAT128)
752   ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);CHKERRQ(ierr);
753   ierr = MPI_Type_commit(&MPIU___FLOAT128);CHKERRQ(ierr);
754 #if defined(PETSC_HAVE_COMPLEX)
755   ierr = MPI_Type_contiguous(4,MPI_DOUBLE,&MPIU___COMPLEX128);CHKERRQ(ierr);
756   ierr = MPI_Type_commit(&MPIU___COMPLEX128);CHKERRQ(ierr);
757 #endif
758   ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);CHKERRQ(ierr);
759   ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);CHKERRQ(ierr);
760 #endif
761 
762 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
763   ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);CHKERRQ(ierr);
764 #endif
765 
766   ierr = MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);CHKERRQ(ierr);
767   ierr = MPI_Type_commit(&MPIU_2SCALAR);CHKERRQ(ierr);
768   ierr = MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op);CHKERRQ(ierr);
769   ierr = MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op);CHKERRQ(ierr);
770 
771 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
772   ierr = MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);CHKERRQ(ierr);
773   ierr = MPI_Type_commit(&MPIU_2INT);CHKERRQ(ierr);
774 #endif
775 
776 
777   /*
778      Attributes to be set on PETSc communicators
779   */
780   ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0);CHKERRQ(ierr);
781   ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm_Outer,&Petsc_InnerComm_keyval,(void*)0);CHKERRQ(ierr);
782   ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm_Inner,&Petsc_OuterComm_keyval,(void*)0);CHKERRQ(ierr);
783 
784   /*
785      Build the options database
786   */
787   ierr = PetscOptionsInsert(argc,args,file);CHKERRQ(ierr);
788 
789 
790   /*
791      Print main application help message
792   */
793   ierr = PetscOptionsHasName(NULL,"-help",&flg);CHKERRQ(ierr);
794   if (help && flg) {
795     ierr = PetscPrintf(PETSC_COMM_WORLD,help);CHKERRQ(ierr);
796   }
797   ierr = PetscOptionsCheckInitial_Private();CHKERRQ(ierr);
798 
799   ierr = PetscCitationsInitialize();CHKERRQ(ierr);
800 
801 #if defined(PETSC_HAVE_SAWS)
802   if (!PetscGlobalRank) {
803     char        cert[PETSC_MAX_PATH_LEN],root[PETSC_MAX_PATH_LEN],intro[4024],programname[64],appline[256];
804     int         port;
805     PetscBool   rootlocal = PETSC_FALSE,flg2;
806 
807     ierr = PetscOptionsHasName(NULL,"-saws_log",&flg);CHKERRQ(ierr);
808     if (flg) {
809       char  sawslog[PETSC_MAX_PATH_LEN];
810 
811       ierr = PetscOptionsGetString(NULL,"-saws_log",sawslog,PETSC_MAX_PATH_LEN,NULL);CHKERRQ(ierr);
812       if (sawslog[0]) {
813         PetscStackCallSAWs(SAWs_Set_Use_Logfile,(sawslog));
814       } else {
815         PetscStackCallSAWs(SAWs_Set_Use_Logfile,(NULL));
816       }
817     }
818     ierr = PetscOptionsGetString(NULL,"-saws_https",cert,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
819     if (flg) {
820       PetscStackCallSAWs(SAWs_Set_Use_HTTPS,(cert));
821     }
822     ierr = PetscOptionsGetInt(NULL,"-saws_port",&port,&flg);CHKERRQ(ierr);
823     if (flg) {
824       PetscStackCallSAWs(SAWs_Set_Port,(port));
825     }
826     ierr = PetscOptionsGetString(NULL,"-saws_root",root,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
827     if (flg) {
828       PetscStackCallSAWs(SAWs_Set_Document_Root,(root));CHKERRQ(ierr);
829       ierr = PetscStrcmp(root,".",&rootlocal);CHKERRQ(ierr);
830     }
831     ierr = PetscOptionsHasName(NULL,"-saws_local",&flg2);CHKERRQ(ierr);
832     if (flg2) {
833       char jsdir[PETSC_MAX_PATH_LEN];
834       if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"-saws_local option requires -saws_root option");
835       ierr = PetscSNPrintf(jsdir,PETSC_MAX_PATH_LEN,"%s/js",root);CHKERRQ(ierr);
836       ierr = PetscTestDirectory(jsdir,'r',&flg);CHKERRQ(ierr);
837       if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_FILE_READ,"-saws_local option requires js directory in root directory");
838       PetscStackCallSAWs(SAWs_Set_Local_JSHeader,());CHKERRQ(ierr);
839     }
840     ierr = PetscGetProgramName(programname,64);CHKERRQ(ierr);
841     if (rootlocal) {
842       ierr = PetscSNPrintf(appline,256,"%s.c.html",programname);CHKERRQ(ierr);
843       ierr = PetscTestFile(appline,'r',&rootlocal);CHKERRQ(ierr);
844     }
845     if (rootlocal && help) {
846       ierr = PetscSNPrintf(appline,256,"<center> Running <a href=\"%s.c.html\">%s</a> which %s </center><br>\n",programname,programname,help);
847     } else if (help) {
848       ierr = PetscSNPrintf(appline,256,"<center> Running %s which %s </center><br>\n",programname,help);
849     } else {
850       ierr = PetscSNPrintf(appline,256,"<center> Running %s</center><br>\n",programname);
851     }
852 
853     ierr = PetscSNPrintf(intro,4024,"<body>\n"
854                                     "<center><h2> <a href=\"http://www.mcs.anl.gov/petsc\">PETSc</a> Application Web server powered by <a href=\"https://bitbucket.org/saws/saws\">SAWs</a> </h2></center>\n"
855                                     "%s"
856                                     "<center>This is the default PETSc application dashboard, from it you can access any published PETSc objects or logging data</center><br>\n",appline);
857     PetscStackCallSAWs(SAWs_Set_Body,("index.html",0,intro));
858     PetscStackCallSAWs(SAWs_Initialize,());
859     ierr = PetscCitationsRegister("@TechReport{ saws,"
860                                   "Author = {Matt Otten and Jed Brown and Barry Smith},"
861                                   "Title  = {Scientific Application Web Server (SAWs) Users Manual},"
862                                   "Institution = {Argonne National Laboratory},"
863                                   "Year   = 2013}",NULL);CHKERRQ(ierr);
864   }
865 
866 #endif
867 
868   /* SHOULD PUT IN GUARDS: Make sure logging is initialized, even if we do not print it out */
869 #if defined(PETSC_USE_LOG)
870   ierr = PetscLogBegin_Private();CHKERRQ(ierr);
871 #endif
872 
873   /*
874      Load the dynamic libraries (on machines that support them), this registers all
875      the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
876   */
877   ierr = PetscInitialize_DynamicLibraries();CHKERRQ(ierr);
878 
879   ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
880   ierr = PetscInfo1(0,"PETSc successfully started: number of processors = %d\n",size);CHKERRQ(ierr);
881   ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr);
882   ierr = PetscInfo1(0,"Running on machine: %s\n",hostname);CHKERRQ(ierr);
883 
884   ierr = PetscOptionsCheckInitial_Components();CHKERRQ(ierr);
885   /* Check the options database for options related to the options database itself */
886   ierr = PetscOptionsSetFromOptions();CHKERRQ(ierr);
887 
888 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
889   /*
890       Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI
891 
892       Currently not used because it is not supported by MPICH.
893   */
894 #if !defined(PETSC_WORDS_BIGENDIAN)
895   ierr = MPI_Register_datarep((char*)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,NULL);CHKERRQ(ierr);
896 #endif
897 #endif
898 
899   ierr = PetscOptionsGetInt(NULL,"-hmpi_spawn_size",&nodesize,&flg);CHKERRQ(ierr);
900   if (flg) {
901 #if defined(PETSC_HAVE_MPI_COMM_SPAWN)
902     ierr = PetscHMPISpawn((PetscMPIInt) nodesize);CHKERRQ(ierr); /* worker nodes never return from here; they go directly to PetscEnd() */
903 #else
904     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead");
905 #endif
906   } else {
907     ierr = PetscOptionsGetInt(NULL,"-hmpi_merge_size",&nodesize,&flg);CHKERRQ(ierr);
908     if (flg) {
909       ierr = PetscHMPIMerge((PetscMPIInt) nodesize,NULL,NULL);CHKERRQ(ierr);
910       if (PetscHMPIWorker) { /* if worker then never enter user code */
911         PetscInitializeCalled = PETSC_TRUE;
912         PetscEnd();
913       }
914     }
915   }
916 
917 #if defined(PETSC_HAVE_CUDA)
918   {
919     PetscMPIInt p;
920     for (p = 0; p < PetscGlobalSize; ++p) {
921       if (p == PetscGlobalRank) cublasInit();
922       ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
923     }
924   }
925 #endif
926 
927   ierr = PetscOptionsHasName(NULL,"-python",&flg);CHKERRQ(ierr);
928   if (flg) {
929     PetscInitializeCalled = PETSC_TRUE;
930     ierr = PetscPythonInitialize(NULL,NULL);CHKERRQ(ierr);
931   }
932 
933   ierr = PetscThreadCommInitializePackage();CHKERRQ(ierr);
934 
935   /*
936       Setup building of stack frames for all function calls
937   */
938 #if defined(PETSC_USE_DEBUG)
939   PetscThreadLocalRegister((PetscThreadKey*)&petscstack); /* Creates petscstack_key if needed */
940   ierr = PetscStackCreate();CHKERRQ(ierr);
941 #endif
942 
943 #if defined(PETSC_SERIALIZE_FUNCTIONS)
944   ierr = PetscFPTCreate(10000);CHKERRQ(ierr);
945 #endif
946 
947 
948   /*
949       Once we are completedly initialized then we can set this variables
950   */
951   PetscInitializeCalled = PETSC_TRUE;
952   PetscFunctionReturn(0);
953 }
954 
955 extern PetscObject *PetscObjects;
956 extern PetscInt    PetscObjectsCounts, PetscObjectsMaxCounts;
957 
958 #undef __FUNCT__
959 #define __FUNCT__ "PetscFinalize"
960 /*@C
961    PetscFinalize - Checks for options to be called at the conclusion
962    of the program. MPI_Finalize() is called only if the user had not
963    called MPI_Init() before calling PetscInitialize().
964 
965    Collective on PETSC_COMM_WORLD
966 
967    Options Database Keys:
968 +  -options_table - Calls PetscOptionsView()
969 .  -options_left - Prints unused options that remain in the database
970 .  -objects_dump [all] - Prints list of objects allocated by the user that have not been freed, the option all cause all outstanding objects to be listed
971 .  -mpidump - Calls PetscMPIDump()
972 .  -malloc_dump - Calls PetscMallocDump()
973 .  -malloc_info - Prints total memory usage
974 -  -malloc_log - Prints summary of memory usage
975 
976    Level: beginner
977 
978    Note:
979    See PetscInitialize() for more general runtime options.
980 
981 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscEnd()
982 @*/
983 PetscErrorCode  PetscFinalize(void)
984 {
985   PetscErrorCode ierr;
986   PetscMPIInt    rank;
987   PetscInt       nopt;
988   PetscBool      flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE;
989   PetscBool      flg;
990 #if defined(PETSC_USE_LOG)
991   char           mname[PETSC_MAX_PATH_LEN];
992 #endif
993 
994   PetscFunctionBegin;
995   if (!PetscInitializeCalled) {
996     printf("PetscInitialize() must be called before PetscFinalize()\n");
997     PetscFunctionReturn(PETSC_ERR_ARG_WRONGSTATE);
998   }
999   ierr = PetscInfo(NULL,"PetscFinalize() called\n");CHKERRQ(ierr);
1000 
1001   ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
1002 
1003   ierr = PetscOptionsHasName(NULL,"-citations",&flg);CHKERRQ(ierr);
1004   if (flg) {
1005     char  *cits, filename[PETSC_MAX_PATH_LEN];
1006     FILE  *fd = PETSC_STDOUT;
1007 
1008     ierr = PetscOptionsGetString(NULL,"-citations",filename,PETSC_MAX_PATH_LEN,NULL);CHKERRQ(ierr);
1009     if (filename[0]) {
1010       ierr = PetscFOpen(PETSC_COMM_WORLD,filename,"w",&fd);CHKERRQ(ierr);
1011     }
1012     ierr = PetscSegBufferGet(PetscCitationsList,1,&cits);CHKERRQ(ierr);
1013     cits[0] = 0;
1014     ierr = PetscSegBufferExtractAlloc(PetscCitationsList,&cits);CHKERRQ(ierr);
1015     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"If you publish results based on this computation please cite the following:\n");CHKERRQ(ierr);
1016     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"===========================================================================\n");CHKERRQ(ierr);
1017     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"%s",cits);CHKERRQ(ierr);
1018     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"===========================================================================\n");CHKERRQ(ierr);
1019     ierr = PetscFClose(PETSC_COMM_WORLD,fd);CHKERRQ(ierr);
1020     ierr = PetscFree(cits);CHKERRQ(ierr);
1021   }
1022   ierr = PetscSegBufferDestroy(&PetscCitationsList);CHKERRQ(ierr);
1023 
1024 #if defined(PETSC_SERIALIZE_FUNCTIONS)
1025   ierr = PetscFPTDestroy();CHKERRQ(ierr);
1026 #endif
1027 
1028 
1029 #if defined(PETSC_HAVE_SAWS)
1030   flg = PETSC_FALSE;
1031   ierr = PetscOptionsGetBool(NULL,"-options_gui",&flg,NULL);CHKERRQ(ierr);
1032   if (flg) {
1033     ierr = PetscOptionsSAWsDestroy();CHKERRQ(ierr);
1034   }
1035 #endif
1036 
1037 #if defined(PETSC_HAVE_SERVER)
1038   flg1 = PETSC_FALSE;
1039   ierr = PetscOptionsGetBool(NULL,"-server",&flg1,NULL);CHKERRQ(ierr);
1040   if (flg1) {
1041     /*  this is a crude hack, but better than nothing */
1042     ierr = PetscPOpen(PETSC_COMM_WORLD,NULL,"pkill -9 petscwebserver","r",NULL);CHKERRQ(ierr);
1043   }
1044 #endif
1045 
1046 #if defined(PETSC_HAVE_X)
1047   flg1 = PETSC_FALSE;
1048   ierr = PetscOptionsGetBool(NULL,"-x_virtual",&flg1,NULL);CHKERRQ(ierr);
1049   if (flg1) {
1050     /*  this is a crude hack, but better than nothing */
1051     ierr = PetscPOpen(PETSC_COMM_WORLD,NULL,"pkill -9 Xvfb","r",NULL);CHKERRQ(ierr);
1052   }
1053 #endif
1054 
1055   ierr = PetscHMPIFinalize();CHKERRQ(ierr);
1056 
1057   ierr = PetscOptionsGetBool(NULL,"-malloc_info",&flg2,NULL);CHKERRQ(ierr);
1058   if (!flg2) {
1059     flg2 = PETSC_FALSE;
1060     ierr = PetscOptionsGetBool(NULL,"-memory_info",&flg2,NULL);CHKERRQ(ierr);
1061   }
1062   if (flg2) {
1063     ierr = PetscMemoryShowUsage(PETSC_VIEWER_STDOUT_WORLD,"Summary of Memory Usage in PETSc\n");CHKERRQ(ierr);
1064   }
1065 
1066 #if defined(PETSC_USE_LOG)
1067   flg1 = PETSC_FALSE;
1068   ierr = PetscOptionsGetBool(NULL,"-get_total_flops",&flg1,NULL);CHKERRQ(ierr);
1069   if (flg1) {
1070     PetscLogDouble flops = 0;
1071     ierr = MPI_Reduce(&petsc_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);CHKERRQ(ierr);
1072     ierr = PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %g\n",flops);CHKERRQ(ierr);
1073   }
1074 #endif
1075 
1076 
1077 #if defined(PETSC_USE_LOG)
1078 #if defined(PETSC_HAVE_MPE)
1079   mname[0] = 0;
1080 
1081   ierr = PetscOptionsGetString(NULL,"-log_mpe",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr);
1082   if (flg1) {
1083     if (mname[0]) {ierr = PetscLogMPEDump(mname);CHKERRQ(ierr);}
1084     else          {ierr = PetscLogMPEDump(0);CHKERRQ(ierr);}
1085   }
1086 #endif
1087   mname[0] = 0;
1088 
1089   ierr = PetscOptionsGetString(NULL,"-log_summary",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr);
1090   if (flg1) {
1091     PetscViewer viewer;
1092     if (mname[0]) {
1093       ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr);
1094       ierr = PetscLogView(viewer);CHKERRQ(ierr);
1095       ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1096     } else {
1097       viewer = PETSC_VIEWER_STDOUT_WORLD;
1098       ierr   = PetscLogView(viewer);CHKERRQ(ierr);
1099     }
1100   }
1101 
1102   mname[0] = 0;
1103 
1104   ierr = PetscOptionsGetString(NULL,"-log_summary_python",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr);
1105   if (flg1) {
1106     PetscViewer viewer;
1107     if (mname[0]) {
1108       ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr);
1109       ierr = PetscLogViewPython(viewer);CHKERRQ(ierr);
1110       ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1111     } else {
1112       viewer = PETSC_VIEWER_STDOUT_WORLD;
1113       ierr   = PetscLogViewPython(viewer);CHKERRQ(ierr);
1114     }
1115   }
1116 
1117   ierr = PetscOptionsGetString(NULL,"-log_detailed",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr);
1118   if (flg1) {
1119     if (mname[0])  {ierr = PetscLogPrintDetailed(PETSC_COMM_WORLD,mname);CHKERRQ(ierr);}
1120     else           {ierr = PetscLogPrintDetailed(PETSC_COMM_WORLD,0);CHKERRQ(ierr);}
1121   }
1122 
1123   mname[0] = 0;
1124 
1125   ierr = PetscOptionsGetString(NULL,"-log_all",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr);
1126   ierr = PetscOptionsGetString(NULL,"-log",mname,PETSC_MAX_PATH_LEN,&flg2);CHKERRQ(ierr);
1127   if (flg1 || flg2) {
1128     if (mname[0]) PetscLogDump(mname);
1129     else          PetscLogDump(0);
1130   }
1131 #endif
1132 
1133   /*
1134      Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_().
1135   */
1136   ierr = PetscObjectRegisterDestroyAll();CHKERRQ(ierr);
1137 
1138   ierr = PetscStackDestroy();CHKERRQ(ierr);
1139 
1140   flg1 = PETSC_FALSE;
1141   ierr = PetscOptionsGetBool(NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr);
1142   if (!flg1) { ierr = PetscPopSignalHandler();CHKERRQ(ierr);}
1143   flg1 = PETSC_FALSE;
1144   ierr = PetscOptionsGetBool(NULL,"-mpidump",&flg1,NULL);CHKERRQ(ierr);
1145   if (flg1) {
1146     ierr = PetscMPIDump(stdout);CHKERRQ(ierr);
1147   }
1148   flg1 = PETSC_FALSE;
1149   flg2 = PETSC_FALSE;
1150   /* preemptive call to avoid listing this option in options table as unused */
1151   ierr = PetscOptionsHasName(NULL,"-malloc_dump",&flg1);CHKERRQ(ierr);
1152   ierr = PetscOptionsHasName(NULL,"-objects_dump",&flg1);CHKERRQ(ierr);
1153   ierr = PetscOptionsGetBool(NULL,"-options_table",&flg2,NULL);CHKERRQ(ierr);
1154 
1155   if (flg2) {
1156     PetscViewer viewer;
1157     ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
1158     ierr = PetscOptionsView(viewer);CHKERRQ(ierr);
1159     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1160   }
1161 
1162   /* to prevent PETSc -options_left from warning */
1163   ierr = PetscOptionsHasName(NULL,"-nox",&flg1);CHKERRQ(ierr);
1164   ierr = PetscOptionsHasName(NULL,"-nox_warning",&flg1);CHKERRQ(ierr);
1165 
1166   if (!PetscHMPIWorker) { /* worker processes skip this because they do not usually process options */
1167     flg3 = PETSC_FALSE; /* default value is required */
1168     ierr = PetscOptionsGetBool(NULL,"-options_left",&flg3,&flg1);CHKERRQ(ierr);
1169     ierr = PetscOptionsAllUsed(&nopt);CHKERRQ(ierr);
1170     if (flg3) {
1171       if (!flg2) { /* have not yet printed the options */
1172         PetscViewer viewer;
1173         ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
1174         ierr = PetscOptionsView(viewer);CHKERRQ(ierr);
1175         ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1176       }
1177       if (!nopt) {
1178         ierr = PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");CHKERRQ(ierr);
1179       } else if (nopt == 1) {
1180         ierr = PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");CHKERRQ(ierr);
1181       } else {
1182         ierr = PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);CHKERRQ(ierr);
1183       }
1184     }
1185 #if defined(PETSC_USE_DEBUG)
1186     if (nopt && !flg3 && !flg1) {
1187       ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!\n");CHKERRQ(ierr);
1188       ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!\n");CHKERRQ(ierr);
1189       ierr = PetscOptionsLeft();CHKERRQ(ierr);
1190     } else if (nopt && flg3) {
1191 #else
1192     if (nopt && flg3) {
1193 #endif
1194       ierr = PetscOptionsLeft();CHKERRQ(ierr);
1195     }
1196   }
1197 
1198 #if defined(PETSC_HAVE_SAWS)
1199   if (!PetscGlobalRank) {
1200     ierr = PetscStackSAWsViewOff();CHKERRQ(ierr);
1201     PetscStackCallSAWs(SAWs_Finalize,());
1202   }
1203 #endif
1204 
1205   {
1206     PetscThreadComm tcomm_world;
1207     ierr = PetscGetThreadCommWorld(&tcomm_world);CHKERRQ(ierr);
1208     /* Free global thread communicator */
1209     ierr = PetscThreadCommDestroy(&tcomm_world);CHKERRQ(ierr);
1210   }
1211 
1212   /*
1213        List all objects the user may have forgot to free
1214   */
1215   ierr = PetscOptionsHasName(NULL,"-objects_dump",&flg1);CHKERRQ(ierr);
1216   if (flg1) {
1217     MPI_Comm local_comm;
1218     char     string[64];
1219 
1220     ierr = PetscOptionsGetString(NULL,"-objects_dump",string,64,NULL);CHKERRQ(ierr);
1221     ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRQ(ierr);
1222     ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr);
1223     ierr = PetscObjectsDump(stdout,(string[0] == 'a') ? PETSC_TRUE : PETSC_FALSE);CHKERRQ(ierr);
1224     ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr);
1225     ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr);
1226   }
1227   PetscObjectsCounts    = 0;
1228   PetscObjectsMaxCounts = 0;
1229 
1230   ierr = PetscFree(PetscObjects);CHKERRQ(ierr);
1231 
1232 #if defined(PETSC_USE_LOG)
1233   ierr = PetscLogDestroy();CHKERRQ(ierr);
1234 #endif
1235 
1236   /*
1237      Destroy any packages that registered a finalize
1238   */
1239   ierr = PetscRegisterFinalizeAll();CHKERRQ(ierr);
1240 
1241   /*
1242      Destroy all the function registration lists created
1243   */
1244   ierr = PetscFinalize_DynamicLibraries();CHKERRQ(ierr);
1245 
1246   /*
1247      Print PetscFunctionLists that have not been properly freed
1248 
1249   ierr = PetscFunctionListPrintAll();CHKERRQ(ierr);
1250   */
1251 
1252   if (petsc_history) {
1253     ierr = PetscCloseHistoryFile(&petsc_history);CHKERRQ(ierr);
1254     petsc_history = 0;
1255   }
1256 
1257   ierr = PetscInfoAllow(PETSC_FALSE,NULL);CHKERRQ(ierr);
1258 
1259   {
1260     char fname[PETSC_MAX_PATH_LEN];
1261     FILE *fd;
1262     int  err;
1263 
1264     fname[0] = 0;
1265 
1266     ierr = PetscOptionsGetString(NULL,"-malloc_dump",fname,250,&flg1);CHKERRQ(ierr);
1267     flg2 = PETSC_FALSE;
1268     ierr = PetscOptionsGetBool(NULL,"-malloc_test",&flg2,NULL);CHKERRQ(ierr);
1269 #if defined(PETSC_USE_DEBUG)
1270     if (PETSC_RUNNING_ON_VALGRIND) flg2 = PETSC_FALSE;
1271 #else
1272     flg2 = PETSC_FALSE;         /* Skip reporting for optimized builds regardless of -malloc_test */
1273 #endif
1274     if (flg1 && fname[0]) {
1275       char sname[PETSC_MAX_PATH_LEN];
1276 
1277       sprintf(sname,"%s_%d",fname,rank);
1278       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1279       ierr = PetscMallocDump(fd);CHKERRQ(ierr);
1280       err  = fclose(fd);
1281       if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1282     } else if (flg1 || flg2) {
1283       MPI_Comm local_comm;
1284 
1285       ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRQ(ierr);
1286       ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr);
1287       ierr = PetscMallocDump(stdout);CHKERRQ(ierr);
1288       ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr);
1289       ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr);
1290     }
1291   }
1292 
1293   {
1294     char fname[PETSC_MAX_PATH_LEN];
1295     FILE *fd = NULL;
1296 
1297     fname[0] = 0;
1298 
1299     ierr = PetscOptionsGetString(NULL,"-malloc_log",fname,250,&flg1);CHKERRQ(ierr);
1300     ierr = PetscOptionsHasName(NULL,"-malloc_log_threshold",&flg2);CHKERRQ(ierr);
1301     if (flg1 && fname[0]) {
1302       int err;
1303 
1304       if (!rank) {
1305         fd = fopen(fname,"w");
1306         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",fname);
1307       }
1308       ierr = PetscMallocDumpLog(fd);CHKERRQ(ierr);
1309       if (fd) {
1310         err = fclose(fd);
1311         if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1312       }
1313     } else if (flg1 || flg2) {
1314       ierr = PetscMallocDumpLog(stdout);CHKERRQ(ierr);
1315     }
1316   }
1317   /* Can be destroyed only after all the options are used */
1318   ierr = PetscOptionsDestroy();CHKERRQ(ierr);
1319 
1320   PetscGlobalArgc = 0;
1321   PetscGlobalArgs = 0;
1322 
1323 #if defined(PETSC_USE_REAL___FLOAT128)
1324   ierr = MPI_Type_free(&MPIU___FLOAT128);CHKERRQ(ierr);
1325 #if defined(PETSC_HAVE_COMPLEX)
1326   ierr = MPI_Type_free(&MPIU___COMPLEX128);CHKERRQ(ierr);
1327 #endif
1328   ierr = MPI_Op_free(&MPIU_MAX);CHKERRQ(ierr);
1329   ierr = MPI_Op_free(&MPIU_MIN);CHKERRQ(ierr);
1330 #endif
1331 
1332 #if defined(PETSC_HAVE_COMPLEX)
1333 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
1334   ierr = MPI_Type_free(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr);
1335   ierr = MPI_Type_free(&MPIU_C_COMPLEX);CHKERRQ(ierr);
1336 #endif
1337 #endif
1338 
1339 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
1340   ierr = MPI_Op_free(&MPIU_SUM);CHKERRQ(ierr);
1341 #endif
1342 
1343   ierr = MPI_Type_free(&MPIU_2SCALAR);CHKERRQ(ierr);
1344 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
1345   ierr = MPI_Type_free(&MPIU_2INT);CHKERRQ(ierr);
1346 #endif
1347   ierr = MPI_Op_free(&PetscMaxSum_Op);CHKERRQ(ierr);
1348   ierr = MPI_Op_free(&PetscADMax_Op);CHKERRQ(ierr);
1349   ierr = MPI_Op_free(&PetscADMin_Op);CHKERRQ(ierr);
1350 
1351   /*
1352      Destroy any known inner MPI_Comm's and attributes pointing to them
1353      Note this will not destroy any new communicators the user has created.
1354 
1355      If all PETSc objects were not destroyed those left over objects will have hanging references to
1356      the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again
1357  */
1358   {
1359     PetscCommCounter *counter;
1360     PetscMPIInt      flg;
1361     MPI_Comm         icomm;
1362     union {MPI_Comm comm; void *ptr;} ucomm;
1363     ierr = MPI_Attr_get(PETSC_COMM_SELF,Petsc_InnerComm_keyval,&ucomm,&flg);CHKERRQ(ierr);
1364     if (flg) {
1365       icomm = ucomm.comm;
1366       ierr = MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRQ(ierr);
1367       if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1368 
1369       ierr = MPI_Attr_delete(PETSC_COMM_SELF,Petsc_InnerComm_keyval);CHKERRQ(ierr);
1370       ierr = MPI_Attr_delete(icomm,Petsc_Counter_keyval);CHKERRQ(ierr);
1371       ierr = MPI_Comm_free(&icomm);CHKERRQ(ierr);
1372     }
1373     ierr = MPI_Attr_get(PETSC_COMM_WORLD,Petsc_InnerComm_keyval,&ucomm,&flg);CHKERRQ(ierr);
1374     if (flg) {
1375       icomm = ucomm.comm;
1376       ierr = MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRQ(ierr);
1377       if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1378 
1379       ierr = MPI_Attr_delete(PETSC_COMM_WORLD,Petsc_InnerComm_keyval);CHKERRQ(ierr);
1380       ierr = MPI_Attr_delete(icomm,Petsc_Counter_keyval);CHKERRQ(ierr);
1381       ierr = MPI_Comm_free(&icomm);CHKERRQ(ierr);
1382     }
1383   }
1384 
1385   ierr = MPI_Keyval_free(&Petsc_Counter_keyval);CHKERRQ(ierr);
1386   ierr = MPI_Keyval_free(&Petsc_InnerComm_keyval);CHKERRQ(ierr);
1387   ierr = MPI_Keyval_free(&Petsc_OuterComm_keyval);CHKERRQ(ierr);
1388 
1389 #if defined(PETSC_HAVE_CUDA)
1390   {
1391     PetscInt p;
1392     for (p = 0; p < PetscGlobalSize; ++p) {
1393       if (p == PetscGlobalRank) cublasShutdown();
1394       ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
1395     }
1396   }
1397 #endif
1398 
1399   if (PetscBeganMPI) {
1400 #if defined(PETSC_HAVE_MPI_FINALIZED)
1401     PetscMPIInt flag;
1402     ierr = MPI_Finalized(&flag);CHKERRQ(ierr);
1403     if (flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()");
1404 #endif
1405     ierr = MPI_Finalize();CHKERRQ(ierr);
1406   }
1407 /*
1408 
1409      Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because
1410    the communicator has some outstanding requests on it. Specifically if the
1411    flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See
1412    src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate()
1413    is never freed as it should be. Thus one may obtain messages of the form
1414    [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the
1415    memory was not freed.
1416 
1417 */
1418   ierr = PetscMallocClear();CHKERRQ(ierr);
1419 
1420   PetscInitializeCalled = PETSC_FALSE;
1421   PetscFinalizeCalled   = PETSC_TRUE;
1422   PetscFunctionReturn(ierr);
1423 }
1424 
1425 #if defined(PETSC_MISSING_LAPACK_lsame_)
1426 PETSC_EXTERN int lsame_(char *a,char *b)
1427 {
1428   if (*a == *b) return 1;
1429   if (*a + 32 == *b) return 1;
1430   if (*a - 32 == *b) return 1;
1431   return 0;
1432 }
1433 #endif
1434 
1435 #if defined(PETSC_MISSING_LAPACK_lsame)
1436 PETSC_EXTERN int lsame(char *a,char *b)
1437 {
1438   if (*a == *b) return 1;
1439   if (*a + 32 == *b) return 1;
1440   if (*a - 32 == *b) return 1;
1441   return 0;
1442 }
1443 #endif
1444