xref: /petsc/src/sys/objects/pinit.c (revision 5fedec97950c19de564efaecd0f125b1a6cb2b20)
1 #define PETSC_DESIRE_FEATURE_TEST_MACROS
2 /*
3    This file defines the initialization of PETSc, including PetscInitialize()
4 */
5 #include <petsc/private/petscimpl.h>        /*I  "petscsys.h"   I*/
6 #include <petscviewer.h>
7 
8 #if !defined(PETSC_HAVE_WINDOWS_COMPILERS)
9 #include <petsc/private/valgrind/valgrind.h>
10 #endif
11 
12 #if defined(PETSC_HAVE_FORTRAN)
13 #include <petsc/private/fortranimpl.h>
14 #endif
15 
16 #if defined(PETSC_USE_GCOV)
17 EXTERN_C_BEGIN
18 void  __gcov_flush(void);
19 EXTERN_C_END
20 #endif
21 
22 #if defined(PETSC_SERIALIZE_FUNCTIONS)
23 PETSC_INTERN PetscFPT PetscFPTData;
24 PetscFPT PetscFPTData = 0;
25 #endif
26 
27 #if PetscDefined(HAVE_SAWS)
28 #include <petscviewersaws.h>
29 #endif
30 
31 /* -----------------------------------------------------------------------------------------*/
32 
33 PETSC_INTERN FILE *petsc_history;
34 
35 PETSC_INTERN PetscErrorCode PetscInitialize_DynamicLibraries(void);
36 PETSC_INTERN PetscErrorCode PetscFinalize_DynamicLibraries(void);
37 PETSC_INTERN PetscErrorCode PetscFunctionListPrintAll(void);
38 PETSC_INTERN PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm,int);
39 PETSC_INTERN PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm,int);
40 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE**);
41 
42 /* user may set these BEFORE calling PetscInitialize() */
43 MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL;
44 #if PetscDefined(HAVE_MPI_INIT_THREAD)
45 PetscMPIInt PETSC_MPI_THREAD_REQUIRED = MPI_THREAD_FUNNELED;
46 #else
47 PetscMPIInt PETSC_MPI_THREAD_REQUIRED = 0;
48 #endif
49 
50 PetscMPIInt Petsc_Counter_keyval   = MPI_KEYVAL_INVALID;
51 PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID;
52 PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID;
53 PetscMPIInt Petsc_ShmComm_keyval   = MPI_KEYVAL_INVALID;
54 
55 /*
56      Declare and set all the string names of the PETSc enums
57 */
58 const char *const PetscBools[]     = {"FALSE","TRUE","PetscBool","PETSC_",NULL};
59 const char *const PetscCopyModes[] = {"COPY_VALUES","OWN_POINTER","USE_POINTER","PetscCopyMode","PETSC_",NULL};
60 
61 PetscBool PetscPreLoadingUsed = PETSC_FALSE;
62 PetscBool PetscPreLoadingOn   = PETSC_FALSE;
63 
64 PetscInt PetscHotRegionDepth;
65 
66 PetscBool PETSC_RUNNING_ON_VALGRIND = PETSC_FALSE;
67 
68 #if defined(PETSC_HAVE_THREADSAFETY)
69 PetscSpinlock PetscViewerASCIISpinLockOpen;
70 PetscSpinlock PetscViewerASCIISpinLockStdout;
71 PetscSpinlock PetscViewerASCIISpinLockStderr;
72 PetscSpinlock PetscCommSpinLock;
73 #endif
74 
75 /*
76       PetscInitializeNoPointers - Calls PetscInitialize() from C/C++ without the pointers to argc and args
77 
78    Collective
79 
80    Level: advanced
81 
82     Notes:
83     this is called only by the PETSc Julia interface. Even though it might start MPI it sets the flag to
84      indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to
85      be called multiple times from Julia without the problem of trying to initialize MPI more than once.
86 
87      Developer Note: Turns off PETSc signal handling to allow Julia to manage signals
88 
89 .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments()
90 */
91 PetscErrorCode  PetscInitializeNoPointers(int argc,char **args,const char *filename,const char *help)
92 {
93   PetscErrorCode ierr;
94   int            myargc   = argc;
95   char           **myargs = args;
96 
97   PetscFunctionBegin;
98   ierr = PetscInitialize(&myargc,&myargs,filename,help);if (ierr) PetscFunctionReturn(ierr);
99   ierr = PetscPopSignalHandler();CHKERRQ(ierr);
100   PetscBeganMPI = PETSC_FALSE;
101   PetscFunctionReturn(0);
102 }
103 
104 /*
105       Used by Julia interface to get communicator
106 */
107 PetscErrorCode  PetscGetPETSC_COMM_SELF(MPI_Comm *comm)
108 {
109   PetscFunctionBegin;
110   if (PetscInitializeCalled) PetscValidPointer(comm,1);
111   *comm = PETSC_COMM_SELF;
112   PetscFunctionReturn(0);
113 }
114 
115 /*@C
116       PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without
117         the command line arguments.
118 
119    Collective
120 
121    Level: advanced
122 
123 .seealso: PetscInitialize(), PetscInitializeFortran()
124 @*/
125 PetscErrorCode  PetscInitializeNoArguments(void)
126 {
127   PetscErrorCode ierr;
128   int            argc   = 0;
129   char           **args = NULL;
130 
131   PetscFunctionBegin;
132   ierr = PetscInitialize(&argc,&args,NULL,NULL);
133   PetscFunctionReturn(ierr);
134 }
135 
136 /*@
137       PetscInitialized - Determine whether PETSc is initialized.
138 
139    Level: beginner
140 
141 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
142 @*/
143 PetscErrorCode PetscInitialized(PetscBool *isInitialized)
144 {
145   PetscFunctionBegin;
146   if (PetscInitializeCalled) PetscValidBoolPointer(isInitialized,1);
147   *isInitialized = PetscInitializeCalled;
148   PetscFunctionReturn(0);
149 }
150 
151 /*@
152       PetscFinalized - Determine whether PetscFinalize() has been called yet
153 
154    Level: developer
155 
156 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran()
157 @*/
158 PetscErrorCode  PetscFinalized(PetscBool  *isFinalized)
159 {
160   PetscFunctionBegin;
161   if (!PetscFinalizeCalled) PetscValidBoolPointer(isFinalized,1);
162   *isFinalized = PetscFinalizeCalled;
163   PetscFunctionReturn(0);
164 }
165 
166 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char []);
167 
168 /*
169        This function is the MPI reduction operation used to compute the sum of the
170    first half of the datatype and the max of the second half.
171 */
172 MPI_Op MPIU_MAXSUM_OP = 0;
173 
174 PETSC_INTERN void MPIAPI MPIU_MaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype)
175 {
176   PetscInt *xin = (PetscInt*)in,*xout = (PetscInt*)out,i,count = *cnt;
177 
178   PetscFunctionBegin;
179   if (*datatype != MPIU_2INT) {
180     (*PetscErrorPrintf)("Can only handle MPIU_2INT data types");
181     PETSCABORT(MPI_COMM_SELF,PETSC_ERR_ARG_WRONG);
182   }
183 
184   for (i=0; i<count; i++) {
185     xout[2*i]    = PetscMax(xout[2*i],xin[2*i]);
186     xout[2*i+1] += xin[2*i+1];
187   }
188   PetscFunctionReturnVoid();
189 }
190 
191 /*
192     Returns the max of the first entry owned by this processor and the
193 sum of the second entry.
194 
195     The reason sizes[2*i] contains lengths sizes[2*i+1] contains flag of 1 if length is nonzero
196 is so that the MPIU_MAXSUM_OP() can set TWO values, if we passed in only sizes[i] with lengths
197 there would be no place to store the both needed results.
198 */
199 PetscErrorCode  PetscMaxSum(MPI_Comm comm,const PetscInt sizes[],PetscInt *max,PetscInt *sum)
200 {
201   PetscErrorCode ierr;
202 
203   PetscFunctionBegin;
204 #if defined(PETSC_HAVE_MPI_REDUCE_SCATTER_BLOCK)
205   {
206     struct {PetscInt max,sum;} work;
207     ierr = MPI_Reduce_scatter_block((void*)sizes,&work,1,MPIU_2INT,MPIU_MAXSUM_OP,comm);CHKERRMPI(ierr);
208     *max = work.max;
209     *sum = work.sum;
210   }
211 #else
212   {
213     PetscMPIInt    size,rank;
214     struct {PetscInt max,sum;} *work;
215     ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr);
216     ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr);
217     ierr = PetscMalloc1(size,&work);CHKERRQ(ierr);
218     ierr = MPIU_Allreduce((void*)sizes,work,size,MPIU_2INT,MPIU_MAXSUM_OP,comm);CHKERRMPI(ierr);
219     *max = work[rank].max;
220     *sum = work[rank].sum;
221     ierr = PetscFree(work);CHKERRQ(ierr);
222   }
223 #endif
224   PetscFunctionReturn(0);
225 }
226 
227 /* ----------------------------------------------------------------------------*/
228 
229 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
230 MPI_Op MPIU_SUM = 0;
231 
232 PETSC_EXTERN void MPIAPI PetscSum_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
233 {
234   PetscInt i,count = *cnt;
235 
236   PetscFunctionBegin;
237   if (*datatype == MPIU_REAL) {
238     PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out;
239     for (i=0; i<count; i++) xout[i] += xin[i];
240   }
241 #if defined(PETSC_HAVE_COMPLEX)
242   else if (*datatype == MPIU_COMPLEX) {
243     PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out;
244     for (i=0; i<count; i++) xout[i] += xin[i];
245   }
246 #endif
247   else {
248     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types");
249     PETSCABORT(MPI_COMM_SELF,PETSC_ERR_ARG_WRONG);
250   }
251   PetscFunctionReturnVoid();
252 }
253 #endif
254 
255 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
256 MPI_Op MPIU_MAX = 0;
257 MPI_Op MPIU_MIN = 0;
258 
259 PETSC_EXTERN void MPIAPI PetscMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
260 {
261   PetscInt i,count = *cnt;
262 
263   PetscFunctionBegin;
264   if (*datatype == MPIU_REAL) {
265     PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out;
266     for (i=0; i<count; i++) xout[i] = PetscMax(xout[i],xin[i]);
267   }
268 #if defined(PETSC_HAVE_COMPLEX)
269   else if (*datatype == MPIU_COMPLEX) {
270     PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out;
271     for (i=0; i<count; i++) {
272       xout[i] = PetscRealPartComplex(xout[i])<PetscRealPartComplex(xin[i]) ? xin[i] : xout[i];
273     }
274   }
275 #endif
276   else {
277     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types");
278     PETSCABORT(MPI_COMM_SELF,PETSC_ERR_ARG_WRONG);
279   }
280   PetscFunctionReturnVoid();
281 }
282 
283 PETSC_EXTERN void MPIAPI PetscMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype)
284 {
285   PetscInt    i,count = *cnt;
286 
287   PetscFunctionBegin;
288   if (*datatype == MPIU_REAL) {
289     PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out;
290     for (i=0; i<count; i++) xout[i] = PetscMin(xout[i],xin[i]);
291   }
292 #if defined(PETSC_HAVE_COMPLEX)
293   else if (*datatype == MPIU_COMPLEX) {
294     PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out;
295     for (i=0; i<count; i++) {
296       xout[i] = PetscRealPartComplex(xout[i])>PetscRealPartComplex(xin[i]) ? xin[i] : xout[i];
297     }
298   }
299 #endif
300   else {
301     (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types");
302     PETSCABORT(MPI_COMM_SELF,PETSC_ERR_ARG_WRONG);
303   }
304   PetscFunctionReturnVoid();
305 }
306 #endif
307 
308 /*
309    Private routine to delete internal tag/name counter storage when a communicator is freed.
310 
311    This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this  data as an attribute is freed.
312 
313    Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
314 
315 */
316 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_Counter_Attr_Delete_Fn(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
317 {
318   PetscErrorCode   ierr;
319   PetscCommCounter *counter=(PetscCommCounter*)count_val;
320 
321   PetscFunctionBegin;
322   ierr = PetscInfo1(NULL,"Deleting counter data in an MPI_Comm %ld\n",(long)comm);CHKERRMPI(ierr);
323   ierr = PetscFree(counter->iflags);CHKERRMPI(ierr);
324   ierr = PetscFree(counter);CHKERRMPI(ierr);
325   PetscFunctionReturn(MPI_SUCCESS);
326 }
327 
328 /*
329   This is invoked on the outer comm as a result of either PetscCommDestroy() (via MPI_Comm_delete_attr) or when the user
330   calls MPI_Comm_free().
331 
332   This is the only entry point for breaking the links between inner and outer comms.
333 
334   This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator.
335 
336   Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
337 
338 */
339 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_InnerComm_Attr_Delete_Fn(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state)
340 {
341   PetscErrorCode                    ierr;
342   union {MPI_Comm comm; void *ptr;} icomm;
343 
344   PetscFunctionBegin;
345   if (keyval != Petsc_InnerComm_keyval) SETERRMPI(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Unexpected keyval");
346   icomm.ptr = attr_val;
347   if (PetscDefined(USE_DEBUG)) {
348     /* Error out if the inner/outer comms are not correctly linked through their Outer/InnterComm attributes */
349     PetscMPIInt flg;
350     union {MPI_Comm comm; void *ptr;} ocomm;
351     ierr = MPI_Comm_get_attr(icomm.comm,Petsc_OuterComm_keyval,&ocomm,&flg);CHKERRMPI(ierr);
352     if (!flg) SETERRMPI(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner comm does not have OuterComm attribute");
353     if (ocomm.comm != comm) SETERRMPI(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner comm's OuterComm attribute does not point to outer PETSc comm");
354   }
355   ierr = MPI_Comm_delete_attr(icomm.comm,Petsc_OuterComm_keyval);CHKERRMPI(ierr);
356   ierr = PetscInfo2(NULL,"User MPI_Comm %ld is being unlinked from inner PETSc comm %ld\n",(long)comm,(long)icomm.comm);CHKERRMPI(ierr);
357   PetscFunctionReturn(MPI_SUCCESS);
358 }
359 
360 /*
361  * This is invoked on the inner comm when Petsc_InnerComm_Attr_Delete_Fn calls MPI_Comm_delete_attr().  It should not be reached any other way.
362  */
363 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_OuterComm_Attr_Delete_Fn(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state)
364 {
365   PetscErrorCode ierr;
366 
367   PetscFunctionBegin;
368   ierr = PetscInfo1(NULL,"Removing reference to PETSc communicator embedded in a user MPI_Comm %ld\n",(long)comm);CHKERRMPI(ierr);
369   PetscFunctionReturn(MPI_SUCCESS);
370 }
371 
372 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_ShmComm_Attr_Delete_Fn(MPI_Comm,PetscMPIInt,void *,void *);
373 
374 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
375 PETSC_EXTERN PetscMPIInt PetscDataRep_extent_fn(MPI_Datatype,MPI_Aint*,void*);
376 PETSC_EXTERN PetscMPIInt PetscDataRep_read_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*);
377 PETSC_EXTERN PetscMPIInt PetscDataRep_write_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*);
378 #endif
379 
380 PetscMPIInt PETSC_MPI_ERROR_CLASS=MPI_ERR_LASTCODE,PETSC_MPI_ERROR_CODE;
381 
382 PETSC_INTERN int  PetscGlobalArgc;
383 PETSC_INTERN char **PetscGlobalArgs;
384 int  PetscGlobalArgc   = 0;
385 char **PetscGlobalArgs = NULL;
386 PetscSegBuffer PetscCitationsList;
387 
388 PetscErrorCode PetscCitationsInitialize(void)
389 {
390   PetscErrorCode ierr;
391 
392   PetscFunctionBegin;
393   ierr = PetscSegBufferCreate(1,10000,&PetscCitationsList);CHKERRQ(ierr);
394   ierr = PetscCitationsRegister("@TechReport{petsc-user-ref,\n  Author = {Satish Balay and Shrirang Abhyankar and Mark F. Adams and Jed Brown \n            and Peter Brune and Kris Buschelman and Lisandro Dalcin and\n            Victor Eijkhout and William D. Gropp and Dmitry Karpeyev and\n            Dinesh Kaushik and Matthew G. Knepley and Dave A. May and Lois Curfman McInnes\n            and Richard Tran Mills and Todd Munson and Karl Rupp and Patrick Sanan\n            and Barry F. Smith and Stefano Zampini and Hong Zhang and Hong Zhang},\n  Title = {{PETS}c Users Manual},\n  Number = {ANL-95/11 - Revision 3.11},\n  Institution = {Argonne National Laboratory},\n  Year = {2019}\n}\n",NULL);CHKERRQ(ierr);
395   ierr = PetscCitationsRegister("@InProceedings{petsc-efficient,\n  Author = {Satish Balay and William D. Gropp and Lois Curfman McInnes and Barry F. Smith},\n  Title = {Efficient Management of Parallelism in Object Oriented Numerical Software Libraries},\n  Booktitle = {Modern Software Tools in Scientific Computing},\n  Editor = {E. Arge and A. M. Bruaset and H. P. Langtangen},\n  Pages = {163--202},\n  Publisher = {Birkh{\\\"{a}}user Press},\n  Year = {1997}\n}\n",NULL);CHKERRQ(ierr);
396   PetscFunctionReturn(0);
397 }
398 
399 static char programname[PETSC_MAX_PATH_LEN] = ""; /* HP includes entire path in name */
400 
401 PetscErrorCode  PetscSetProgramName(const char name[])
402 {
403   PetscErrorCode ierr;
404 
405   PetscFunctionBegin;
406   ierr  = PetscStrncpy(programname,name,sizeof(programname));CHKERRQ(ierr);
407   PetscFunctionReturn(0);
408 }
409 
410 /*@C
411     PetscGetProgramName - Gets the name of the running program.
412 
413     Not Collective
414 
415     Input Parameter:
416 .   len - length of the string name
417 
418     Output Parameter:
419 .   name - the name of the running program
420 
421    Level: advanced
422 
423     Notes:
424     The name of the program is copied into the user-provided character
425     array of length len.  On some machines the program name includes
426     its entire path, so one should generally set len >= PETSC_MAX_PATH_LEN.
427 @*/
428 PetscErrorCode  PetscGetProgramName(char name[],size_t len)
429 {
430   PetscErrorCode ierr;
431 
432   PetscFunctionBegin;
433   ierr = PetscStrncpy(name,programname,len);CHKERRQ(ierr);
434   PetscFunctionReturn(0);
435 }
436 
437 /*@C
438    PetscGetArgs - Allows you to access the raw command line arguments anywhere
439      after PetscInitialize() is called but before PetscFinalize().
440 
441    Not Collective
442 
443    Output Parameters:
444 +  argc - count of number of command line arguments
445 -  args - the command line arguments
446 
447    Level: intermediate
448 
449    Notes:
450       This is usually used to pass the command line arguments into other libraries
451    that are called internally deep in PETSc or the application.
452 
453       The first argument contains the program name as is normal for C arguments.
454 
455 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArguments()
456 
457 @*/
458 PetscErrorCode  PetscGetArgs(int *argc,char ***args)
459 {
460   PetscFunctionBegin;
461   if (PetscUnlikely(!PetscInitializeCalled && PetscFinalizeCalled)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
462   *argc = PetscGlobalArgc;
463   *args = PetscGlobalArgs;
464   PetscFunctionReturn(0);
465 }
466 
467 /*@C
468    PetscGetArguments - Allows you to access the  command line arguments anywhere
469      after PetscInitialize() is called but before PetscFinalize().
470 
471    Not Collective
472 
473    Output Parameters:
474 .  args - the command line arguments
475 
476    Level: intermediate
477 
478    Notes:
479       This does NOT start with the program name and IS null terminated (final arg is void)
480 
481 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscFreeArguments()
482 
483 @*/
484 PetscErrorCode  PetscGetArguments(char ***args)
485 {
486   PetscInt       i,argc = PetscGlobalArgc;
487   PetscErrorCode ierr;
488 
489   PetscFunctionBegin;
490   if (PetscUnlikely(!PetscInitializeCalled && PetscFinalizeCalled)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()");
491   if (!argc) {*args = NULL; PetscFunctionReturn(0);}
492   ierr = PetscMalloc1(argc,args);CHKERRQ(ierr);
493   for (i=0; i<argc-1; i++) {
494     ierr = PetscStrallocpy(PetscGlobalArgs[i+1],&(*args)[i]);CHKERRQ(ierr);
495   }
496   (*args)[argc-1] = NULL;
497   PetscFunctionReturn(0);
498 }
499 
500 /*@C
501    PetscFreeArguments - Frees the memory obtained with PetscGetArguments()
502 
503    Not Collective
504 
505    Output Parameters:
506 .  args - the command line arguments
507 
508    Level: intermediate
509 
510 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscGetArguments()
511 
512 @*/
513 PetscErrorCode  PetscFreeArguments(char **args)
514 {
515   PetscInt       i = 0;
516   PetscErrorCode ierr;
517 
518   PetscFunctionBegin;
519   if (!args) PetscFunctionReturn(0);
520   while (args[i]) {
521     ierr = PetscFree(args[i]);CHKERRQ(ierr);
522     i++;
523   }
524   ierr = PetscFree(args);CHKERRQ(ierr);
525   PetscFunctionReturn(0);
526 }
527 
528 #if PetscDefined(HAVE_SAWS)
529 #include <petscconfiginfo.h>
530 
531 PETSC_INTERN PetscErrorCode PetscInitializeSAWs(const char help[])
532 {
533   PetscFunctionBegin;
534   if (!PetscGlobalRank) {
535     char           cert[PETSC_MAX_PATH_LEN],root[PETSC_MAX_PATH_LEN],*intro,programname[64],*appline,*options,version[64];
536     int            port;
537     PetscBool      flg,rootlocal = PETSC_FALSE,flg2,selectport = PETSC_FALSE;
538     size_t         applinelen,introlen;
539     PetscErrorCode ierr;
540     char           sawsurl[256];
541 
542     ierr = PetscOptionsHasName(NULL,NULL,"-saws_log",&flg);CHKERRQ(ierr);
543     if (flg) {
544       char  sawslog[PETSC_MAX_PATH_LEN];
545 
546       ierr = PetscOptionsGetString(NULL,NULL,"-saws_log",sawslog,sizeof(sawslog),NULL);CHKERRQ(ierr);
547       if (sawslog[0]) {
548         PetscStackCallSAWs(SAWs_Set_Use_Logfile,(sawslog));
549       } else {
550         PetscStackCallSAWs(SAWs_Set_Use_Logfile,(NULL));
551       }
552     }
553     ierr = PetscOptionsGetString(NULL,NULL,"-saws_https",cert,sizeof(cert),&flg);CHKERRQ(ierr);
554     if (flg) {
555       PetscStackCallSAWs(SAWs_Set_Use_HTTPS,(cert));
556     }
557     ierr = PetscOptionsGetBool(NULL,NULL,"-saws_port_auto_select",&selectport,NULL);CHKERRQ(ierr);
558     if (selectport) {
559         PetscStackCallSAWs(SAWs_Get_Available_Port,(&port));
560         PetscStackCallSAWs(SAWs_Set_Port,(port));
561     } else {
562       ierr = PetscOptionsGetInt(NULL,NULL,"-saws_port",&port,&flg);CHKERRQ(ierr);
563       if (flg) {
564         PetscStackCallSAWs(SAWs_Set_Port,(port));
565       }
566     }
567     ierr = PetscOptionsGetString(NULL,NULL,"-saws_root",root,sizeof(root),&flg);CHKERRQ(ierr);
568     if (flg) {
569       PetscStackCallSAWs(SAWs_Set_Document_Root,(root));
570       ierr = PetscStrcmp(root,".",&rootlocal);CHKERRQ(ierr);
571     } else {
572       ierr = PetscOptionsHasName(NULL,NULL,"-saws_options",&flg);CHKERRQ(ierr);
573       if (flg) {
574         ierr = PetscStrreplace(PETSC_COMM_WORLD,"${PETSC_DIR}/share/petsc/saws",root,sizeof(root));CHKERRQ(ierr);
575         PetscStackCallSAWs(SAWs_Set_Document_Root,(root));
576       }
577     }
578     ierr = PetscOptionsHasName(NULL,NULL,"-saws_local",&flg2);CHKERRQ(ierr);
579     if (flg2) {
580       char jsdir[PETSC_MAX_PATH_LEN];
581       if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"-saws_local option requires -saws_root option");
582       ierr = PetscSNPrintf(jsdir,sizeof(jsdir),"%s/js",root);CHKERRQ(ierr);
583       ierr = PetscTestDirectory(jsdir,'r',&flg);CHKERRQ(ierr);
584       if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_FILE_READ,"-saws_local option requires js directory in root directory");
585       PetscStackCallSAWs(SAWs_Push_Local_Header,());
586     }
587     ierr = PetscGetProgramName(programname,sizeof(programname));CHKERRQ(ierr);
588     ierr = PetscStrlen(help,&applinelen);CHKERRQ(ierr);
589     introlen   = 4096 + applinelen;
590     applinelen += 1024;
591     ierr = PetscMalloc(applinelen,&appline);CHKERRQ(ierr);
592     ierr = PetscMalloc(introlen,&intro);CHKERRQ(ierr);
593 
594     if (rootlocal) {
595       ierr = PetscSNPrintf(appline,applinelen,"%s.c.html",programname);CHKERRQ(ierr);
596       ierr = PetscTestFile(appline,'r',&rootlocal);CHKERRQ(ierr);
597     }
598     ierr = PetscOptionsGetAll(NULL,&options);CHKERRQ(ierr);
599     if (rootlocal && help) {
600       ierr = PetscSNPrintf(appline,applinelen,"<center> Running <a href=\"%s.c.html\">%s</a> %s</center><br><center><pre>%s</pre></center><br>\n",programname,programname,options,help);CHKERRQ(ierr);
601     } else if (help) {
602       ierr = PetscSNPrintf(appline,applinelen,"<center>Running %s %s</center><br><center><pre>%s</pre></center><br>",programname,options,help);CHKERRQ(ierr);
603     } else {
604       ierr = PetscSNPrintf(appline,applinelen,"<center> Running %s %s</center><br>\n",programname,options);CHKERRQ(ierr);
605     }
606     ierr = PetscFree(options);CHKERRQ(ierr);
607     ierr = PetscGetVersion(version,sizeof(version));CHKERRQ(ierr);
608     ierr = PetscSNPrintf(intro,introlen,"<body>\n"
609                                     "<center><h2> <a href=\"https://petsc.org/\">PETSc</a> Application Web server powered by <a href=\"https://bitbucket.org/saws/saws\">SAWs</a> </h2></center>\n"
610                                     "<center>This is the default PETSc application dashboard, from it you can access any published PETSc objects or logging data</center><br><center>%s configured with %s</center><br>\n"
611                                     "%s",version,petscconfigureoptions,appline);CHKERRQ(ierr);
612     PetscStackCallSAWs(SAWs_Push_Body,("index.html",0,intro));
613     ierr = PetscFree(intro);CHKERRQ(ierr);
614     ierr = PetscFree(appline);CHKERRQ(ierr);
615     if (selectport) {
616       PetscBool silent;
617 
618       ierr = SAWs_Initialize();
619       /* another process may have grabbed the port so keep trying */
620       while (ierr) {
621         PetscStackCallSAWs(SAWs_Get_Available_Port,(&port));
622         PetscStackCallSAWs(SAWs_Set_Port,(port));
623         ierr = SAWs_Initialize();
624       }
625 
626       ierr = PetscOptionsGetBool(NULL,NULL,"-saws_port_auto_select_silent",&silent,NULL);CHKERRQ(ierr);
627       if (!silent) {
628         PetscStackCallSAWs(SAWs_Get_FullURL,(sizeof(sawsurl),sawsurl));
629         ierr = PetscPrintf(PETSC_COMM_WORLD,"Point your browser to %s for SAWs\n",sawsurl);CHKERRQ(ierr);
630       }
631     } else {
632       PetscStackCallSAWs(SAWs_Initialize,());
633     }
634     ierr = PetscCitationsRegister("@TechReport{ saws,\n"
635                                   "  Author = {Matt Otten and Jed Brown and Barry Smith},\n"
636                                   "  Title  = {Scientific Application Web Server (SAWs) Users Manual},\n"
637                                   "  Institution = {Argonne National Laboratory},\n"
638                                   "  Year   = 2013\n}\n",NULL);CHKERRQ(ierr);
639   }
640   PetscFunctionReturn(0);
641 }
642 #endif
643 
644 /* Things must be done before MPI_Init() when MPI is not yet initialized, and can be shared between C init and Fortran init */
645 PETSC_INTERN PetscErrorCode PetscPreMPIInit_Private(void)
646 {
647   PetscFunctionBegin;
648 #if defined(PETSC_HAVE_HWLOC_SOLARIS_BUG)
649     /* see MPI.py for details on this bug */
650     (void) setenv("HWLOC_COMPONENTS","-x86",1);
651 #endif
652   PetscFunctionReturn(0);
653 }
654 
655 #if PetscDefined(HAVE_ADIOS)
656 #include <adios.h>
657 #include <adios_read.h>
658 int64_t Petsc_adios_group;
659 #endif
660 #if PetscDefined(HAVE_OPENMP)
661 #include <omp.h>
662 PetscInt PetscNumOMPThreads;
663 #endif
664 
665 #if PetscDefined(HAVE_DEVICE)
666 #include <petsc/private/deviceimpl.h>
667 #  if PetscDefined(HAVE_CUDA)
668 // REMOVE ME
669 cudaStream_t PetscDefaultCudaStream = NULL;
670 #  endif
671 #  if PetscDefined(HAVE_HIP)
672 // REMOVE ME
673 hipStream_t PetscDefaultHipStream = NULL;
674 #  endif
675 #endif
676 
677 #if PetscDefined(HAVE_DLFCN_H)
678 #include <dlfcn.h>
679 #endif
680 #if PetscDefined(USE_LOG)
681 PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
682 #endif
683 #if PetscDefined(HAVE_VIENNACL)
684 PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
685 PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
686 #endif
687 
688 /*
689   PetscInitialize_Common  - shared code between C and Fortran initialization
690 
691   prog:     program name
692   file:     optional PETSc database file name. Might be in Fortran string format when 'ftn' is true
693   help:     program help message
694   ftn:      is it called from Fortran initilization (petscinitializef_)?
695   readarguments,len: used when fortran is true
696 */
697 PETSC_INTERN PetscErrorCode PetscInitialize_Common(const char* prog,const char* file,const char *help,PetscBool ftn,PetscBool readarguments,PetscInt len)
698 {
699   PetscErrorCode ierr;
700   PetscMPIInt    size;
701   PetscBool      flg = PETSC_TRUE;
702   char           hostname[256];
703 
704   PetscFunctionBegin;
705   if (PetscInitializeCalled) PetscFunctionReturn(0);
706   /*
707       The checking over compatible runtime libraries is complicated by the MPI ABI initiative
708       https://wiki.mpich.org/mpich/index.php/ABI_Compatibility_Initiative which started with
709         MPICH v3.1 (Released February 2014)
710         IBM MPI v2.1 (December 2014)
711         Intel MPI Library v5.0 (2014)
712         Cray MPT v7.0.0 (June 2014)
713       As of July 31, 2017 the ABI number still appears to be 12, that is all of the versions
714       listed above and since that time are compatible.
715 
716       Unfortunately the MPI ABI initiative has not defined a way to determine the ABI number
717       at compile time or runtime. Thus we will need to systematically track the allowed versions
718       and how they are represented in the mpi.h and MPI_Get_library_version() output in order
719       to perform the checking.
720 
721       Currently we only check for pre MPI ABI versions (and packages that do not follow the MPI ABI).
722 
723       Questions:
724 
725         Should the checks for ABI incompatibility be only on the major version number below?
726         Presumably the output to stderr will be removed before a release.
727   */
728 
729 #if defined(PETSC_HAVE_MPI_GET_LIBRARY_VERSION)
730   {
731     char        mpilibraryversion[MPI_MAX_LIBRARY_VERSION_STRING];
732     PetscMPIInt mpilibraryversionlength;
733     ierr = MPI_Get_library_version(mpilibraryversion,&mpilibraryversionlength);
734     if (ierr) PetscFunctionReturn(ierr);
735     /* check for MPICH versions before MPI ABI initiative */
736 #if defined(MPICH_VERSION)
737 #if MPICH_NUMVERSION < 30100000
738     {
739       char      *ver,*lf;
740       PetscBool flg = PETSC_FALSE;
741       ierr = PetscStrstr(mpilibraryversion,"MPICH Version:",&ver);
742       if (ierr) PetscFunctionReturn(ierr);
743       else if (ver) {
744         ierr = PetscStrchr(ver,'\n',&lf);
745         if (ierr) PetscFunctionReturn(ierr);
746         else if (lf) {
747           *lf = 0;
748           ierr = PetscStrendswith(ver,MPICH_VERSION,&flg);if (ierr) PetscFunctionReturn(ierr);
749         }
750       }
751       if (!flg) {
752         PetscInfo1(NULL,"PETSc warning --- MPICH library version \n%s does not match what PETSc was compiled with %.\n",mpilibraryversion,MPICH_VESION);
753         flg = PETSC_TRUE;
754       }
755     }
756 #endif
757     /* check for OpenMPI version, it is not part of the MPI ABI initiative (is it part of another initiative that needs to be handled?) */
758 #elif defined(OMPI_MAJOR_VERSION)
759     {
760       char *ver,bs[MPI_MAX_LIBRARY_VERSION_STRING],*bsf;
761       PetscBool flg = PETSC_FALSE;
762 #define PSTRSZ 2
763       char ompistr1[PSTRSZ][MPI_MAX_LIBRARY_VERSION_STRING] = {"Open MPI","FUJITSU MPI"};
764       char ompistr2[PSTRSZ][MPI_MAX_LIBRARY_VERSION_STRING] = {"v","Library "};
765       int i;
766       for (i=0; i<PSTRSZ; i++) {
767         ierr = PetscStrstr(mpilibraryversion,ompistr1[i],&ver);
768         if (ierr) PetscFunctionReturn(ierr);
769         else if (ver) {
770           PetscSNPrintf(bs,MPI_MAX_LIBRARY_VERSION_STRING,"%s%d.%d",ompistr2[i],OMPI_MAJOR_VERSION,OMPI_MINOR_VERSION);
771           ierr = PetscStrstr(ver,bs,&bsf);
772           if (ierr) PetscFunctionReturn(ierr);
773           else if (bsf) flg = PETSC_TRUE;
774           break;
775         }
776       }
777       if (!flg) {
778         PetscInfo3(NULL,"PETSc warning --- Open MPI library version \n%s does not match what PETSc was compiled with %d.%d.\n",mpilibraryversion,OMPI_MAJOR_VERSION,OMPI_MINOR_VERSION);
779         flg = PETSC_TRUE;
780       }
781     }
782 #endif
783   }
784 #endif
785 
786 #if defined(PETSC_HAVE_DLSYM)
787   /* These symbols are currently in the OpenMPI and MPICH libraries; they may not always be, in that case the test will simply not detect the problem */
788   if (PetscUnlikely(dlsym(RTLD_DEFAULT,"ompi_mpi_init") && dlsym(RTLD_DEFAULT,"MPID_Abort"))) {
789     fprintf(stderr,"PETSc Error --- Application was linked against both OpenMPI and MPICH based MPI libraries and will not run correctly\n");
790     ierr = PetscStackView(stderr);CHKERRQ(ierr);
791     PetscFunctionReturn(PETSC_ERR_MPI_LIB_INCOMP);
792   }
793 #endif
794 
795   /* these must be initialized in a routine, not as a constant declaration*/
796   PETSC_STDOUT = stdout;
797   PETSC_STDERR = stderr;
798 
799   /*CHKERRQ can be used from now */
800   PetscErrorHandlingInitialized = PETSC_TRUE;
801 
802   /* on Windows - set printf to default to printing 2 digit exponents */
803 #if defined(PETSC_HAVE__SET_OUTPUT_FORMAT)
804   _set_output_format(_TWO_DIGIT_EXPONENT);
805 #endif
806 
807   ierr = PetscOptionsCreateDefault();CHKERRQ(ierr);
808 
809   PetscFinalizeCalled = PETSC_FALSE;
810 
811   ierr = PetscSetProgramName(prog);CHKERRQ(ierr);
812   ierr = PetscSpinlockCreate(&PetscViewerASCIISpinLockOpen);CHKERRQ(ierr);
813   ierr = PetscSpinlockCreate(&PetscViewerASCIISpinLockStdout);CHKERRQ(ierr);
814   ierr = PetscSpinlockCreate(&PetscViewerASCIISpinLockStderr);CHKERRQ(ierr);
815   ierr = PetscSpinlockCreate(&PetscCommSpinLock);CHKERRQ(ierr);
816 
817   if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD;
818   ierr = MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);CHKERRMPI(ierr);
819 
820   if (PETSC_MPI_ERROR_CLASS == MPI_ERR_LASTCODE) {
821     ierr = MPI_Add_error_class(&PETSC_MPI_ERROR_CLASS);CHKERRMPI(ierr);
822     ierr = MPI_Add_error_code(PETSC_MPI_ERROR_CLASS,&PETSC_MPI_ERROR_CODE);CHKERRMPI(ierr);
823   }
824 
825   /* Done after init due to a bug in MPICH-GM? */
826   ierr = PetscErrorPrintfInitialize();CHKERRQ(ierr);
827 
828   ierr = MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);CHKERRMPI(ierr);
829   ierr = MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);CHKERRMPI(ierr);
830 
831   MPIU_BOOL = MPI_INT;
832   MPIU_ENUM = MPI_INT;
833   MPIU_FORTRANADDR = (sizeof(void*) == sizeof(int)) ? MPI_INT : MPIU_INT64;
834   if (sizeof(size_t) == sizeof(unsigned)) MPIU_SIZE_T = MPI_UNSIGNED;
835   else if (sizeof(size_t) == sizeof(unsigned long)) MPIU_SIZE_T = MPI_UNSIGNED_LONG;
836 #if defined(PETSC_SIZEOF_LONG_LONG)
837   else if (sizeof(size_t) == sizeof(unsigned long long)) MPIU_SIZE_T = MPI_UNSIGNED_LONG_LONG;
838 #endif
839   else {
840     (*PetscErrorPrintf)("PetscInitialize_Common: Could not find MPI type for size_t\n");
841     PetscFunctionReturn(PETSC_ERR_SUP_SYS);
842   }
843 
844   /*
845      Initialized the global complex variable; this is because with
846      shared libraries the constructors for global variables
847      are not called; at least on IRIX.
848   */
849 #if defined(PETSC_HAVE_COMPLEX)
850   {
851 #if defined(PETSC_CLANGUAGE_CXX) && !defined(PETSC_USE_REAL___FLOAT128)
852     PetscComplex ic(0.0,1.0);
853     PETSC_i = ic;
854 #else
855     PETSC_i = _Complex_I;
856 #endif
857   }
858 #endif /* PETSC_HAVE_COMPLEX */
859 
860   /*
861      Create the PETSc MPI reduction operator that sums of the first
862      half of the entries and maxes the second half.
863   */
864   ierr = MPI_Op_create(MPIU_MaxSum_Local,1,&MPIU_MAXSUM_OP);CHKERRMPI(ierr);
865 
866 #if defined(PETSC_USE_REAL___FLOAT128)
867   ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);CHKERRMPI(ierr);
868   ierr = MPI_Type_commit(&MPIU___FLOAT128);CHKERRMPI(ierr);
869 #if defined(PETSC_HAVE_COMPLEX)
870   ierr = MPI_Type_contiguous(4,MPI_DOUBLE,&MPIU___COMPLEX128);CHKERRMPI(ierr);
871   ierr = MPI_Type_commit(&MPIU___COMPLEX128);CHKERRMPI(ierr);
872 #endif
873   ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);CHKERRMPI(ierr);
874   ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);CHKERRMPI(ierr);
875 #elif defined(PETSC_USE_REAL___FP16)
876   ierr = MPI_Type_contiguous(2,MPI_CHAR,&MPIU___FP16);CHKERRMPI(ierr);
877   ierr = MPI_Type_commit(&MPIU___FP16);CHKERRMPI(ierr);
878   ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);CHKERRMPI(ierr);
879   ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);CHKERRMPI(ierr);
880 #endif
881 
882 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
883   ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);CHKERRMPI(ierr);
884 #endif
885 
886   ierr = MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);CHKERRMPI(ierr);
887   ierr = MPI_Type_commit(&MPIU_2SCALAR);CHKERRMPI(ierr);
888 
889   /* create datatypes used by MPIU_MAXLOC, MPIU_MINLOC and PetscSplitReduction_Op */
890 #if !defined(PETSC_HAVE_MPIUNI)
891   {
892     struct PetscRealInt { PetscReal v; PetscInt i; };
893     PetscMPIInt  blockSizes[2] = {1,1};
894     MPI_Aint     blockOffsets[2] = {offsetof(struct PetscRealInt,v),offsetof(struct PetscRealInt,i)};
895     MPI_Datatype blockTypes[2] = {MPIU_REAL,MPIU_INT}, tmpStruct;
896 
897     ierr = MPI_Type_create_struct(2,blockSizes,blockOffsets,blockTypes,&tmpStruct);CHKERRMPI(ierr);
898     ierr = MPI_Type_create_resized(tmpStruct,0,sizeof(struct PetscRealInt),&MPIU_REAL_INT);CHKERRMPI(ierr);
899     ierr = MPI_Type_free(&tmpStruct);CHKERRMPI(ierr);
900     ierr = MPI_Type_commit(&MPIU_REAL_INT);CHKERRMPI(ierr);
901   }
902   {
903     struct PetscScalarInt { PetscScalar v; PetscInt i; };
904     PetscMPIInt  blockSizes[2] = {1,1};
905     MPI_Aint     blockOffsets[2] = {offsetof(struct PetscScalarInt,v),offsetof(struct PetscScalarInt,i)};
906     MPI_Datatype blockTypes[2] = {MPIU_SCALAR,MPIU_INT}, tmpStruct;
907 
908     ierr = MPI_Type_create_struct(2,blockSizes,blockOffsets,blockTypes,&tmpStruct);CHKERRMPI(ierr);
909     ierr = MPI_Type_create_resized(tmpStruct,0,sizeof(struct PetscScalarInt),&MPIU_SCALAR_INT);CHKERRMPI(ierr);
910     ierr = MPI_Type_free(&tmpStruct);CHKERRMPI(ierr);
911     ierr = MPI_Type_commit(&MPIU_SCALAR_INT);CHKERRMPI(ierr);
912   }
913 #endif
914 
915 #if defined(PETSC_USE_64BIT_INDICES)
916   ierr = MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);CHKERRMPI(ierr);
917   ierr = MPI_Type_commit(&MPIU_2INT);CHKERRMPI(ierr);
918 #endif
919   ierr = MPI_Type_contiguous(4,MPI_INT,&MPI_4INT);CHKERRMPI(ierr);
920   ierr = MPI_Type_commit(&MPI_4INT);CHKERRMPI(ierr);
921   ierr = MPI_Type_contiguous(4,MPIU_INT,&MPIU_4INT);CHKERRMPI(ierr);
922   ierr = MPI_Type_commit(&MPIU_4INT);CHKERRMPI(ierr);
923 
924   /*
925      Attributes to be set on PETSc communicators
926   */
927   ierr = MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_Counter_Attr_Delete_Fn,&Petsc_Counter_keyval,(void*)0);CHKERRMPI(ierr);
928   ierr = MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_InnerComm_Attr_Delete_Fn,&Petsc_InnerComm_keyval,(void*)0);CHKERRMPI(ierr);
929   ierr = MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_OuterComm_Attr_Delete_Fn,&Petsc_OuterComm_keyval,(void*)0);CHKERRMPI(ierr);
930   ierr = MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_ShmComm_Attr_Delete_Fn,&Petsc_ShmComm_keyval,(void*)0);CHKERRMPI(ierr);
931 
932 #if defined(PETSC_HAVE_FORTRAN)
933   if (ftn) {ierr = PetscInitFortran_Private(readarguments,file,len);CHKERRQ(ierr);}
934   else
935 #endif
936   {ierr = PetscOptionsInsert(NULL,&PetscGlobalArgc,&PetscGlobalArgs,file);CHKERRQ(ierr);}
937 
938   /* call a second time so it can look in the options database */
939   ierr = PetscErrorPrintfInitialize();CHKERRQ(ierr);
940 
941   /*
942      Check system options and print help
943   */
944   ierr = PetscOptionsCheckInitial_Private(help);CHKERRQ(ierr);
945 
946   /*
947    Initialize PetscDevice and PetscDeviceContext
948 
949    Note to any future devs thinking of moving this, proper initialization requires:
950    1. MPI initialized
951    2. Options DB initialized
952    3. Petsc error handling initialized, specifically signal handlers. This expects to set up its own SIGSEV handler via
953       the push/pop interface.
954   */
955 #if (PetscDefined(HAVE_CUDA) || PetscDefined(HAVE_HIP) || PetscDefined(HAVE_SYCL))
956   ierr = PetscDeviceInitializeFromOptions_Internal(PETSC_COMM_WORLD);CHKERRQ(ierr);
957 #endif
958 
959 #if PetscDefined(HAVE_VIENNACL)
960   flg = PETSC_FALSE;
961   ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg);CHKERRQ(ierr);
962   if (!flg) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg);CHKERRQ(ierr);}
963   if (!flg) {ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg,NULL);CHKERRQ(ierr);}
964   PetscViennaCLSynchronize = flg;
965   ierr = PetscViennaCLInit();CHKERRQ(ierr);
966 #endif
967 
968   /*
969      Creates the logging data structures; this is enabled even if logging is not turned on
970      This is the last thing we do before returning to the user code to prevent having the
971      logging numbers contaminated by any startup time associated with MPI
972   */
973 #if defined(PETSC_USE_LOG)
974   ierr = PetscLogInitialize();CHKERRQ(ierr);
975 #endif
976 
977   ierr = PetscCitationsInitialize();CHKERRQ(ierr);
978 
979 #if defined(PETSC_HAVE_SAWS)
980   ierr = PetscInitializeSAWs(ftn ? NULL : help);CHKERRQ(ierr);
981   flg = PETSC_FALSE;
982   ierr = PetscOptionsHasName(NULL,NULL,"-stack_view",&flg);CHKERRQ(ierr);
983   if (flg) PetscStackViewSAWs();
984 #endif
985 
986   /*
987      Load the dynamic libraries (on machines that support them), this registers all
988      the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
989   */
990   ierr = PetscInitialize_DynamicLibraries();CHKERRQ(ierr);
991 
992   ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr);
993   ierr = PetscInfo1(NULL,"PETSc successfully started: number of processors = %d\n",size);CHKERRQ(ierr);
994   ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr);
995   ierr = PetscInfo1(NULL,"Running on machine: %s\n",hostname);CHKERRQ(ierr);
996 #if defined(PETSC_HAVE_OPENMP)
997   {
998     PetscBool omp_view_flag;
999     char      *threads = getenv("OMP_NUM_THREADS");
1000 
1001     if (threads) {
1002       ierr = PetscInfo1(NULL,"Number of OpenMP threads %s (as given by OMP_NUM_THREADS)\n",threads);CHKERRQ(ierr);
1003       (void) sscanf(threads, "%" PetscInt_FMT,&PetscNumOMPThreads);
1004     } else {
1005       PetscNumOMPThreads = (PetscInt) omp_get_max_threads();
1006       ierr = PetscInfo1(NULL,"Number of OpenMP threads %" PetscInt_FMT " (as given by omp_get_max_threads())\n",PetscNumOMPThreads);CHKERRQ(ierr);
1007     }
1008     ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"OpenMP options","Sys");CHKERRQ(ierr);
1009     ierr = PetscOptionsInt("-omp_num_threads","Number of OpenMP threads to use (can also use environmental variable OMP_NUM_THREADS","None",PetscNumOMPThreads,&PetscNumOMPThreads,&flg);CHKERRQ(ierr);
1010     ierr = PetscOptionsName("-omp_view","Display OpenMP number of threads",NULL,&omp_view_flag);CHKERRQ(ierr);
1011     ierr = PetscOptionsEnd();CHKERRQ(ierr);
1012     if (flg) {
1013       ierr = PetscInfo1(NULL,"Number of OpenMP theads %" PetscInt_FMT " (given by -omp_num_threads)\n",PetscNumOMPThreads);CHKERRQ(ierr);
1014       omp_set_num_threads((int)PetscNumOMPThreads);
1015     }
1016     if (omp_view_flag) {
1017       ierr = PetscPrintf(PETSC_COMM_WORLD,"OpenMP: number of threads %" PetscInt_FMT "\n",PetscNumOMPThreads);CHKERRQ(ierr);
1018     }
1019   }
1020 #endif
1021 
1022 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
1023   /*
1024       Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI
1025 
1026       Currently not used because it is not supported by MPICH.
1027   */
1028   if (!PetscBinaryBigEndian()) {
1029     ierr = MPI_Register_datarep((char*)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,NULL);CHKERRMPI(ierr);
1030   }
1031 #endif
1032 
1033 #if defined(PETSC_SERIALIZE_FUNCTIONS)
1034   ierr = PetscFPTCreate(10000);CHKERRQ(ierr);
1035 #endif
1036 
1037 #if defined(PETSC_HAVE_HWLOC)
1038   {
1039     PetscViewer viewer;
1040     ierr = PetscOptionsGetViewer(PETSC_COMM_WORLD,NULL,NULL,"-process_view",&viewer,NULL,&flg);CHKERRQ(ierr);
1041     if (flg) {
1042       ierr = PetscProcessPlacementView(viewer);CHKERRQ(ierr);
1043       ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1044     }
1045   }
1046 #endif
1047 
1048   flg  = PETSC_TRUE;
1049   ierr = PetscOptionsGetBool(NULL,NULL,"-viewfromoptions",&flg,NULL);CHKERRQ(ierr);
1050   if (!flg) {ierr = PetscOptionsPushGetViewerOff(PETSC_TRUE);CHKERRQ(ierr);}
1051 
1052 #if defined(PETSC_HAVE_ADIOS)
1053   ierr = adios_init_noxml(PETSC_COMM_WORLD);CHKERRQ(ierr);
1054   ierr = adios_declare_group(&Petsc_adios_group,"PETSc","",adios_stat_default);CHKERRQ(ierr);
1055   ierr = adios_select_method(Petsc_adios_group,"MPI","","");CHKERRQ(ierr);
1056   ierr = adios_read_init_method(ADIOS_READ_METHOD_BP,PETSC_COMM_WORLD,"");CHKERRQ(ierr);
1057 #endif
1058 
1059 #if defined(__VALGRIND_H)
1060   PETSC_RUNNING_ON_VALGRIND = RUNNING_ON_VALGRIND? PETSC_TRUE: PETSC_FALSE;
1061 #if defined(PETSC_USING_DARWIN) && defined(PETSC_BLASLAPACK_SDOT_RETURNS_DOUBLE)
1062   if (PETSC_RUNNING_ON_VALGRIND) {
1063     ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING: Running valgrind with the MacOS native BLAS and LAPACK can fail. If it fails suggest configuring with --download-fblaslapack or --download-f2cblaslapack");CHKERRQ(ierr);
1064     }
1065 #endif
1066 #endif
1067   /*
1068       Set flag that we are completely initialized
1069   */
1070   PetscInitializeCalled = PETSC_TRUE;
1071 
1072   ierr = PetscOptionsHasName(NULL,NULL,"-python",&flg);CHKERRQ(ierr);
1073   if (flg) {ierr = PetscPythonInitialize(NULL,NULL);CHKERRQ(ierr);}
1074   PetscFunctionReturn(0);
1075 }
1076 
1077 /*@C
1078    PetscInitialize - Initializes the PETSc database and MPI.
1079    PetscInitialize() calls MPI_Init() if that has yet to be called,
1080    so this routine should always be called near the beginning of
1081    your program -- usually the very first line!
1082 
1083    Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set
1084 
1085    Input Parameters:
1086 +  argc - count of number of command line arguments
1087 .  args - the command line arguments
1088 .  file - [optional] PETSc database file, append ":yaml" to filename to specify YAML options format.
1089           Use NULL or empty string to not check for code specific file.
1090           Also checks ~/.petscrc, .petscrc and petscrc.
1091           Use -skip_petscrc in the code specific file (or command line) to skip ~/.petscrc, .petscrc and petscrc files.
1092 -  help - [optional] Help message to print, use NULL for no message
1093 
1094    If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that
1095    communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a
1096    four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not,
1097    then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even
1098    if different subcommunicators of the job are doing different things with PETSc.
1099 
1100    Options Database Keys:
1101 +  -help [intro] - prints help method for each option; if intro is given the program stops after printing the introductory help message
1102 .  -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
1103 .  -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
1104 .  -on_error_emacs <machinename> - causes emacsclient to jump to error file
1105 .  -on_error_abort - calls abort() when error detected (no traceback)
1106 .  -on_error_mpiabort - calls MPI_abort() when error detected
1107 .  -error_output_stderr - prints error messages to stderr instead of the default stdout
1108 .  -error_output_none - does not print the error messages (but handles errors in the same way as if this was not called)
1109 .  -debugger_ranks [rank1,rank2,...] - Indicates ranks to start in debugger
1110 .  -debugger_pause [sleeptime] (in seconds) - Pauses debugger
1111 .  -stop_for_debugger - Print message on how to attach debugger manually to
1112                         process and wait (-debugger_pause) seconds for attachment
1113 .  -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries) (deprecated, use -malloc_debug)
1114 .  -malloc no - Indicates not to use error-checking malloc (deprecated, use -malloc_debug no)
1115 .  -malloc_debug - check for memory corruption at EVERY malloc or free, see PetscMallocSetDebug()
1116 .  -malloc_dump - prints a list of all unfreed memory at the end of the run
1117 .  -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds, ignored in optimized build. May want to set in PETSC_OPTIONS environmental variable
1118 .  -malloc_view - show a list of all allocated memory during PetscFinalize()
1119 .  -malloc_view_threshold <t> - only list memory allocations of size greater than t with -malloc_view
1120 .  -malloc_requested_size - malloc logging will record the requested size rather than size after alignment
1121 .  -fp_trap - Stops on floating point exceptions
1122 .  -no_signal_handler - Indicates not to trap error signals
1123 .  -shared_tmp - indicates /tmp directory is shared by all processors
1124 .  -not_shared_tmp - each processor has own /tmp
1125 .  -tmp - alternative name of /tmp directory
1126 .  -get_total_flops - returns total flops done by all processors
1127 -  -memory_view - Print memory usage at end of run
1128 
1129    Options Database Keys for Option Database:
1130 +  -skip_petscrc - skip the default option files ~/.petscrc, .petscrc, petscrc
1131 .  -options_monitor - monitor all set options to standard output for the whole program run
1132 -  -options_monitor_cancel - cancel options monitoring hard-wired using PetscOptionsMonitorSet()
1133 
1134    Options -options_monitor_{all,cancel} are
1135    position-independent and apply to all options set since the PETSc start.
1136    They can be used also in option files.
1137 
1138    See PetscOptionsMonitorSet() to do monitoring programmatically.
1139 
1140    Options Database Keys for Profiling:
1141    See Users-Manual: ch_profiling for details.
1142 +  -info [filename][:[~]<list,of,classnames>[:[~]self]] - Prints verbose information. See PetscInfo().
1143 .  -log_sync - Enable barrier synchronization for all events. This option is useful to debug imbalance within each event,
1144         however it slows things down and gives a distorted view of the overall runtime.
1145 .  -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program
1146         hangs without running in the debugger).  See PetscLogTraceBegin().
1147 .  -log_view [:filename:format] - Prints summary of flop and timing information to screen or file, see PetscLogView().
1148 .  -log_view_memory - Includes in the summary from -log_view the memory used in each method, see PetscLogView().
1149 .  -log_summary [filename] - (Deprecated, use -log_view) Prints summary of flop and timing information to screen. If the filename is specified the
1150         summary is written to the file.  See PetscLogView().
1151 .  -log_exclude: <vec,mat,pc,ksp,snes> - excludes subset of object classes from logging
1152 .  -log_all [filename] - Logs extensive profiling information  See PetscLogDump().
1153 .  -log [filename] - Logs basic profiline information  See PetscLogDump().
1154 .  -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution)
1155 .  -viewfromoptions on,off - Enable or disable XXXSetFromOptions() calls, for applications with many small solves turn this off
1156 -  -check_pointer_intensity 0,1,2 - if pointers are checked for validity (debug version only), using 0 will result in faster code
1157 
1158     Only one of -log_trace, -log_view, -log_view, -log_all, -log, or -log_mpe may be used at a time
1159 
1160    Options Database Keys for SAWs:
1161 +  -saws_port <portnumber> - port number to publish SAWs data, default is 8080
1162 .  -saws_port_auto_select - have SAWs select a new unique port number where it publishes the data, the URL is printed to the screen
1163                             this is useful when you are running many jobs that utilize SAWs at the same time
1164 .  -saws_log <filename> - save a log of all SAWs communication
1165 .  -saws_https <certificate file> - have SAWs use HTTPS instead of HTTP
1166 -  -saws_root <directory> - allow SAWs to have access to the given directory to search for requested resources and files
1167 
1168    Environmental Variables:
1169 +   PETSC_TMP - alternative tmp directory
1170 .   PETSC_SHARED_TMP - tmp is shared by all processes
1171 .   PETSC_NOT_SHARED_TMP - each process has its own private tmp
1172 .   PETSC_OPTIONS - a string containing additional options for petsc in the form of command line "-key value" pairs
1173 .   PETSC_OPTIONS_YAML - (requires configuring PETSc to use libyaml) a string containing additional options for petsc in the form of a YAML document
1174 .   PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer
1175 -   PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to
1176 
1177    Level: beginner
1178 
1179    Notes:
1180    If for some reason you must call MPI_Init() separately, call
1181    it before PetscInitialize().
1182 
1183    Fortran Version:
1184    In Fortran this routine has the format
1185 $       call PetscInitialize(file,ierr)
1186 
1187 +  ierr - error return code
1188 -  file - [optional] PETSc database file, also checks ~/.petscrc, .petscrc and petscrc.
1189           Use PETSC_NULL_CHARACTER to not check for code specific file.
1190           Use -skip_petscrc in the code specific file (or command line) to skip ~/.petscrc, .petscrc and petscrc files.
1191 
1192    Important Fortran Note:
1193    In Fortran, you MUST use PETSC_NULL_CHARACTER to indicate a
1194    null character string; you CANNOT just use NULL as
1195    in the C version. See Users-Manual: ch_fortran for details.
1196 
1197    If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after
1198    calling PetscInitialize().
1199 
1200 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments()
1201 
1202 @*/
1203 PetscErrorCode  PetscInitialize(int *argc,char ***args,const char file[],const char help[])
1204 {
1205   PetscErrorCode ierr;
1206   PetscMPIInt    flag;
1207   const char     *prog = "Unknown Name";
1208 
1209   PetscFunctionBegin;
1210   if (PetscInitializeCalled) PetscFunctionReturn(0);
1211   ierr = MPI_Initialized(&flag);CHKERRMPI(ierr);
1212   if (!flag) {
1213     if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first");
1214     ierr = PetscPreMPIInit_Private();CHKERRQ(ierr);
1215 #if defined(PETSC_HAVE_MPI_INIT_THREAD)
1216     {
1217       PetscMPIInt provided;
1218       ierr = MPI_Init_thread(argc,args,PETSC_MPI_THREAD_REQUIRED,&provided);CHKERRMPI(ierr);
1219     }
1220 #else
1221     ierr = MPI_Init(argc,args);CHKERRMPI(ierr);
1222 #endif
1223     PetscBeganMPI = PETSC_TRUE;
1224   }
1225 
1226   if (argc && *argc) prog = **args;
1227   if (argc && args) {
1228     PetscGlobalArgc = *argc;
1229     PetscGlobalArgs = *args;
1230   }
1231   ierr = PetscInitialize_Common(prog,file,help,PETSC_FALSE/*C*/,PETSC_FALSE,0);CHKERRQ(ierr);
1232   PetscFunctionReturn(0);
1233 }
1234 
1235 #if PetscDefined(USE_LOG)
1236 PETSC_INTERN PetscObject *PetscObjects;
1237 PETSC_INTERN PetscInt    PetscObjectsCounts;
1238 PETSC_INTERN PetscInt    PetscObjectsMaxCounts;
1239 PETSC_INTERN PetscBool   PetscObjectsLog;
1240 #endif
1241 
1242 /*
1243     Frees all the MPI types and operations that PETSc may have created
1244 */
1245 PetscErrorCode  PetscFreeMPIResources(void)
1246 {
1247   PetscErrorCode ierr;
1248 
1249   PetscFunctionBegin;
1250 #if defined(PETSC_USE_REAL___FLOAT128)
1251   ierr = MPI_Type_free(&MPIU___FLOAT128);CHKERRMPI(ierr);
1252 #if defined(PETSC_HAVE_COMPLEX)
1253   ierr = MPI_Type_free(&MPIU___COMPLEX128);CHKERRMPI(ierr);
1254 #endif
1255   ierr = MPI_Op_free(&MPIU_MAX);CHKERRMPI(ierr);
1256   ierr = MPI_Op_free(&MPIU_MIN);CHKERRMPI(ierr);
1257 #elif defined(PETSC_USE_REAL___FP16)
1258   ierr = MPI_Type_free(&MPIU___FP16);CHKERRMPI(ierr);
1259   ierr = MPI_Op_free(&MPIU_MAX);CHKERRMPI(ierr);
1260   ierr = MPI_Op_free(&MPIU_MIN);CHKERRMPI(ierr);
1261 #endif
1262 
1263 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
1264   ierr = MPI_Op_free(&MPIU_SUM);CHKERRMPI(ierr);
1265 #endif
1266 
1267   ierr = MPI_Type_free(&MPIU_2SCALAR);CHKERRMPI(ierr);
1268   ierr = MPI_Type_free(&MPIU_REAL_INT);CHKERRMPI(ierr);
1269   ierr = MPI_Type_free(&MPIU_SCALAR_INT);CHKERRMPI(ierr);
1270 #if defined(PETSC_USE_64BIT_INDICES)
1271   ierr = MPI_Type_free(&MPIU_2INT);CHKERRMPI(ierr);
1272 #endif
1273   ierr = MPI_Type_free(&MPI_4INT);CHKERRMPI(ierr);
1274   ierr = MPI_Type_free(&MPIU_4INT);CHKERRMPI(ierr);
1275   ierr = MPI_Op_free(&MPIU_MAXSUM_OP);CHKERRMPI(ierr);
1276   PetscFunctionReturn(0);
1277 }
1278 
1279 #if PetscDefined(USE_LOG)
1280 PETSC_INTERN PetscErrorCode PetscLogFinalize(void);
1281 #endif
1282 
1283 /*@C
1284    PetscFinalize - Checks for options to be called at the conclusion
1285    of the program. MPI_Finalize() is called only if the user had not
1286    called MPI_Init() before calling PetscInitialize().
1287 
1288    Collective on PETSC_COMM_WORLD
1289 
1290    Options Database Keys:
1291 +  -options_view - Calls PetscOptionsView()
1292 .  -options_left - Prints unused options that remain in the database
1293 .  -objects_dump [all] - Prints list of objects allocated by the user that have not been freed, the option all cause all outstanding objects to be listed
1294 .  -mpidump - Calls PetscMPIDump()
1295 .  -malloc_dump <optional filename> - Calls PetscMallocDump(), displays all memory allocated that has not been freed
1296 .  -malloc_info - Prints total memory usage
1297 -  -malloc_view <optional filename> - Prints list of all memory allocated and where
1298 
1299    Level: beginner
1300 
1301    Note:
1302    See PetscInitialize() for more general runtime options.
1303 
1304 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscEnd()
1305 @*/
1306 PetscErrorCode  PetscFinalize(void)
1307 {
1308   PetscErrorCode ierr;
1309   PetscMPIInt    rank;
1310   PetscInt       nopt;
1311   PetscBool      flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE;
1312   PetscBool      flg;
1313 #if defined(PETSC_USE_LOG)
1314   char           mname[PETSC_MAX_PATH_LEN];
1315 #endif
1316 
1317   PetscFunctionBegin;
1318   if (PetscUnlikely(!PetscInitializeCalled)) {
1319     fprintf(PETSC_STDOUT,"PetscInitialize() must be called before PetscFinalize()\n");
1320     ierr = PetscStackView(PETSC_STDOUT);CHKERRQ(ierr);
1321     PetscStackClearTop;
1322     return PETSC_ERR_ARG_WRONGSTATE;
1323   }
1324   ierr = PetscInfo(NULL,"PetscFinalize() called\n");CHKERRQ(ierr);
1325 
1326   ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr);
1327 #if defined(PETSC_HAVE_ADIOS)
1328   ierr = adios_read_finalize_method(ADIOS_READ_METHOD_BP_AGGREGATE);CHKERRQ(ierr);
1329   ierr = adios_finalize(rank);CHKERRQ(ierr);
1330 #endif
1331   ierr = PetscOptionsHasName(NULL,NULL,"-citations",&flg);CHKERRQ(ierr);
1332   if (flg) {
1333     char  *cits, filename[PETSC_MAX_PATH_LEN];
1334     FILE  *fd = PETSC_STDOUT;
1335 
1336     ierr = PetscOptionsGetString(NULL,NULL,"-citations",filename,sizeof(filename),NULL);CHKERRQ(ierr);
1337     if (filename[0]) {
1338       ierr = PetscFOpen(PETSC_COMM_WORLD,filename,"w",&fd);CHKERRQ(ierr);
1339     }
1340     ierr = PetscSegBufferGet(PetscCitationsList,1,&cits);CHKERRQ(ierr);
1341     cits[0] = 0;
1342     ierr = PetscSegBufferExtractAlloc(PetscCitationsList,&cits);CHKERRQ(ierr);
1343     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"If you publish results based on this computation please cite the following:\n");CHKERRQ(ierr);
1344     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"===========================================================================\n");CHKERRQ(ierr);
1345     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"%s",cits);CHKERRQ(ierr);
1346     ierr = PetscFPrintf(PETSC_COMM_WORLD,fd,"===========================================================================\n");CHKERRQ(ierr);
1347     ierr = PetscFClose(PETSC_COMM_WORLD,fd);CHKERRQ(ierr);
1348     ierr = PetscFree(cits);CHKERRQ(ierr);
1349   }
1350   ierr = PetscSegBufferDestroy(&PetscCitationsList);CHKERRQ(ierr);
1351 
1352 #if defined(PETSC_HAVE_SSL) && defined(PETSC_USE_SOCKET_VIEWER)
1353   /* TextBelt is run for testing purposes only, please do not use this feature often */
1354   {
1355     PetscInt nmax = 2;
1356     char     **buffs;
1357     ierr = PetscMalloc1(2,&buffs);CHKERRQ(ierr);
1358     ierr = PetscOptionsGetStringArray(NULL,NULL,"-textbelt",buffs,&nmax,&flg1);CHKERRQ(ierr);
1359     if (flg1) {
1360       if (PetscUnlikely(!nmax)) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"-textbelt requires either the phone number or number,\"message\"");
1361       if (nmax == 1) {
1362         ierr = PetscMalloc1(128,&buffs[1]);CHKERRQ(ierr);
1363         ierr = PetscGetProgramName(buffs[1],32);CHKERRQ(ierr);
1364         ierr = PetscStrcat(buffs[1]," has completed");CHKERRQ(ierr);
1365       }
1366       ierr = PetscTextBelt(PETSC_COMM_WORLD,buffs[0],buffs[1],NULL);CHKERRQ(ierr);
1367       ierr = PetscFree(buffs[0]);CHKERRQ(ierr);
1368       ierr = PetscFree(buffs[1]);CHKERRQ(ierr);
1369     }
1370     ierr = PetscFree(buffs);CHKERRQ(ierr);
1371   }
1372   {
1373     PetscInt nmax = 2;
1374     char     **buffs;
1375     ierr = PetscMalloc1(2,&buffs);CHKERRQ(ierr);
1376     ierr = PetscOptionsGetStringArray(NULL,NULL,"-tellmycell",buffs,&nmax,&flg1);CHKERRQ(ierr);
1377     if (flg1) {
1378       if (!nmax) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"-tellmycell requires either the phone number or number,\"message\"");
1379       if (nmax == 1) {
1380         ierr = PetscMalloc1(128,&buffs[1]);CHKERRQ(ierr);
1381         ierr = PetscGetProgramName(buffs[1],32);CHKERRQ(ierr);
1382         ierr = PetscStrcat(buffs[1]," has completed");CHKERRQ(ierr);
1383       }
1384       ierr = PetscTellMyCell(PETSC_COMM_WORLD,buffs[0],buffs[1],NULL);CHKERRQ(ierr);
1385       ierr = PetscFree(buffs[0]);CHKERRQ(ierr);
1386       ierr = PetscFree(buffs[1]);CHKERRQ(ierr);
1387     }
1388     ierr = PetscFree(buffs);CHKERRQ(ierr);
1389   }
1390 #endif
1391 
1392 #if defined(PETSC_SERIALIZE_FUNCTIONS)
1393   ierr = PetscFPTDestroy();CHKERRQ(ierr);
1394 #endif
1395 
1396 #if defined(PETSC_HAVE_SAWS)
1397   flg = PETSC_FALSE;
1398   ierr = PetscOptionsGetBool(NULL,NULL,"-saw_options",&flg,NULL);CHKERRQ(ierr);
1399   if (flg) {
1400     ierr = PetscOptionsSAWsDestroy();CHKERRQ(ierr);
1401   }
1402 #endif
1403 
1404 #if defined(PETSC_HAVE_X)
1405   flg1 = PETSC_FALSE;
1406   ierr = PetscOptionsGetBool(NULL,NULL,"-x_virtual",&flg1,NULL);CHKERRQ(ierr);
1407   if (flg1) {
1408     /*  this is a crude hack, but better than nothing */
1409     ierr = PetscPOpen(PETSC_COMM_WORLD,NULL,"pkill -9 Xvfb","r",NULL);CHKERRQ(ierr);
1410   }
1411 #endif
1412 
1413 #if !defined(PETSC_HAVE_THREADSAFETY)
1414   ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg2,NULL);CHKERRQ(ierr);
1415   if (!flg2) {
1416     flg2 = PETSC_FALSE;
1417     ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg2,NULL);CHKERRQ(ierr);
1418   }
1419   if (flg2) {
1420     ierr = PetscMemoryView(PETSC_VIEWER_STDOUT_WORLD,"Summary of Memory Usage in PETSc\n");CHKERRQ(ierr);
1421   }
1422 #endif
1423 
1424 #if defined(PETSC_USE_LOG)
1425   flg1 = PETSC_FALSE;
1426   ierr = PetscOptionsGetBool(NULL,NULL,"-get_total_flops",&flg1,NULL);CHKERRQ(ierr);
1427   if (flg1) {
1428     PetscLogDouble flops = 0;
1429     ierr = MPI_Reduce(&petsc_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);CHKERRMPI(ierr);
1430     ierr = PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %g\n",flops);CHKERRQ(ierr);
1431   }
1432 #endif
1433 
1434 #if defined(PETSC_USE_LOG)
1435 #if defined(PETSC_HAVE_MPE)
1436   mname[0] = 0;
1437   ierr = PetscOptionsGetString(NULL,NULL,"-log_mpe",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
1438   if (flg1) {
1439     if (mname[0]) {ierr = PetscLogMPEDump(mname);CHKERRQ(ierr);}
1440     else          {ierr = PetscLogMPEDump(0);CHKERRQ(ierr);}
1441   }
1442 #endif
1443 #endif
1444 
1445   /*
1446      Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_().
1447   */
1448   ierr = PetscObjectRegisterDestroyAll();CHKERRQ(ierr);
1449 
1450 #if defined(PETSC_USE_LOG)
1451   ierr = PetscOptionsPushGetViewerOff(PETSC_FALSE);CHKERRQ(ierr);
1452   ierr = PetscLogViewFromOptions();CHKERRQ(ierr);
1453   ierr = PetscOptionsPopGetViewerOff();CHKERRQ(ierr);
1454 
1455   mname[0] = 0;
1456   ierr = PetscOptionsGetString(NULL,NULL,"-log_summary",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
1457   if (flg1) {
1458     PetscViewer viewer;
1459     ierr = (*PetscHelpPrintf)(PETSC_COMM_WORLD,"\n\n WARNING:   -log_summary is being deprecated; switch to -log_view\n\n\n");CHKERRQ(ierr);
1460     if (mname[0]) {
1461       ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr);
1462       ierr = PetscLogView(viewer);CHKERRQ(ierr);
1463       ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1464     } else {
1465       viewer = PETSC_VIEWER_STDOUT_WORLD;
1466       ierr   = PetscViewerPushFormat(viewer,PETSC_VIEWER_DEFAULT);CHKERRQ(ierr);
1467       ierr   = PetscLogView(viewer);CHKERRQ(ierr);
1468       ierr   = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
1469     }
1470   }
1471 
1472   /*
1473      Free any objects created by the last block of code.
1474   */
1475   ierr = PetscObjectRegisterDestroyAll();CHKERRQ(ierr);
1476 
1477   mname[0] = 0;
1478   ierr = PetscOptionsGetString(NULL,NULL,"-log_all",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
1479   ierr = PetscOptionsGetString(NULL,NULL,"-log",mname,sizeof(mname),&flg2);CHKERRQ(ierr);
1480   if (flg1 || flg2) {ierr = PetscLogDump(mname);CHKERRQ(ierr);}
1481 #endif
1482 
1483   flg1 = PETSC_FALSE;
1484   ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr);
1485   if (!flg1) { ierr = PetscPopSignalHandler();CHKERRQ(ierr);}
1486   flg1 = PETSC_FALSE;
1487   ierr = PetscOptionsGetBool(NULL,NULL,"-mpidump",&flg1,NULL);CHKERRQ(ierr);
1488   if (flg1) {
1489     ierr = PetscMPIDump(stdout);CHKERRQ(ierr);
1490   }
1491   flg1 = PETSC_FALSE;
1492   flg2 = PETSC_FALSE;
1493   /* preemptive call to avoid listing this option in options table as unused */
1494   ierr = PetscOptionsHasName(NULL,NULL,"-malloc_dump",&flg1);CHKERRQ(ierr);
1495   ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&flg1);CHKERRQ(ierr);
1496   ierr = PetscOptionsGetBool(NULL,NULL,"-options_view",&flg2,NULL);CHKERRQ(ierr);
1497 
1498   if (flg2) {
1499     PetscViewer viewer;
1500     ierr = PetscViewerCreate(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
1501     ierr = PetscViewerSetType(viewer,PETSCVIEWERASCII);CHKERRQ(ierr);
1502     ierr = PetscOptionsView(NULL,viewer);CHKERRQ(ierr);
1503     ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1504   }
1505 
1506   /* to prevent PETSc -options_left from warning */
1507   ierr = PetscOptionsHasName(NULL,NULL,"-nox",&flg1);CHKERRQ(ierr);
1508   ierr = PetscOptionsHasName(NULL,NULL,"-nox_warning",&flg1);CHKERRQ(ierr);
1509 
1510   flg3 = PETSC_FALSE; /* default value is required */
1511   ierr = PetscOptionsGetBool(NULL,NULL,"-options_left",&flg3,&flg1);CHKERRQ(ierr);
1512   if (PetscUnlikelyDebug(!flg1)) flg3 = PETSC_TRUE;
1513   if (flg3) {
1514     if (!flg2 && flg1) { /* have not yet printed the options */
1515       PetscViewer viewer;
1516       ierr = PetscViewerCreate(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
1517       ierr = PetscViewerSetType(viewer,PETSCVIEWERASCII);CHKERRQ(ierr);
1518       ierr = PetscOptionsView(NULL,viewer);CHKERRQ(ierr);
1519       ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
1520     }
1521     ierr = PetscOptionsAllUsed(NULL,&nopt);CHKERRQ(ierr);
1522     if (nopt) {
1523       ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!\n");CHKERRQ(ierr);
1524       ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!\n");CHKERRQ(ierr);
1525       if (nopt == 1) {
1526         ierr = PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");CHKERRQ(ierr);
1527       } else {
1528         ierr = PetscPrintf(PETSC_COMM_WORLD,"There are %" PetscInt_FMT " unused database options. They are:\n",nopt);CHKERRQ(ierr);
1529       }
1530     } else if (flg3 && flg1) {
1531       ierr = PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");CHKERRQ(ierr);
1532     }
1533     ierr = PetscOptionsLeft(NULL);CHKERRQ(ierr);
1534   }
1535 
1536 #if defined(PETSC_HAVE_SAWS)
1537   if (!PetscGlobalRank) {
1538     ierr = PetscStackSAWsViewOff();CHKERRQ(ierr);
1539     PetscStackCallSAWs(SAWs_Finalize,());
1540   }
1541 #endif
1542 
1543 #if defined(PETSC_USE_LOG)
1544   /*
1545        List all objects the user may have forgot to free
1546   */
1547   if (PetscObjectsLog) {
1548     ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&flg1);CHKERRQ(ierr);
1549     if (flg1) {
1550       MPI_Comm local_comm;
1551       char     string[64];
1552 
1553       ierr = PetscOptionsGetString(NULL,NULL,"-objects_dump",string,sizeof(string),NULL);CHKERRQ(ierr);
1554       ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRMPI(ierr);
1555       ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr);
1556       ierr = PetscObjectsDump(stdout,(string[0] == 'a') ? PETSC_TRUE : PETSC_FALSE);CHKERRQ(ierr);
1557       ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr);
1558       ierr = MPI_Comm_free(&local_comm);CHKERRMPI(ierr);
1559     }
1560   }
1561 #endif
1562 
1563 #if defined(PETSC_USE_LOG)
1564   PetscObjectsCounts    = 0;
1565   PetscObjectsMaxCounts = 0;
1566   ierr = PetscFree(PetscObjects);CHKERRQ(ierr);
1567 #endif
1568 
1569   /*
1570      Destroy any packages that registered a finalize
1571   */
1572   ierr = PetscRegisterFinalizeAll();CHKERRQ(ierr);
1573 
1574 #if defined(PETSC_USE_LOG)
1575   ierr = PetscLogFinalize();CHKERRQ(ierr);
1576 #endif
1577 
1578   /*
1579      Print PetscFunctionLists that have not been properly freed
1580 
1581   ierr = PetscFunctionListPrintAll();CHKERRQ(ierr);
1582   */
1583 
1584   if (petsc_history) {
1585     ierr = PetscCloseHistoryFile(&petsc_history);CHKERRQ(ierr);
1586     petsc_history = NULL;
1587   }
1588   ierr = PetscOptionsHelpPrintedDestroy(&PetscOptionsHelpPrintedSingleton);CHKERRQ(ierr);
1589   ierr = PetscInfoDestroy();CHKERRQ(ierr);
1590 
1591 #if !defined(PETSC_HAVE_THREADSAFETY)
1592   if (!(PETSC_RUNNING_ON_VALGRIND)) {
1593     char fname[PETSC_MAX_PATH_LEN];
1594     char sname[PETSC_MAX_PATH_LEN];
1595     FILE *fd;
1596     int  err;
1597 
1598     flg2 = PETSC_FALSE;
1599     flg3 = PETSC_FALSE;
1600     if (PetscDefined(USE_DEBUG)) {ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg2,NULL);CHKERRQ(ierr);}
1601     ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg3,NULL);CHKERRQ(ierr);
1602     fname[0] = 0;
1603     ierr = PetscOptionsGetString(NULL,NULL,"-malloc_dump",fname,sizeof(fname),&flg1);CHKERRQ(ierr);
1604     if (flg1 && fname[0]) {
1605 
1606       PetscSNPrintf(sname,sizeof(sname),"%s_%d",fname,rank);
1607       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1608       ierr = PetscMallocDump(fd);CHKERRQ(ierr);
1609       err  = fclose(fd);
1610       if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1611     } else if (flg1 || flg2 || flg3) {
1612       MPI_Comm local_comm;
1613 
1614       ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRMPI(ierr);
1615       ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr);
1616       ierr = PetscMallocDump(stdout);CHKERRQ(ierr);
1617       ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr);
1618       ierr = MPI_Comm_free(&local_comm);CHKERRMPI(ierr);
1619     }
1620     fname[0] = 0;
1621     ierr = PetscOptionsGetString(NULL,NULL,"-malloc_view",fname,sizeof(fname),&flg1);CHKERRQ(ierr);
1622     if (flg1 && fname[0]) {
1623 
1624       PetscSNPrintf(sname,sizeof(sname),"%s_%d",fname,rank);
1625       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname);
1626       ierr = PetscMallocView(fd);CHKERRQ(ierr);
1627       err  = fclose(fd);
1628       if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
1629     } else if (flg1) {
1630       MPI_Comm local_comm;
1631 
1632       ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRMPI(ierr);
1633       ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr);
1634       ierr = PetscMallocView(stdout);CHKERRQ(ierr);
1635       ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr);
1636       ierr = MPI_Comm_free(&local_comm);CHKERRMPI(ierr);
1637     }
1638   }
1639 #endif
1640 
1641   /*
1642      Close any open dynamic libraries
1643   */
1644   ierr = PetscFinalize_DynamicLibraries();CHKERRQ(ierr);
1645 
1646   /* Can be destroyed only after all the options are used */
1647   ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr);
1648 
1649   PetscGlobalArgc = 0;
1650   PetscGlobalArgs = NULL;
1651 
1652 #if defined(PETSC_HAVE_KOKKOS)
1653   if (PetscBeganKokkos) {
1654     ierr = PetscKokkosFinalize_Private();CHKERRQ(ierr);
1655     PetscBeganKokkos = PETSC_FALSE;
1656     PetscKokkosInitialized = PETSC_FALSE;
1657   }
1658 #endif
1659 
1660 #if defined(PETSC_HAVE_NVSHMEM)
1661   if (PetscBeganNvshmem) {
1662     ierr = PetscNvshmemFinalize();CHKERRQ(ierr);
1663     PetscBeganNvshmem = PETSC_FALSE;
1664   }
1665 #endif
1666 
1667   ierr = PetscFreeMPIResources();CHKERRQ(ierr);
1668 
1669   /*
1670      Destroy any known inner MPI_Comm's and attributes pointing to them
1671      Note this will not destroy any new communicators the user has created.
1672 
1673      If all PETSc objects were not destroyed those left over objects will have hanging references to
1674      the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again
1675  */
1676   {
1677     PetscCommCounter *counter;
1678     PetscMPIInt      flg;
1679     MPI_Comm         icomm;
1680     union {MPI_Comm comm; void *ptr;} ucomm;
1681     ierr = MPI_Comm_get_attr(PETSC_COMM_SELF,Petsc_InnerComm_keyval,&ucomm,&flg);CHKERRMPI(ierr);
1682     if (flg) {
1683       icomm = ucomm.comm;
1684       ierr = MPI_Comm_get_attr(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRMPI(ierr);
1685       if (PetscUnlikely(!flg)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1686 
1687       ierr = MPI_Comm_delete_attr(PETSC_COMM_SELF,Petsc_InnerComm_keyval);CHKERRMPI(ierr);
1688       ierr = MPI_Comm_delete_attr(icomm,Petsc_Counter_keyval);CHKERRMPI(ierr);
1689       ierr = MPI_Comm_free(&icomm);CHKERRMPI(ierr);
1690     }
1691     ierr = MPI_Comm_get_attr(PETSC_COMM_WORLD,Petsc_InnerComm_keyval,&ucomm,&flg);CHKERRMPI(ierr);
1692     if (flg) {
1693       icomm = ucomm.comm;
1694       ierr = MPI_Comm_get_attr(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRMPI(ierr);
1695       if (PetscUnlikely(!flg)) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1696 
1697       ierr = MPI_Comm_delete_attr(PETSC_COMM_WORLD,Petsc_InnerComm_keyval);CHKERRMPI(ierr);
1698       ierr = MPI_Comm_delete_attr(icomm,Petsc_Counter_keyval);CHKERRMPI(ierr);
1699       ierr = MPI_Comm_free(&icomm);CHKERRMPI(ierr);
1700     }
1701   }
1702 
1703   ierr = MPI_Comm_free_keyval(&Petsc_Counter_keyval);CHKERRMPI(ierr);
1704   ierr = MPI_Comm_free_keyval(&Petsc_InnerComm_keyval);CHKERRMPI(ierr);
1705   ierr = MPI_Comm_free_keyval(&Petsc_OuterComm_keyval);CHKERRMPI(ierr);
1706   ierr = MPI_Comm_free_keyval(&Petsc_ShmComm_keyval);CHKERRMPI(ierr);
1707 
1708   ierr = PetscSpinlockDestroy(&PetscViewerASCIISpinLockOpen);CHKERRQ(ierr);
1709   ierr = PetscSpinlockDestroy(&PetscViewerASCIISpinLockStdout);CHKERRQ(ierr);
1710   ierr = PetscSpinlockDestroy(&PetscViewerASCIISpinLockStderr);CHKERRQ(ierr);
1711   ierr = PetscSpinlockDestroy(&PetscCommSpinLock);CHKERRQ(ierr);
1712 
1713   if (PetscBeganMPI) {
1714 #if defined(PETSC_HAVE_MPI_FINALIZED)
1715     PetscMPIInt flag;
1716     ierr = MPI_Finalized(&flag);CHKERRMPI(ierr);
1717     if (PetscUnlikely(flag)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()");
1718 #endif
1719     ierr = MPI_Finalize();CHKERRMPI(ierr);
1720   }
1721 /*
1722 
1723      Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because
1724    the communicator has some outstanding requests on it. Specifically if the
1725    flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See
1726    src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate()
1727    is never freed as it should be. Thus one may obtain messages of the form
1728    [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the
1729    memory was not freed.
1730 
1731 */
1732   ierr = PetscMallocClear();CHKERRQ(ierr);
1733   ierr = PetscStackReset();CHKERRQ(ierr);
1734 
1735   PetscErrorHandlingInitialized = PETSC_FALSE;
1736   PetscInitializeCalled = PETSC_FALSE;
1737   PetscFinalizeCalled   = PETSC_TRUE;
1738 #if defined(PETSC_USE_GCOV)
1739   /*
1740      flush gcov, otherwise during CI the flushing continues into the next pipeline resulting in git not being able to delete directories since the
1741      gcov files are still being added to the directories as git tries to remove the directories.
1742    */
1743   __gcov_flush();
1744 #endif
1745   /* To match PetscFunctionBegin() at the beginning of this function */
1746   PetscStackClearTop;
1747   return 0;
1748 }
1749 
1750 #if defined(PETSC_MISSING_LAPACK_lsame_)
1751 PETSC_EXTERN int lsame_(char *a,char *b)
1752 {
1753   if (*a == *b) return 1;
1754   if (*a + 32 == *b) return 1;
1755   if (*a - 32 == *b) return 1;
1756   return 0;
1757 }
1758 #endif
1759 
1760 #if defined(PETSC_MISSING_LAPACK_lsame)
1761 PETSC_EXTERN int lsame(char *a,char *b)
1762 {
1763   if (*a == *b) return 1;
1764   if (*a + 32 == *b) return 1;
1765   if (*a - 32 == *b) return 1;
1766   return 0;
1767 }
1768 #endif
1769