1 #define PETSC_DESIRE_FEATURE_TEST_MACROS 2 /* 3 This file defines the initialization of PETSc, including PetscInitialize() 4 */ 5 #include <petsc/private/petscimpl.h> /*I "petscsys.h" I*/ 6 #include <petscviewer.h> 7 #include <petsc/private/garbagecollector.h> 8 9 #if !defined(PETSC_HAVE_WINDOWS_COMPILERS) 10 #include <petsc/private/valgrind/valgrind.h> 11 #endif 12 13 #if defined(PETSC_USE_FORTRAN_BINDINGS) 14 #include <petsc/private/fortranimpl.h> 15 #endif 16 17 #if PetscDefined(USE_COVERAGE) 18 EXTERN_C_BEGIN 19 #if defined(PETSC_HAVE___GCOV_DUMP) 20 #define __gcov_flush(x) __gcov_dump(x) 21 #endif 22 void __gcov_flush(void); 23 EXTERN_C_END 24 #endif 25 26 #if defined(PETSC_SERIALIZE_FUNCTIONS) 27 PETSC_INTERN PetscFPT PetscFPTData; 28 PetscFPT PetscFPTData = 0; 29 #endif 30 31 #if PetscDefined(HAVE_SAWS) 32 #include <petscviewersaws.h> 33 #endif 34 35 PETSC_INTERN FILE *petsc_history; 36 37 PETSC_INTERN PetscErrorCode PetscInitialize_DynamicLibraries(void); 38 PETSC_INTERN PetscErrorCode PetscFinalize_DynamicLibraries(void); 39 PETSC_INTERN PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm, int); 40 PETSC_INTERN PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm, int); 41 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **); 42 43 /* user may set these BEFORE calling PetscInitialize() */ 44 MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL; 45 #if PetscDefined(HAVE_MPI_INIT_THREAD) 46 PetscMPIInt PETSC_MPI_THREAD_REQUIRED = MPI_THREAD_FUNNELED; 47 #else 48 PetscMPIInt PETSC_MPI_THREAD_REQUIRED = 0; 49 #endif 50 51 PetscMPIInt Petsc_Counter_keyval = MPI_KEYVAL_INVALID; 52 PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID; 53 PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID; 54 PetscMPIInt Petsc_ShmComm_keyval = MPI_KEYVAL_INVALID; 55 PetscMPIInt Petsc_CreationIdx_keyval = MPI_KEYVAL_INVALID; 56 PetscMPIInt Petsc_Garbage_HMap_keyval = MPI_KEYVAL_INVALID; 57 58 PetscMPIInt Petsc_SharedWD_keyval = MPI_KEYVAL_INVALID; 59 PetscMPIInt Petsc_SharedTmp_keyval = MPI_KEYVAL_INVALID; 60 61 /* 62 Declare and set all the string names of the PETSc enums 63 */ 64 const char *const PetscBools[] = {"FALSE", "TRUE", "PetscBool", "PETSC_", NULL}; 65 const char *const PetscCopyModes[] = {"COPY_VALUES", "OWN_POINTER", "USE_POINTER", "PetscCopyMode", "PETSC_", NULL}; 66 67 PetscBool PetscPreLoadingUsed = PETSC_FALSE; 68 PetscBool PetscPreLoadingOn = PETSC_FALSE; 69 70 PetscInt PetscHotRegionDepth; 71 72 PetscBool PETSC_RUNNING_ON_VALGRIND = PETSC_FALSE; 73 74 #if defined(PETSC_HAVE_THREADSAFETY) 75 PetscSpinlock PetscViewerASCIISpinLockOpen; 76 PetscSpinlock PetscViewerASCIISpinLockStdout; 77 PetscSpinlock PetscViewerASCIISpinLockStderr; 78 PetscSpinlock PetscCommSpinLock; 79 #endif 80 81 /*@C 82 PetscInitializeNoPointers - Calls PetscInitialize() from C/C++ without the pointers to argc and args 83 84 Collective 85 86 Input Parameters: 87 + argc - number of args 88 . args - array of command line arguments 89 . filename - optional name of the program file, pass `NULL` to ignore 90 - help - optional help, pass `NULL` to ignore 91 92 Level: advanced 93 94 Notes: 95 this is called only by the PETSc Julia interface. Even though it might start MPI it sets the flag to 96 indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to 97 be called multiple times from Julia without the problem of trying to initialize MPI more than once. 98 99 Developer Notes: 100 Turns off PETSc signal handling to allow Julia to manage signals 101 102 .seealso: `PetscInitialize()`, `PetscInitializeFortran()`, `PetscInitializeNoArguments()` 103 */ 104 PetscErrorCode PetscInitializeNoPointers(int argc, char **args, const char *filename, const char *help) 105 { 106 int myargc = argc; 107 char **myargs = args; 108 109 PetscFunctionBegin; 110 PetscCall(PetscInitialize(&myargc, &myargs, filename, help)); 111 PetscCall(PetscPopSignalHandler()); 112 PetscBeganMPI = PETSC_FALSE; 113 PetscFunctionReturn(PETSC_SUCCESS); 114 } 115 116 /* 117 Used by Julia interface to get communicator 118 */ 119 PetscErrorCode PetscGetPETSC_COMM_SELF(MPI_Comm *comm) 120 { 121 PetscFunctionBegin; 122 if (PetscInitializeCalled) PetscValidPointer(comm, 1); 123 *comm = PETSC_COMM_SELF; 124 PetscFunctionReturn(PETSC_SUCCESS); 125 } 126 127 /*@C 128 PetscInitializeNoArguments - Calls `PetscInitialize()` from C/C++ without 129 the command line arguments. 130 131 Collective 132 133 Level: advanced 134 135 .seealso: `PetscInitialize()`, `PetscInitializeFortran()` 136 @*/ 137 PetscErrorCode PetscInitializeNoArguments(void) 138 { 139 int argc = 0; 140 char **args = NULL; 141 142 PetscFunctionBegin; 143 PetscCall(PetscInitialize(&argc, &args, NULL, NULL)); 144 PetscFunctionReturn(PETSC_SUCCESS); 145 } 146 147 /*@ 148 PetscInitialized - Determine whether PETSc is initialized. 149 150 Output Parameter: 151 . isInitialized - `PETSC_TRUE` if PETSc is initialized, `PETSC_FALSE` otherwise 152 153 Level: beginner 154 155 .seealso: `PetscInitialize()`, `PetscInitializeNoArguments()`, `PetscInitializeFortran()` 156 @*/ 157 PetscErrorCode PetscInitialized(PetscBool *isInitialized) 158 { 159 PetscFunctionBegin; 160 if (PetscInitializeCalled) PetscValidBoolPointer(isInitialized, 1); 161 *isInitialized = PetscInitializeCalled; 162 PetscFunctionReturn(PETSC_SUCCESS); 163 } 164 165 /*@ 166 PetscFinalized - Determine whether `PetscFinalize()` has been called yet 167 168 Output Parameter: 169 . isFinalized - `PETSC_TRUE` if PETSc is finalized, `PETSC_FALSE` otherwise 170 171 Level: developer 172 173 .seealso: `PetscInitialize()`, `PetscInitializeNoArguments()`, `PetscInitializeFortran()` 174 @*/ 175 PetscErrorCode PetscFinalized(PetscBool *isFinalized) 176 { 177 PetscFunctionBegin; 178 if (!PetscFinalizeCalled) PetscValidBoolPointer(isFinalized, 1); 179 *isFinalized = PetscFinalizeCalled; 180 PetscFunctionReturn(PETSC_SUCCESS); 181 } 182 183 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char[]); 184 185 /* 186 This function is the MPI reduction operation used to compute the sum of the 187 first half of the datatype and the max of the second half. 188 */ 189 MPI_Op MPIU_MAXSUM_OP = 0; 190 MPI_Op Petsc_Garbage_SetIntersectOp = 0; 191 192 PETSC_INTERN void MPIAPI MPIU_MaxSum_Local(void *in, void *out, int *cnt, MPI_Datatype *datatype) 193 { 194 PetscInt *xin = (PetscInt *)in, *xout = (PetscInt *)out, i, count = *cnt; 195 196 PetscFunctionBegin; 197 if (*datatype != MPIU_2INT) { 198 PetscErrorCode ierr = (*PetscErrorPrintf)("Can only handle MPIU_2INT data types"); 199 (void)ierr; 200 PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG); 201 } 202 203 for (i = 0; i < count; i++) { 204 xout[2 * i] = PetscMax(xout[2 * i], xin[2 * i]); 205 xout[2 * i + 1] += xin[2 * i + 1]; 206 } 207 PetscFunctionReturnVoid(); 208 } 209 210 /* 211 Returns the max of the first entry owned by this processor and the 212 sum of the second entry. 213 214 The reason sizes[2*i] contains lengths sizes[2*i+1] contains flag of 1 if length is nonzero 215 is so that the MPIU_MAXSUM_OP() can set TWO values, if we passed in only sizes[i] with lengths 216 there would be no place to store the both needed results. 217 */ 218 PetscErrorCode PetscMaxSum(MPI_Comm comm, const PetscInt sizes[], PetscInt *max, PetscInt *sum) 219 { 220 PetscFunctionBegin; 221 #if defined(PETSC_HAVE_MPI_REDUCE_SCATTER_BLOCK) 222 { 223 struct { 224 PetscInt max, sum; 225 } work; 226 PetscCallMPI(MPI_Reduce_scatter_block((void *)sizes, &work, 1, MPIU_2INT, MPIU_MAXSUM_OP, comm)); 227 *max = work.max; 228 *sum = work.sum; 229 } 230 #else 231 { 232 PetscMPIInt size, rank; 233 struct { 234 PetscInt max, sum; 235 } *work; 236 PetscCallMPI(MPI_Comm_size(comm, &size)); 237 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 238 PetscCall(PetscMalloc1(size, &work)); 239 PetscCall(MPIU_Allreduce((void *)sizes, work, size, MPIU_2INT, MPIU_MAXSUM_OP, comm)); 240 *max = work[rank].max; 241 *sum = work[rank].sum; 242 PetscCall(PetscFree(work)); 243 } 244 #endif 245 PetscFunctionReturn(PETSC_SUCCESS); 246 } 247 248 #if defined(PETSC_HAVE_REAL___FLOAT128) || defined(PETSC_HAVE_REAL___FP16) 249 #if defined(PETSC_HAVE_REAL___FLOAT128) 250 #include <quadmath.h> 251 #endif 252 MPI_Op MPIU_SUM___FP16___FLOAT128 = 0; 253 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16) 254 MPI_Op MPIU_SUM = 0; 255 #endif 256 257 PETSC_EXTERN void MPIAPI PetscSum_Local(void *in, void *out, PetscMPIInt *cnt, MPI_Datatype *datatype) 258 { 259 PetscInt i, count = *cnt; 260 261 PetscFunctionBegin; 262 if (*datatype == MPIU_REAL) { 263 PetscReal *xin = (PetscReal *)in, *xout = (PetscReal *)out; 264 for (i = 0; i < count; i++) xout[i] += xin[i]; 265 } 266 #if defined(PETSC_HAVE_COMPLEX) 267 else if (*datatype == MPIU_COMPLEX) { 268 PetscComplex *xin = (PetscComplex *)in, *xout = (PetscComplex *)out; 269 for (i = 0; i < count; i++) xout[i] += xin[i]; 270 } 271 #endif 272 #if defined(PETSC_HAVE_REAL___FLOAT128) 273 else if (*datatype == MPIU___FLOAT128) { 274 __float128 *xin = (__float128 *)in, *xout = (__float128 *)out; 275 for (i = 0; i < count; i++) xout[i] += xin[i]; 276 #if defined(PETSC_HAVE_COMPLEX) 277 } else if (*datatype == MPIU___COMPLEX128) { 278 __complex128 *xin = (__complex128 *)in, *xout = (__complex128 *)out; 279 for (i = 0; i < count; i++) xout[i] += xin[i]; 280 #endif 281 } 282 #endif 283 #if defined(PETSC_HAVE_REAL___FP16) 284 else if (*datatype == MPIU___FP16) { 285 __fp16 *xin = (__fp16 *)in, *xout = (__fp16 *)out; 286 for (i = 0; i < count; i++) xout[i] += xin[i]; 287 } 288 #endif 289 else { 290 #if !defined(PETSC_HAVE_REAL___FLOAT128) && !defined(PETSC_HAVE_REAL___FP16) 291 PetscCallAbort(MPI_COMM_SElF, (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types")); 292 #elif !defined(PETSC_HAVE_REAL___FP16) 293 PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL, MPIU_COMPLEX, MPIU___FLOAT128, or MPIU___COMPLEX128 data types")); 294 #elif !defined(PETSC_HAVE_REAL___FLOAT128) 295 PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL, MPIU_COMPLEX, or MPIU___FP16 data types")); 296 #else 297 PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL, MPIU_COMPLEX, MPIU___FLOAT128, MPIU___COMPLEX128, or MPIU___FP16 data types")); 298 #endif 299 PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG); 300 } 301 PetscFunctionReturnVoid(); 302 } 303 #endif 304 305 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16) 306 MPI_Op MPIU_MAX = 0; 307 MPI_Op MPIU_MIN = 0; 308 309 PETSC_EXTERN void MPIAPI PetscMax_Local(void *in, void *out, PetscMPIInt *cnt, MPI_Datatype *datatype) 310 { 311 PetscInt i, count = *cnt; 312 313 PetscFunctionBegin; 314 if (*datatype == MPIU_REAL) { 315 PetscReal *xin = (PetscReal *)in, *xout = (PetscReal *)out; 316 for (i = 0; i < count; i++) xout[i] = PetscMax(xout[i], xin[i]); 317 } 318 #if defined(PETSC_HAVE_COMPLEX) 319 else if (*datatype == MPIU_COMPLEX) { 320 PetscComplex *xin = (PetscComplex *)in, *xout = (PetscComplex *)out; 321 for (i = 0; i < count; i++) xout[i] = PetscRealPartComplex(xout[i]) < PetscRealPartComplex(xin[i]) ? xin[i] : xout[i]; 322 } 323 #endif 324 else { 325 PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types")); 326 PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG); 327 } 328 PetscFunctionReturnVoid(); 329 } 330 331 PETSC_EXTERN void MPIAPI PetscMin_Local(void *in, void *out, PetscMPIInt *cnt, MPI_Datatype *datatype) 332 { 333 PetscInt i, count = *cnt; 334 335 PetscFunctionBegin; 336 if (*datatype == MPIU_REAL) { 337 PetscReal *xin = (PetscReal *)in, *xout = (PetscReal *)out; 338 for (i = 0; i < count; i++) xout[i] = PetscMin(xout[i], xin[i]); 339 } 340 #if defined(PETSC_HAVE_COMPLEX) 341 else if (*datatype == MPIU_COMPLEX) { 342 PetscComplex *xin = (PetscComplex *)in, *xout = (PetscComplex *)out; 343 for (i = 0; i < count; i++) xout[i] = PetscRealPartComplex(xout[i]) > PetscRealPartComplex(xin[i]) ? xin[i] : xout[i]; 344 } 345 #endif 346 else { 347 PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types")); 348 PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG); 349 } 350 PetscFunctionReturnVoid(); 351 } 352 #endif 353 354 /* 355 Private routine to delete internal tag/name counter storage when a communicator is freed. 356 357 This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this data as an attribute is freed. 358 359 Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval() 360 361 */ 362 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_Counter_Attr_Delete_Fn(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state) 363 { 364 PetscCommCounter *counter = (PetscCommCounter *)count_val; 365 struct PetscCommStash *comms = counter->comms, *pcomm; 366 367 PetscFunctionBegin; 368 PetscCallMPI(PetscInfo(NULL, "Deleting counter data in an MPI_Comm %ld\n", (long)comm)); 369 PetscCallMPI(PetscFree(counter->iflags)); 370 while (comms) { 371 PetscCallMPI(MPI_Comm_free(&comms->comm)); 372 pcomm = comms; 373 comms = comms->next; 374 PetscCall(PetscFree(pcomm)); 375 } 376 PetscCallMPI(PetscFree(counter)); 377 PetscFunctionReturn(MPI_SUCCESS); 378 } 379 380 /* 381 This is invoked on the outer comm as a result of either PetscCommDestroy() (via MPI_Comm_delete_attr) or when the user 382 calls MPI_Comm_free(). 383 384 This is the only entry point for breaking the links between inner and outer comms. 385 386 This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator. 387 388 Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval() 389 390 */ 391 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_InnerComm_Attr_Delete_Fn(MPI_Comm comm, PetscMPIInt keyval, void *attr_val, void *extra_state) 392 { 393 union 394 { 395 MPI_Comm comm; 396 void *ptr; 397 } icomm; 398 399 PetscFunctionBegin; 400 if (keyval != Petsc_InnerComm_keyval) SETERRMPI(PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Unexpected keyval"); 401 icomm.ptr = attr_val; 402 if (PetscDefined(USE_DEBUG)) { 403 /* Error out if the inner/outer comms are not correctly linked through their Outer/InnterComm attributes */ 404 PetscMPIInt flg; 405 union 406 { 407 MPI_Comm comm; 408 void *ptr; 409 } ocomm; 410 PetscCallMPI(MPI_Comm_get_attr(icomm.comm, Petsc_OuterComm_keyval, &ocomm, &flg)); 411 if (!flg) SETERRMPI(PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Inner comm does not have OuterComm attribute"); 412 if (ocomm.comm != comm) SETERRMPI(PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Inner comm's OuterComm attribute does not point to outer PETSc comm"); 413 } 414 PetscCallMPI(MPI_Comm_delete_attr(icomm.comm, Petsc_OuterComm_keyval)); 415 PetscCallMPI(PetscInfo(NULL, "User MPI_Comm %ld is being unlinked from inner PETSc comm %ld\n", (long)comm, (long)icomm.comm)); 416 PetscFunctionReturn(MPI_SUCCESS); 417 } 418 419 /* 420 * This is invoked on the inner comm when Petsc_InnerComm_Attr_Delete_Fn calls MPI_Comm_delete_attr(). It should not be reached any other way. 421 */ 422 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_OuterComm_Attr_Delete_Fn(MPI_Comm comm, PetscMPIInt keyval, void *attr_val, void *extra_state) 423 { 424 PetscFunctionBegin; 425 PetscCallMPI(PetscInfo(NULL, "Removing reference to PETSc communicator embedded in a user MPI_Comm %ld\n", (long)comm)); 426 PetscFunctionReturn(MPI_SUCCESS); 427 } 428 429 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_ShmComm_Attr_Delete_Fn(MPI_Comm, PetscMPIInt, void *, void *); 430 431 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32) 432 PETSC_EXTERN PetscMPIInt PetscDataRep_extent_fn(MPI_Datatype, MPI_Aint *, void *); 433 PETSC_EXTERN PetscMPIInt PetscDataRep_read_conv_fn(void *, MPI_Datatype, PetscMPIInt, void *, MPI_Offset, void *); 434 PETSC_EXTERN PetscMPIInt PetscDataRep_write_conv_fn(void *, MPI_Datatype, PetscMPIInt, void *, MPI_Offset, void *); 435 #endif 436 437 PetscMPIInt PETSC_MPI_ERROR_CLASS = MPI_ERR_LASTCODE, PETSC_MPI_ERROR_CODE; 438 439 PETSC_INTERN int PetscGlobalArgc; 440 PETSC_INTERN char **PetscGlobalArgs; 441 int PetscGlobalArgc = 0; 442 char **PetscGlobalArgs = NULL; 443 PetscSegBuffer PetscCitationsList; 444 445 PetscErrorCode PetscCitationsInitialize(void) 446 { 447 PetscFunctionBegin; 448 PetscCall(PetscSegBufferCreate(1, 10000, &PetscCitationsList)); 449 450 PetscCall(PetscCitationsRegister("@TechReport{petsc-user-ref,\n\ 451 Author = {Satish Balay and Shrirang Abhyankar and Mark~F. Adams and Steven Benson and Jed Brown\n\ 452 and Peter Brune and Kris Buschelman and Emil Constantinescu and Lisandro Dalcin and Alp Dener\n\ 453 and Victor Eijkhout and Jacob Faibussowitsch and William~D. Gropp and V\'{a}clav Hapla and Tobin Isaac and Pierre Jolivet\n\ 454 and Dmitry Karpeev and Dinesh Kaushik and Matthew~G. Knepley and Fande Kong and Scott Kruger\n\ 455 and Dave~A. May and Lois Curfman McInnes and Richard Tran Mills and Lawrence Mitchell and Todd Munson\n\ 456 and Jose~E. Roman and Karl Rupp and Patrick Sanan and Jason Sarich and Barry~F. Smith\n\ 457 and Stefano Zampini and Hong Zhang and Hong Zhang and Junchao Zhang},\n\ 458 Title = {{PETSc/TAO} Users Manual},\n\ 459 Number = {ANL-21/39 - Revision 3.19},\n\ 460 Doi = {10.2172/1968587},\n\ 461 Institution = {Argonne National Laboratory},\n\ 462 Year = {2023}\n}\n", 463 NULL)); 464 465 PetscCall(PetscCitationsRegister("@InProceedings{petsc-efficient,\n\ 466 Author = {Satish Balay and William D. Gropp and Lois Curfman McInnes and Barry F. Smith},\n\ 467 Title = {Efficient Management of Parallelism in Object Oriented Numerical Software Libraries},\n\ 468 Booktitle = {Modern Software Tools in Scientific Computing},\n\ 469 Editor = {E. Arge and A. M. Bruaset and H. P. Langtangen},\n\ 470 Pages = {163--202},\n\ 471 Publisher = {Birkh{\\\"{a}}user Press},\n\ 472 Year = {1997}\n}\n", 473 NULL)); 474 475 PetscFunctionReturn(PETSC_SUCCESS); 476 } 477 478 static char programname[PETSC_MAX_PATH_LEN] = ""; /* HP includes entire path in name */ 479 480 PetscErrorCode PetscSetProgramName(const char name[]) 481 { 482 PetscFunctionBegin; 483 PetscCall(PetscStrncpy(programname, name, sizeof(programname))); 484 PetscFunctionReturn(PETSC_SUCCESS); 485 } 486 487 /*@C 488 PetscGetProgramName - Gets the name of the running program. 489 490 Not Collective 491 492 Input Parameter: 493 . len - length of the string name 494 495 Output Parameter: 496 . name - the name of the running program, provide a string of length `PETSC_MAX_PATH_LEN` 497 498 Level: advanced 499 500 .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArguments()`, `PetscInitialize()` 501 @*/ 502 PetscErrorCode PetscGetProgramName(char name[], size_t len) 503 { 504 PetscFunctionBegin; 505 PetscCall(PetscStrncpy(name, programname, len)); 506 PetscFunctionReturn(PETSC_SUCCESS); 507 } 508 509 /*@C 510 PetscGetArgs - Allows you to access the raw command line arguments anywhere 511 after PetscInitialize() is called but before `PetscFinalize()`. 512 513 Not Collective 514 515 Output Parameters: 516 + argc - count of number of command line arguments 517 - args - the command line arguments 518 519 Level: intermediate 520 521 Notes: 522 This is usually used to pass the command line arguments into other libraries 523 that are called internally deep in PETSc or the application. 524 525 The first argument contains the program name as is normal for C arguments. 526 527 .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArguments()`, `PetscInitialize()` 528 @*/ 529 PetscErrorCode PetscGetArgs(int *argc, char ***args) 530 { 531 PetscFunctionBegin; 532 PetscCheck(PetscInitializeCalled || !PetscFinalizeCalled, PETSC_COMM_SELF, PETSC_ERR_ORDER, "You must call after PetscInitialize() but before PetscFinalize()"); 533 *argc = PetscGlobalArgc; 534 *args = PetscGlobalArgs; 535 PetscFunctionReturn(PETSC_SUCCESS); 536 } 537 538 /*@C 539 PetscGetArguments - Allows you to access the command line arguments anywhere 540 after `PetscInitialize()` is called but before `PetscFinalize()`. 541 542 Not Collective 543 544 Output Parameter: 545 . args - the command line arguments 546 547 Level: intermediate 548 549 Note: 550 This does NOT start with the program name and IS `NULL` terminated (final arg is void) 551 552 .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscFreeArguments()`, `PetscInitialize()` 553 @*/ 554 PetscErrorCode PetscGetArguments(char ***args) 555 { 556 PetscInt i, argc = PetscGlobalArgc; 557 558 PetscFunctionBegin; 559 PetscCheck(PetscInitializeCalled || !PetscFinalizeCalled, PETSC_COMM_SELF, PETSC_ERR_ORDER, "You must call after PetscInitialize() but before PetscFinalize()"); 560 if (!argc) { 561 *args = NULL; 562 PetscFunctionReturn(PETSC_SUCCESS); 563 } 564 PetscCall(PetscMalloc1(argc, args)); 565 for (i = 0; i < argc - 1; i++) PetscCall(PetscStrallocpy(PetscGlobalArgs[i + 1], &(*args)[i])); 566 (*args)[argc - 1] = NULL; 567 PetscFunctionReturn(PETSC_SUCCESS); 568 } 569 570 /*@C 571 PetscFreeArguments - Frees the memory obtained with `PetscGetArguments()` 572 573 Not Collective 574 575 Output Parameter: 576 . args - the command line arguments 577 578 Level: intermediate 579 580 .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscGetArguments()` 581 @*/ 582 PetscErrorCode PetscFreeArguments(char **args) 583 { 584 PetscFunctionBegin; 585 if (args) { 586 PetscInt i = 0; 587 588 while (args[i]) PetscCall(PetscFree(args[i++])); 589 PetscCall(PetscFree(args)); 590 } 591 PetscFunctionReturn(PETSC_SUCCESS); 592 } 593 594 #if PetscDefined(HAVE_SAWS) 595 #include <petscconfiginfo.h> 596 597 PETSC_INTERN PetscErrorCode PetscInitializeSAWs(const char help[]) 598 { 599 PetscFunctionBegin; 600 if (!PetscGlobalRank) { 601 char cert[PETSC_MAX_PATH_LEN], root[PETSC_MAX_PATH_LEN], *intro, programname[64], *appline, *options, version[64]; 602 int port; 603 PetscBool flg, rootlocal = PETSC_FALSE, flg2, selectport = PETSC_FALSE; 604 size_t applinelen, introlen; 605 char sawsurl[256]; 606 607 PetscCall(PetscOptionsHasName(NULL, NULL, "-saws_log", &flg)); 608 if (flg) { 609 char sawslog[PETSC_MAX_PATH_LEN]; 610 611 PetscCall(PetscOptionsGetString(NULL, NULL, "-saws_log", sawslog, sizeof(sawslog), NULL)); 612 if (sawslog[0]) { 613 PetscCallSAWs(SAWs_Set_Use_Logfile, (sawslog)); 614 } else { 615 PetscCallSAWs(SAWs_Set_Use_Logfile, (NULL)); 616 } 617 } 618 PetscCall(PetscOptionsGetString(NULL, NULL, "-saws_https", cert, sizeof(cert), &flg)); 619 if (flg) PetscCallSAWs(SAWs_Set_Use_HTTPS, (cert)); 620 PetscCall(PetscOptionsGetBool(NULL, NULL, "-saws_port_auto_select", &selectport, NULL)); 621 if (selectport) { 622 PetscCallSAWs(SAWs_Get_Available_Port, (&port)); 623 PetscCallSAWs(SAWs_Set_Port, (port)); 624 } else { 625 PetscCall(PetscOptionsGetInt(NULL, NULL, "-saws_port", &port, &flg)); 626 if (flg) PetscCallSAWs(SAWs_Set_Port, (port)); 627 } 628 PetscCall(PetscOptionsGetString(NULL, NULL, "-saws_root", root, sizeof(root), &flg)); 629 if (flg) { 630 PetscCallSAWs(SAWs_Set_Document_Root, (root)); 631 PetscCall(PetscStrcmp(root, ".", &rootlocal)); 632 } else { 633 PetscCall(PetscOptionsHasName(NULL, NULL, "-saws_options", &flg)); 634 if (flg) { 635 PetscCall(PetscStrreplace(PETSC_COMM_WORLD, "${PETSC_DIR}/share/petsc/saws", root, sizeof(root))); 636 PetscCallSAWs(SAWs_Set_Document_Root, (root)); 637 } 638 } 639 PetscCall(PetscOptionsHasName(NULL, NULL, "-saws_local", &flg2)); 640 if (flg2) { 641 char jsdir[PETSC_MAX_PATH_LEN]; 642 PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_SUP, "-saws_local option requires -saws_root option"); 643 PetscCall(PetscSNPrintf(jsdir, sizeof(jsdir), "%s/js", root)); 644 PetscCall(PetscTestDirectory(jsdir, 'r', &flg)); 645 PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_FILE_READ, "-saws_local option requires js directory in root directory"); 646 PetscCallSAWs(SAWs_Push_Local_Header, ()); 647 } 648 PetscCall(PetscGetProgramName(programname, sizeof(programname))); 649 PetscCall(PetscStrlen(help, &applinelen)); 650 introlen = 4096 + applinelen; 651 applinelen += 1024; 652 PetscCall(PetscMalloc(applinelen, &appline)); 653 PetscCall(PetscMalloc(introlen, &intro)); 654 655 if (rootlocal) { 656 PetscCall(PetscSNPrintf(appline, applinelen, "%s.c.html", programname)); 657 PetscCall(PetscTestFile(appline, 'r', &rootlocal)); 658 } 659 PetscCall(PetscOptionsGetAll(NULL, &options)); 660 if (rootlocal && help) { 661 PetscCall(PetscSNPrintf(appline, applinelen, "<center> Running <a href=\"%s.c.html\">%s</a> %s</center><br><center><pre>%s</pre></center><br>\n", programname, programname, options, help)); 662 } else if (help) { 663 PetscCall(PetscSNPrintf(appline, applinelen, "<center>Running %s %s</center><br><center><pre>%s</pre></center><br>", programname, options, help)); 664 } else { 665 PetscCall(PetscSNPrintf(appline, applinelen, "<center> Running %s %s</center><br>\n", programname, options)); 666 } 667 PetscCall(PetscFree(options)); 668 PetscCall(PetscGetVersion(version, sizeof(version))); 669 PetscCall(PetscSNPrintf(intro, introlen, 670 "<body>\n" 671 "<center><h2> <a href=\"https://petsc.org/\">PETSc</a> Application Web server powered by <a href=\"https://bitbucket.org/saws/saws\">SAWs</a> </h2></center>\n" 672 "<center>This is the default PETSc application dashboard, from it you can access any published PETSc objects or logging data</center><br><center>%s configured with %s</center><br>\n" 673 "%s", 674 version, petscconfigureoptions, appline)); 675 PetscCallSAWs(SAWs_Push_Body, ("index.html", 0, intro)); 676 PetscCall(PetscFree(intro)); 677 PetscCall(PetscFree(appline)); 678 if (selectport) { 679 PetscBool silent; 680 681 /* another process may have grabbed the port so keep trying */ 682 while (SAWs_Initialize()) { 683 PetscCallSAWs(SAWs_Get_Available_Port, (&port)); 684 PetscCallSAWs(SAWs_Set_Port, (port)); 685 } 686 687 PetscCall(PetscOptionsGetBool(NULL, NULL, "-saws_port_auto_select_silent", &silent, NULL)); 688 if (!silent) { 689 PetscCallSAWs(SAWs_Get_FullURL, (sizeof(sawsurl), sawsurl)); 690 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Point your browser to %s for SAWs\n", sawsurl)); 691 } 692 } else { 693 PetscCallSAWs(SAWs_Initialize, ()); 694 } 695 PetscCall(PetscCitationsRegister("@TechReport{ saws,\n" 696 " Author = {Matt Otten and Jed Brown and Barry Smith},\n" 697 " Title = {Scientific Application Web Server (SAWs) Users Manual},\n" 698 " Institution = {Argonne National Laboratory},\n" 699 " Year = 2013\n}\n", 700 NULL)); 701 } 702 PetscFunctionReturn(PETSC_SUCCESS); 703 } 704 #endif 705 706 /* Things must be done before MPI_Init() when MPI is not yet initialized, and can be shared between C init and Fortran init */ 707 PETSC_INTERN PetscErrorCode PetscPreMPIInit_Private(void) 708 { 709 PetscFunctionBegin; 710 #if defined(PETSC_HAVE_HWLOC_SOLARIS_BUG) 711 /* see MPI.py for details on this bug */ 712 (void)setenv("HWLOC_COMPONENTS", "-x86", 1); 713 #endif 714 PetscFunctionReturn(PETSC_SUCCESS); 715 } 716 717 #if PetscDefined(HAVE_ADIOS) 718 #include <adios.h> 719 #include <adios_read.h> 720 int64_t Petsc_adios_group; 721 #endif 722 #if PetscDefined(HAVE_OPENMP) 723 #include <omp.h> 724 PetscInt PetscNumOMPThreads; 725 #endif 726 727 #include <petsc/private/deviceimpl.h> 728 #if PetscDefined(HAVE_CUDA) 729 #include <petscdevice_cuda.h> 730 // REMOVE ME 731 cudaStream_t PetscDefaultCudaStream = NULL; 732 #endif 733 #if PetscDefined(HAVE_HIP) 734 #include <petscdevice_hip.h> 735 // REMOVE ME 736 hipStream_t PetscDefaultHipStream = NULL; 737 #endif 738 739 #if PetscDefined(HAVE_DLFCN_H) 740 #include <dlfcn.h> 741 #endif 742 #if PetscDefined(USE_LOG) 743 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 744 #endif 745 #if PetscDefined(HAVE_VIENNACL) 746 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(void); 747 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 748 #endif 749 750 PetscBool PetscCIEnabled = PETSC_FALSE, PetscCIEnabledPortableErrorOutput = PETSC_FALSE; 751 752 /* 753 PetscInitialize_Common - shared code between C and Fortran initialization 754 755 prog: program name 756 file: optional PETSc database file name. Might be in Fortran string format when 'ftn' is true 757 help: program help message 758 ftn: is it called from Fortran initialization (petscinitializef_)? 759 readarguments,len: used when fortran is true 760 */ 761 PETSC_INTERN PetscErrorCode PetscInitialize_Common(const char *prog, const char *file, const char *help, PetscBool ftn, PetscBool readarguments, PetscInt len) 762 { 763 PetscMPIInt size; 764 PetscBool flg = PETSC_TRUE; 765 char hostname[256]; 766 767 PetscFunctionBegin; 768 if (PetscInitializeCalled) PetscFunctionReturn(PETSC_SUCCESS); 769 /* these must be initialized in a routine, not as a constant declaration */ 770 PETSC_STDOUT = stdout; 771 PETSC_STDERR = stderr; 772 773 /* PetscCall can be used from now */ 774 PetscErrorHandlingInitialized = PETSC_TRUE; 775 776 /* 777 The checking over compatible runtime libraries is complicated by the MPI ABI initiative 778 https://wiki.mpich.org/mpich/index.php/ABI_Compatibility_Initiative which started with 779 MPICH v3.1 (Released February 2014) 780 IBM MPI v2.1 (December 2014) 781 Intel MPI Library v5.0 (2014) 782 Cray MPT v7.0.0 (June 2014) 783 As of July 31, 2017 the ABI number still appears to be 12, that is all of the versions 784 listed above and since that time are compatible. 785 786 Unfortunately the MPI ABI initiative has not defined a way to determine the ABI number 787 at compile time or runtime. Thus we will need to systematically track the allowed versions 788 and how they are represented in the mpi.h and MPI_Get_library_version() output in order 789 to perform the checking. 790 791 Currently we only check for pre MPI ABI versions (and packages that do not follow the MPI ABI). 792 793 Questions: 794 795 Should the checks for ABI incompatibility be only on the major version number below? 796 Presumably the output to stderr will be removed before a release. 797 */ 798 799 #if defined(PETSC_HAVE_MPI_GET_LIBRARY_VERSION) 800 { 801 char mpilibraryversion[MPI_MAX_LIBRARY_VERSION_STRING]; 802 PetscMPIInt mpilibraryversionlength; 803 804 PetscCallMPI(MPI_Get_library_version(mpilibraryversion, &mpilibraryversionlength)); 805 /* check for MPICH versions before MPI ABI initiative */ 806 #if defined(MPICH_VERSION) 807 #if MPICH_NUMVERSION < 30100000 808 { 809 char *ver, *lf; 810 PetscBool flg = PETSC_FALSE; 811 812 PetscCall(PetscStrstr(mpilibraryversion, "MPICH Version:", &ver)); 813 if (ver) { 814 PetscCall(PetscStrchr(ver, '\n', &lf)); 815 if (lf) { 816 *lf = 0; 817 PetscCall(PetscStrendswith(ver, MPICH_VERSION, &flg)); 818 } 819 } 820 if (!flg) { 821 PetscCall(PetscInfo(NULL, "PETSc warning --- MPICH library version \n%s does not match what PETSc was compiled with %s.\n", mpilibraryversion, MPICH_VERSION)); 822 flg = PETSC_TRUE; 823 } 824 } 825 #endif 826 /* check for OpenMPI version, it is not part of the MPI ABI initiative (is it part of another initiative that needs to be handled?) */ 827 #elif defined(OMPI_MAJOR_VERSION) 828 { 829 char *ver, bs[MPI_MAX_LIBRARY_VERSION_STRING], *bsf; 830 PetscBool flg = PETSC_FALSE; 831 #define PSTRSZ 2 832 char ompistr1[PSTRSZ][MPI_MAX_LIBRARY_VERSION_STRING] = {"Open MPI", "FUJITSU MPI"}; 833 char ompistr2[PSTRSZ][MPI_MAX_LIBRARY_VERSION_STRING] = {"v", "Library "}; 834 int i; 835 for (i = 0; i < PSTRSZ; i++) { 836 PetscCall(PetscStrstr(mpilibraryversion, ompistr1[i], &ver)); 837 if (ver) { 838 PetscCall(PetscSNPrintf(bs, MPI_MAX_LIBRARY_VERSION_STRING, "%s%d.%d", ompistr2[i], OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION)); 839 PetscCall(PetscStrstr(ver, bs, &bsf)); 840 if (bsf) flg = PETSC_TRUE; 841 break; 842 } 843 } 844 if (!flg) { 845 PetscCall(PetscInfo(NULL, "PETSc warning --- Open MPI library version \n%s does not match what PETSc was compiled with %d.%d.\n", mpilibraryversion, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION)); 846 flg = PETSC_TRUE; 847 } 848 } 849 #endif 850 } 851 #endif 852 853 #if defined(PETSC_HAVE_DLADDR) && !(defined(__cray__) && defined(__clang__)) 854 /* These symbols are currently in the OpenMPI and MPICH libraries; they may not always be, in that case the test will simply not detect the problem */ 855 PetscCheck(!dlsym(RTLD_DEFAULT, "ompi_mpi_init") || !dlsym(RTLD_DEFAULT, "MPID_Abort"), PETSC_COMM_SELF, PETSC_ERR_MPI_LIB_INCOMP, "Application was linked against both OpenMPI and MPICH based MPI libraries and will not run correctly"); 856 #endif 857 858 /* on Windows - set printf to default to printing 2 digit exponents */ 859 #if defined(PETSC_HAVE__SET_OUTPUT_FORMAT) 860 _set_output_format(_TWO_DIGIT_EXPONENT); 861 #endif 862 863 PetscCall(PetscOptionsCreateDefault()); 864 865 PetscFinalizeCalled = PETSC_FALSE; 866 867 PetscCall(PetscSetProgramName(prog)); 868 PetscCall(PetscSpinlockCreate(&PetscViewerASCIISpinLockOpen)); 869 PetscCall(PetscSpinlockCreate(&PetscViewerASCIISpinLockStdout)); 870 PetscCall(PetscSpinlockCreate(&PetscViewerASCIISpinLockStderr)); 871 PetscCall(PetscSpinlockCreate(&PetscCommSpinLock)); 872 873 if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD; 874 PetscCallMPI(MPI_Comm_set_errhandler(PETSC_COMM_WORLD, MPI_ERRORS_RETURN)); 875 876 if (PETSC_MPI_ERROR_CLASS == MPI_ERR_LASTCODE) { 877 PetscCallMPI(MPI_Add_error_class(&PETSC_MPI_ERROR_CLASS)); 878 PetscCallMPI(MPI_Add_error_code(PETSC_MPI_ERROR_CLASS, &PETSC_MPI_ERROR_CODE)); 879 } 880 881 /* Done after init due to a bug in MPICH-GM? */ 882 PetscCall(PetscErrorPrintfInitialize()); 883 884 PetscCallMPI(MPI_Comm_rank(MPI_COMM_WORLD, &PetscGlobalRank)); 885 PetscCallMPI(MPI_Comm_size(MPI_COMM_WORLD, &PetscGlobalSize)); 886 887 MPIU_BOOL = MPI_INT; 888 MPIU_ENUM = MPI_INT; 889 MPIU_FORTRANADDR = (sizeof(void *) == sizeof(int)) ? MPI_INT : MPIU_INT64; 890 if (sizeof(size_t) == sizeof(unsigned)) MPIU_SIZE_T = MPI_UNSIGNED; 891 else if (sizeof(size_t) == sizeof(unsigned long)) MPIU_SIZE_T = MPI_UNSIGNED_LONG; 892 #if defined(PETSC_SIZEOF_LONG_LONG) 893 else if (sizeof(size_t) == sizeof(unsigned long long)) MPIU_SIZE_T = MPI_UNSIGNED_LONG_LONG; 894 #endif 895 else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_SUP_SYS, "Could not find MPI type for size_t"); 896 897 /* 898 Initialized the global complex variable; this is because with 899 shared libraries the constructors for global variables 900 are not called; at least on IRIX. 901 */ 902 #if defined(PETSC_HAVE_COMPLEX) 903 { 904 #if defined(PETSC_CLANGUAGE_CXX) && !defined(PETSC_USE_REAL___FLOAT128) 905 PetscComplex ic(0.0, 1.0); 906 PETSC_i = ic; 907 #else 908 PETSC_i = _Complex_I; 909 #endif 910 } 911 #endif /* PETSC_HAVE_COMPLEX */ 912 913 /* 914 Create the PETSc MPI reduction operator that sums of the first 915 half of the entries and maxes the second half. 916 */ 917 PetscCallMPI(MPI_Op_create(MPIU_MaxSum_Local, 1, &MPIU_MAXSUM_OP)); 918 919 #if defined(PETSC_HAVE_REAL___FLOAT128) 920 PetscCallMPI(MPI_Type_contiguous(2, MPI_DOUBLE, &MPIU___FLOAT128)); 921 PetscCallMPI(MPI_Type_commit(&MPIU___FLOAT128)); 922 #if defined(PETSC_HAVE_COMPLEX) 923 PetscCallMPI(MPI_Type_contiguous(4, MPI_DOUBLE, &MPIU___COMPLEX128)); 924 PetscCallMPI(MPI_Type_commit(&MPIU___COMPLEX128)); 925 #endif 926 #endif 927 #if defined(PETSC_HAVE_REAL___FP16) 928 PetscCallMPI(MPI_Type_contiguous(2, MPI_CHAR, &MPIU___FP16)); 929 PetscCallMPI(MPI_Type_commit(&MPIU___FP16)); 930 #endif 931 932 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16) 933 PetscCallMPI(MPI_Op_create(PetscSum_Local, 1, &MPIU_SUM)); 934 PetscCallMPI(MPI_Op_create(PetscMax_Local, 1, &MPIU_MAX)); 935 PetscCallMPI(MPI_Op_create(PetscMin_Local, 1, &MPIU_MIN)); 936 #elif defined(PETSC_HAVE_REAL___FLOAT128) || defined(PETSC_HAVE_REAL___FP16) 937 PetscCallMPI(MPI_Op_create(PetscSum_Local, 1, &MPIU_SUM___FP16___FLOAT128)); 938 #endif 939 940 PetscCallMPI(MPI_Type_contiguous(2, MPIU_SCALAR, &MPIU_2SCALAR)); 941 PetscCallMPI(MPI_Op_create(PetscGarbageKeySortedIntersect, 1, &Petsc_Garbage_SetIntersectOp)); 942 PetscCallMPI(MPI_Type_commit(&MPIU_2SCALAR)); 943 944 /* create datatypes used by MPIU_MAXLOC, MPIU_MINLOC and PetscSplitReduction_Op */ 945 #if !defined(PETSC_HAVE_MPIUNI) 946 { 947 PetscMPIInt blockSizes[2] = {1, 1}; 948 MPI_Aint blockOffsets[2] = {offsetof(struct petsc_mpiu_real_int, v), offsetof(struct petsc_mpiu_real_int, i)}; 949 MPI_Datatype blockTypes[2] = {MPIU_REAL, MPIU_INT}, tmpStruct; 950 951 PetscCallMPI(MPI_Type_create_struct(2, blockSizes, blockOffsets, blockTypes, &tmpStruct)); 952 PetscCallMPI(MPI_Type_create_resized(tmpStruct, 0, sizeof(struct petsc_mpiu_real_int), &MPIU_REAL_INT)); 953 PetscCallMPI(MPI_Type_free(&tmpStruct)); 954 PetscCallMPI(MPI_Type_commit(&MPIU_REAL_INT)); 955 } 956 { 957 PetscMPIInt blockSizes[2] = {1, 1}; 958 MPI_Aint blockOffsets[2] = {offsetof(struct petsc_mpiu_scalar_int, v), offsetof(struct petsc_mpiu_scalar_int, i)}; 959 MPI_Datatype blockTypes[2] = {MPIU_SCALAR, MPIU_INT}, tmpStruct; 960 961 PetscCallMPI(MPI_Type_create_struct(2, blockSizes, blockOffsets, blockTypes, &tmpStruct)); 962 PetscCallMPI(MPI_Type_create_resized(tmpStruct, 0, sizeof(struct petsc_mpiu_scalar_int), &MPIU_SCALAR_INT)); 963 PetscCallMPI(MPI_Type_free(&tmpStruct)); 964 PetscCallMPI(MPI_Type_commit(&MPIU_SCALAR_INT)); 965 } 966 #endif 967 968 #if defined(PETSC_USE_64BIT_INDICES) 969 PetscCallMPI(MPI_Type_contiguous(2, MPIU_INT, &MPIU_2INT)); 970 PetscCallMPI(MPI_Type_commit(&MPIU_2INT)); 971 #endif 972 PetscCallMPI(MPI_Type_contiguous(4, MPI_INT, &MPI_4INT)); 973 PetscCallMPI(MPI_Type_commit(&MPI_4INT)); 974 PetscCallMPI(MPI_Type_contiguous(4, MPIU_INT, &MPIU_4INT)); 975 PetscCallMPI(MPI_Type_commit(&MPIU_4INT)); 976 977 /* 978 Attributes to be set on PETSc communicators 979 */ 980 PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_Counter_Attr_Delete_Fn, &Petsc_Counter_keyval, (void *)0)); 981 PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_InnerComm_Attr_Delete_Fn, &Petsc_InnerComm_keyval, (void *)0)); 982 PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_OuterComm_Attr_Delete_Fn, &Petsc_OuterComm_keyval, (void *)0)); 983 PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_ShmComm_Attr_Delete_Fn, &Petsc_ShmComm_keyval, (void *)0)); 984 PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, MPI_COMM_NULL_DELETE_FN, &Petsc_CreationIdx_keyval, (void *)0)); 985 PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, MPI_COMM_NULL_DELETE_FN, &Petsc_Garbage_HMap_keyval, (void *)0)); 986 987 #if defined(PETSC_USE_FORTRAN_BINDINGS) 988 if (ftn) PetscCall(PetscInitFortran_Private(readarguments, file, len)); 989 else 990 #endif 991 PetscCall(PetscOptionsInsert(NULL, &PetscGlobalArgc, &PetscGlobalArgs, file)); 992 993 /* call a second time so it can look in the options database */ 994 PetscCall(PetscErrorPrintfInitialize()); 995 996 /* 997 Check system options and print help 998 */ 999 PetscCall(PetscOptionsCheckInitial_Private(help)); 1000 1001 /* 1002 Creates the logging data structures; this is enabled even if logging is not turned on 1003 This is the last thing we do before returning to the user code to prevent having the 1004 logging numbers contaminated by any startup time associated with MPI 1005 */ 1006 #if defined(PETSC_USE_LOG) 1007 PetscCall(PetscLogInitialize()); 1008 #endif 1009 1010 /* 1011 Initialize PetscDevice and PetscDeviceContext 1012 1013 Note to any future devs thinking of moving this, proper initialization requires: 1014 1. MPI initialized 1015 2. Options DB initialized 1016 3. Petsc error handling initialized, specifically signal handlers. This expects to set up 1017 its own SIGSEV handler via the push/pop interface. 1018 4. Logging initialized 1019 */ 1020 PetscCall(PetscDeviceInitializeFromOptions_Internal(PETSC_COMM_WORLD)); 1021 1022 #if PetscDefined(HAVE_VIENNACL) 1023 flg = PETSC_FALSE; 1024 PetscCall(PetscOptionsHasName(NULL, NULL, "-log_view", &flg)); 1025 if (!flg) PetscCall(PetscOptionsGetBool(NULL, NULL, "-viennacl_synchronize", &flg, NULL)); 1026 PetscViennaCLSynchronize = flg; 1027 PetscCall(PetscViennaCLInit()); 1028 #endif 1029 1030 PetscCall(PetscCitationsInitialize()); 1031 1032 #if defined(PETSC_HAVE_SAWS) 1033 PetscCall(PetscInitializeSAWs(ftn ? NULL : help)); 1034 flg = PETSC_FALSE; 1035 PetscCall(PetscOptionsHasName(NULL, NULL, "-stack_view", &flg)); 1036 if (flg) PetscCall(PetscStackViewSAWs()); 1037 #endif 1038 1039 /* 1040 Load the dynamic libraries (on machines that support them), this registers all 1041 the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes) 1042 */ 1043 PetscCall(PetscInitialize_DynamicLibraries()); 1044 1045 PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size)); 1046 PetscCall(PetscInfo(NULL, "PETSc successfully started: number of processors = %d\n", size)); 1047 PetscCall(PetscGetHostName(hostname, sizeof(hostname))); 1048 PetscCall(PetscInfo(NULL, "Running on machine: %s\n", hostname)); 1049 #if defined(PETSC_HAVE_OPENMP) 1050 { 1051 PetscBool omp_view_flag; 1052 char *threads = getenv("OMP_NUM_THREADS"); 1053 1054 if (threads) { 1055 PetscCall(PetscInfo(NULL, "Number of OpenMP threads %s (as given by OMP_NUM_THREADS)\n", threads)); 1056 (void)sscanf(threads, "%" PetscInt_FMT, &PetscNumOMPThreads); 1057 } else { 1058 PetscNumOMPThreads = (PetscInt)omp_get_max_threads(); 1059 PetscCall(PetscInfo(NULL, "Number of OpenMP threads %" PetscInt_FMT " (as given by omp_get_max_threads())\n", PetscNumOMPThreads)); 1060 } 1061 PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "OpenMP options", "Sys"); 1062 PetscCall(PetscOptionsInt("-omp_num_threads", "Number of OpenMP threads to use (can also use environmental variable OMP_NUM_THREADS", "None", PetscNumOMPThreads, &PetscNumOMPThreads, &flg)); 1063 PetscCall(PetscOptionsName("-omp_view", "Display OpenMP number of threads", NULL, &omp_view_flag)); 1064 PetscOptionsEnd(); 1065 if (flg) { 1066 PetscCall(PetscInfo(NULL, "Number of OpenMP theads %" PetscInt_FMT " (given by -omp_num_threads)\n", PetscNumOMPThreads)); 1067 omp_set_num_threads((int)PetscNumOMPThreads); 1068 } 1069 if (omp_view_flag) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "OpenMP: number of threads %" PetscInt_FMT "\n", PetscNumOMPThreads)); 1070 } 1071 #endif 1072 1073 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32) 1074 /* 1075 Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI 1076 1077 Currently not used because it is not supported by MPICH. 1078 */ 1079 if (!PetscBinaryBigEndian()) PetscCallMPI(MPI_Register_datarep((char *)"petsc", PetscDataRep_read_conv_fn, PetscDataRep_write_conv_fn, PetscDataRep_extent_fn, NULL)); 1080 #endif 1081 1082 #if defined(PETSC_SERIALIZE_FUNCTIONS) 1083 PetscCall(PetscFPTCreate(10000)); 1084 #endif 1085 1086 #if defined(PETSC_HAVE_HWLOC) 1087 { 1088 PetscViewer viewer; 1089 PetscCall(PetscOptionsGetViewer(PETSC_COMM_WORLD, NULL, NULL, "-process_view", &viewer, NULL, &flg)); 1090 if (flg) { 1091 PetscCall(PetscProcessPlacementView(viewer)); 1092 PetscCall(PetscViewerDestroy(&viewer)); 1093 } 1094 } 1095 #endif 1096 1097 flg = PETSC_TRUE; 1098 PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewfromoptions", &flg, NULL)); 1099 if (!flg) PetscCall(PetscOptionsPushGetViewerOff(PETSC_TRUE)); 1100 1101 #if defined(PETSC_HAVE_ADIOS) 1102 PetscCallExternal(adios_init_noxml, PETSC_COMM_WORLD); 1103 PetscCallExternal(adios_declare_group, &Petsc_adios_group, "PETSc", "", adios_stat_default); 1104 PetscCallExternal(adios_select_method, Petsc_adios_group, "MPI", "", ""); 1105 PetscCallExternal(adios_read_init_method, ADIOS_READ_METHOD_BP, PETSC_COMM_WORLD, ""); 1106 #endif 1107 1108 #if defined(__VALGRIND_H) 1109 PETSC_RUNNING_ON_VALGRIND = RUNNING_ON_VALGRIND ? PETSC_TRUE : PETSC_FALSE; 1110 #if defined(PETSC_USING_DARWIN) && defined(PETSC_BLASLAPACK_SDOT_RETURNS_DOUBLE) 1111 if (PETSC_RUNNING_ON_VALGRIND) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "WARNING: Running valgrind with the MacOS native BLAS and LAPACK can fail. If it fails suggest configuring with --download-fblaslapack or --download-f2cblaslapack")); 1112 #endif 1113 #endif 1114 /* 1115 Set flag that we are completely initialized 1116 */ 1117 PetscInitializeCalled = PETSC_TRUE; 1118 1119 PetscCall(PetscOptionsHasName(NULL, NULL, "-python", &flg)); 1120 if (flg) PetscCall(PetscPythonInitialize(NULL, NULL)); 1121 1122 PetscCall(PetscOptionsHasName(NULL, NULL, "-mpi_linear_solver_server", &flg)); 1123 if (PetscDefined(USE_SINGLE_LIBRARY) && flg) PetscCall(PCMPIServerBegin()); 1124 else PetscCheck(!flg, PETSC_COMM_WORLD, PETSC_ERR_SUP, "PETSc configured using -with-single-library=0; -mpi_linear_solver_server not supported in that case"); 1125 PetscFunctionReturn(PETSC_SUCCESS); 1126 } 1127 1128 // "Unknown section 'Environmental Variables'" 1129 // PetscClangLinter pragma disable: -fdoc-section-header-unknown 1130 /*@C 1131 PetscInitialize - Initializes the PETSc database and MPI. 1132 `PetscInitialize()` calls MPI_Init() if that has yet to be called, 1133 so this routine should always be called near the beginning of 1134 your program -- usually the very first line! 1135 1136 Collective on `MPI_COMM_WORLD` or `PETSC_COMM_WORLD` if it has been set 1137 1138 Input Parameters: 1139 + argc - count of number of command line arguments 1140 . args - the command line arguments 1141 . file - [optional] PETSc database file, append ":yaml" to filename to specify YAML options format. 1142 Use NULL or empty string to not check for code specific file. 1143 Also checks ~/.petscrc, .petscrc and petscrc. 1144 Use -skip_petscrc in the code specific file (or command line) to skip ~/.petscrc, .petscrc and petscrc files. 1145 - help - [optional] Help message to print, use NULL for no message 1146 1147 If you wish PETSc code to run ONLY on a subcommunicator of `MPI_COMM_WORLD`, create that 1148 communicator first and assign it to `PETSC_COMM_WORLD` BEFORE calling `PetscInitialize()`. Thus if you are running a 1149 four process job and two processes will run PETSc and have `PetscInitialize()` and PetscFinalize() and two process will not, 1150 then do this. If ALL processes in the job are using `PetscInitialize()` and `PetscFinalize()` then you don't need to do this, even 1151 if different subcommunicators of the job are doing different things with PETSc. 1152 1153 Options Database Keys: 1154 + -help [intro] - prints help method for each option; if intro is given the program stops after printing the introductory help message 1155 . -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger 1156 . -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected 1157 . -on_error_emacs <machinename> - causes emacsclient to jump to error file 1158 . -on_error_abort - calls `abort()` when error detected (no traceback) 1159 . -on_error_mpiabort - calls `MPI_abort()` when error detected 1160 . -error_output_stdout - prints PETSc error messages to stdout instead of the default stderr 1161 . -error_output_none - does not print the error messages (but handles errors in the same way as if this was not called) 1162 . -debugger_ranks [rank1,rank2,...] - Indicates ranks to start in debugger 1163 . -debugger_pause [sleeptime] (in seconds) - Pauses debugger 1164 . -stop_for_debugger - Print message on how to attach debugger manually to 1165 process and wait (-debugger_pause) seconds for attachment 1166 . -malloc_dump - prints a list of all unfreed memory at the end of the run 1167 . -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds, ignored in optimized build. May want to set in PETSC_OPTIONS environmental variable 1168 . -malloc_view - show a list of all allocated memory during `PetscFinalize()` 1169 . -malloc_view_threshold <t> - only list memory allocations of size greater than t with -malloc_view 1170 . -malloc_requested_size - malloc logging will record the requested size rather than size after alignment 1171 . -fp_trap - Stops on floating point exceptions 1172 . -no_signal_handler - Indicates not to trap error signals 1173 . -shared_tmp - indicates /tmp directory is shared by all processors 1174 . -not_shared_tmp - each processor has own /tmp 1175 . -tmp - alternative name of /tmp directory 1176 . -get_total_flops - returns total flops done by all processors 1177 - -memory_view - Print memory usage at end of run 1178 1179 Options Database Keys for Option Database: 1180 + -skip_petscrc - skip the default option files ~/.petscrc, .petscrc, petscrc 1181 . -options_monitor - monitor all set options to standard output for the whole program run 1182 - -options_monitor_cancel - cancel options monitoring hard-wired using `PetscOptionsMonitorSet()` 1183 1184 Options -options_monitor_{all,cancel} are 1185 position-independent and apply to all options set since the PETSc start. 1186 They can be used also in option files. 1187 1188 See `PetscOptionsMonitorSet()` to do monitoring programmatically. 1189 1190 Options Database Keys for Profiling: 1191 See Users-Manual: ch_profiling for details. 1192 + -info [filename][:[~]<list,of,classnames>[:[~]self]] - Prints verbose information. See `PetscInfo()`. 1193 . -log_sync - Enable barrier synchronization for all events. This option is useful to debug imbalance within each event, 1194 however it slows things down and gives a distorted view of the overall runtime. 1195 . -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program 1196 hangs without running in the debugger). See `PetscLogTraceBegin()`. 1197 . -log_view [:filename:format] - Prints summary of flop and timing information to screen or file, see `PetscLogView()`. 1198 . -log_view_memory - Includes in the summary from -log_view the memory used in each event, see `PetscLogView()`. 1199 . -log_view_gpu_time - Includes in the summary from -log_view the time used in each GPU kernel, see `PetscLogView(). 1200 . -log_exclude: <vec,mat,pc,ksp,snes> - excludes subset of object classes from logging 1201 . -log_all [filename] - Logs extensive profiling information See `PetscLogDump()`. 1202 . -log [filename] - Logs basic profiline information See `PetscLogDump()`. 1203 . -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution) 1204 . -viewfromoptions on,off - Enable or disable `XXXSetFromOptions()` calls, for applications with many small solves turn this off 1205 - -check_pointer_intensity 0,1,2 - if pointers are checked for validity (debug version only), using 0 will result in faster code 1206 1207 Only one of -log_trace, -log_view, -log_all, -log, or -log_mpe may be used at a time 1208 1209 Options Database Keys for SAWs: 1210 + -saws_port <portnumber> - port number to publish SAWs data, default is 8080 1211 . -saws_port_auto_select - have SAWs select a new unique port number where it publishes the data, the URL is printed to the screen 1212 this is useful when you are running many jobs that utilize SAWs at the same time 1213 . -saws_log <filename> - save a log of all SAWs communication 1214 . -saws_https <certificate file> - have SAWs use HTTPS instead of HTTP 1215 - -saws_root <directory> - allow SAWs to have access to the given directory to search for requested resources and files 1216 1217 Environmental Variables: 1218 + `PETSC_TMP` - alternative tmp directory 1219 . `PETSC_SHARED_TMP` - tmp is shared by all processes 1220 . `PETSC_NOT_SHARED_TMP` - each process has its own private tmp 1221 . `PETSC_OPTIONS` - a string containing additional options for petsc in the form of command line "-key value" pairs 1222 . `PETSC_OPTIONS_YAML` - (requires configuring PETSc to use libyaml) a string containing additional options for petsc in the form of a YAML document 1223 . `PETSC_VIEWER_SOCKET_PORT` - socket number to use for socket viewer 1224 - `PETSC_VIEWER_SOCKET_MACHINE` - machine to use for socket viewer to connect to 1225 1226 Level: beginner 1227 1228 Note: 1229 If for some reason you must call `MPI_Init()` separately, call 1230 it before `PetscInitialize()`. 1231 1232 Fortran Notes: 1233 In Fortran this routine can be called with 1234 .vb 1235 call PetscInitialize(ierr) 1236 call PetscInitialize(file,ierr) or 1237 call PetscInitialize(file,help,ierr) 1238 .ve 1239 1240 If your main program is C but you call Fortran code that also uses PETSc you need to call `PetscInitializeFortran()` soon after 1241 calling `PetscInitialize()`. 1242 1243 Options Database Key for Developers: 1244 . -checkfunctionlist - automatically checks that function lists associated with objects are correctly cleaned up. Produces messages of the form: 1245 "function name: MatInodeGetInodeSizes_C" if they are not cleaned up. This flag is always set for the test harness (in framework.py) 1246 1247 .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscInitializeNoArguments()`, `PetscLogGpuTime()` 1248 @*/ 1249 PetscErrorCode PetscInitialize(int *argc, char ***args, const char file[], const char help[]) 1250 { 1251 PetscMPIInt flag; 1252 const char *prog = "Unknown Name", *mpienv; 1253 1254 PetscFunctionBegin; 1255 if (PetscInitializeCalled) PetscFunctionReturn(PETSC_SUCCESS); 1256 PetscCallMPI(MPI_Initialized(&flag)); 1257 if (!flag) { 1258 PetscCheck(PETSC_COMM_WORLD == MPI_COMM_NULL, PETSC_COMM_SELF, PETSC_ERR_SUP, "You cannot set PETSC_COMM_WORLD if you have not initialized MPI first"); 1259 PetscCall(PetscPreMPIInit_Private()); 1260 #if defined(PETSC_HAVE_MPI_INIT_THREAD) 1261 { 1262 PetscMPIInt PETSC_UNUSED provided; 1263 PetscCallMPI(MPI_Init_thread(argc, args, PETSC_MPI_THREAD_REQUIRED, &provided)); 1264 } 1265 #else 1266 PetscCallMPI(MPI_Init(argc, args)); 1267 #endif 1268 if (PetscDefined(HAVE_MPIUNI)) { 1269 mpienv = getenv("PMI_SIZE"); 1270 if (!mpienv) mpienv = getenv("OMPI_COMM_WORLD_SIZE"); 1271 if (mpienv) { 1272 PetscInt isize; 1273 PetscCall(PetscOptionsStringToInt(mpienv, &isize)); 1274 if (isize != 1) printf("You are using an MPI-uni (sequential) install of PETSc but trying to launch parallel jobs; you need full MPI version of PETSc\n"); 1275 PetscCheck(isize == 1, MPI_COMM_SELF, PETSC_ERR_MPI, "You are using an MPI-uni (sequential) install of PETSc but trying to launch parallel jobs; you need full MPI version of PETSc"); 1276 } 1277 } 1278 PetscBeganMPI = PETSC_TRUE; 1279 } 1280 1281 if (argc && *argc) prog = **args; 1282 if (argc && args) { 1283 PetscGlobalArgc = *argc; 1284 PetscGlobalArgs = *args; 1285 } 1286 PetscCall(PetscInitialize_Common(prog, file, help, PETSC_FALSE, PETSC_FALSE, 0)); 1287 PetscFunctionReturn(PETSC_SUCCESS); 1288 } 1289 1290 #if PetscDefined(USE_LOG) 1291 PETSC_INTERN PetscObject *PetscObjects; 1292 PETSC_INTERN PetscInt PetscObjectsCounts; 1293 PETSC_INTERN PetscInt PetscObjectsMaxCounts; 1294 PETSC_INTERN PetscBool PetscObjectsLog; 1295 #endif 1296 1297 /* 1298 Frees all the MPI types and operations that PETSc may have created 1299 */ 1300 PetscErrorCode PetscFreeMPIResources(void) 1301 { 1302 PetscFunctionBegin; 1303 #if defined(PETSC_HAVE_REAL___FLOAT128) 1304 PetscCallMPI(MPI_Type_free(&MPIU___FLOAT128)); 1305 #if defined(PETSC_HAVE_COMPLEX) 1306 PetscCallMPI(MPI_Type_free(&MPIU___COMPLEX128)); 1307 #endif 1308 #endif 1309 #if defined(PETSC_HAVE_REAL___FP16) 1310 PetscCallMPI(MPI_Type_free(&MPIU___FP16)); 1311 #endif 1312 1313 #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16) 1314 PetscCallMPI(MPI_Op_free(&MPIU_SUM)); 1315 PetscCallMPI(MPI_Op_free(&MPIU_MAX)); 1316 PetscCallMPI(MPI_Op_free(&MPIU_MIN)); 1317 #elif defined(PETSC_HAVE_REAL___FLOAT128) || defined(PETSC_HAVE_REAL___FP16) 1318 PetscCallMPI(MPI_Op_free(&MPIU_SUM___FP16___FLOAT128)); 1319 #endif 1320 1321 PetscCallMPI(MPI_Type_free(&MPIU_2SCALAR)); 1322 PetscCallMPI(MPI_Type_free(&MPIU_REAL_INT)); 1323 PetscCallMPI(MPI_Type_free(&MPIU_SCALAR_INT)); 1324 #if defined(PETSC_USE_64BIT_INDICES) 1325 PetscCallMPI(MPI_Type_free(&MPIU_2INT)); 1326 #endif 1327 PetscCallMPI(MPI_Type_free(&MPI_4INT)); 1328 PetscCallMPI(MPI_Type_free(&MPIU_4INT)); 1329 PetscCallMPI(MPI_Op_free(&MPIU_MAXSUM_OP)); 1330 PetscCallMPI(MPI_Op_free(&Petsc_Garbage_SetIntersectOp)); 1331 PetscFunctionReturn(PETSC_SUCCESS); 1332 } 1333 1334 #if PetscDefined(USE_LOG) 1335 PETSC_INTERN PetscErrorCode PetscLogFinalize(void); 1336 #endif 1337 1338 /*@C 1339 PetscFinalize - Checks for options to be called at the conclusion 1340 of the program. `MPI_Finalize()` is called only if the user had not 1341 called `MPI_Init()` before calling `PetscInitialize()`. 1342 1343 Collective on `PETSC_COMM_WORLD` 1344 1345 Options Database Keys: 1346 + -options_view - Calls `PetscOptionsView()` 1347 . -options_left - Prints unused options that remain in the database 1348 . -objects_dump [all] - Prints list of objects allocated by the user that have not been freed, the option all cause all outstanding objects to be listed 1349 . -mpidump - Calls PetscMPIDump() 1350 . -malloc_dump <optional filename> - Calls `PetscMallocDump()`, displays all memory allocated that has not been freed 1351 . -memory_view - Prints total memory usage 1352 - -malloc_view <optional filename> - Prints list of all memory allocated and in what functions 1353 1354 Level: beginner 1355 1356 Note: 1357 See `PetscInitialize()` for other runtime options. 1358 1359 .seealso: `PetscInitialize()`, `PetscOptionsView()`, `PetscMallocDump()`, `PetscMPIDump()`, `PetscEnd()` 1360 @*/ 1361 PetscErrorCode PetscFinalize(void) 1362 { 1363 PetscMPIInt rank; 1364 PetscInt nopt; 1365 PetscBool flg1 = PETSC_FALSE, flg2 = PETSC_FALSE, flg3 = PETSC_FALSE; 1366 PetscBool flg; 1367 #if defined(PETSC_USE_LOG) 1368 char mname[PETSC_MAX_PATH_LEN]; 1369 #endif 1370 1371 PetscFunctionBegin; 1372 PetscCheck(PetscInitializeCalled, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "PetscInitialize() must be called before PetscFinalize()"); 1373 PetscCall(PetscInfo(NULL, "PetscFinalize() called\n")); 1374 1375 PetscCall(PetscOptionsHasName(NULL, NULL, "-mpi_linear_solver_server", &flg)); 1376 if (PetscDefined(USE_SINGLE_LIBRARY) && flg) PetscCall(PCMPIServerEnd()); 1377 1378 /* Clean up Garbage automatically on COMM_SELF and COMM_WORLD at finalize */ 1379 { 1380 union 1381 { 1382 MPI_Comm comm; 1383 void *ptr; 1384 } ucomm; 1385 PetscMPIInt flg; 1386 void *tmp; 1387 1388 PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_SELF, Petsc_InnerComm_keyval, &ucomm, &flg)); 1389 if (flg) PetscCallMPI(MPI_Comm_get_attr(ucomm.comm, Petsc_Garbage_HMap_keyval, &tmp, &flg)); 1390 if (flg) PetscCall(PetscGarbageCleanup(PETSC_COMM_SELF)); 1391 PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_WORLD, Petsc_InnerComm_keyval, &ucomm, &flg)); 1392 if (flg) PetscCallMPI(MPI_Comm_get_attr(ucomm.comm, Petsc_Garbage_HMap_keyval, &tmp, &flg)); 1393 if (flg) PetscCall(PetscGarbageCleanup(PETSC_COMM_WORLD)); 1394 } 1395 1396 PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank)); 1397 #if defined(PETSC_HAVE_ADIOS) 1398 PetscCallExternal(adios_read_finalize_method, ADIOS_READ_METHOD_BP_AGGREGATE); 1399 PetscCallExternal(adios_finalize, rank); 1400 #endif 1401 PetscCall(PetscOptionsHasName(NULL, NULL, "-citations", &flg)); 1402 if (flg) { 1403 char *cits, filename[PETSC_MAX_PATH_LEN]; 1404 FILE *fd = PETSC_STDOUT; 1405 1406 PetscCall(PetscOptionsGetString(NULL, NULL, "-citations", filename, sizeof(filename), NULL)); 1407 if (filename[0]) PetscCall(PetscFOpen(PETSC_COMM_WORLD, filename, "w", &fd)); 1408 PetscCall(PetscSegBufferGet(PetscCitationsList, 1, &cits)); 1409 cits[0] = 0; 1410 PetscCall(PetscSegBufferExtractAlloc(PetscCitationsList, &cits)); 1411 PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "If you publish results based on this computation please cite the following:\n")); 1412 PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "===========================================================================\n")); 1413 PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "%s", cits)); 1414 PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "===========================================================================\n")); 1415 PetscCall(PetscFClose(PETSC_COMM_WORLD, fd)); 1416 PetscCall(PetscFree(cits)); 1417 } 1418 PetscCall(PetscSegBufferDestroy(&PetscCitationsList)); 1419 1420 #if defined(PETSC_SERIALIZE_FUNCTIONS) 1421 PetscCall(PetscFPTDestroy()); 1422 #endif 1423 1424 #if defined(PETSC_HAVE_SAWS) 1425 flg = PETSC_FALSE; 1426 PetscCall(PetscOptionsGetBool(NULL, NULL, "-saw_options", &flg, NULL)); 1427 if (flg) PetscCall(PetscOptionsSAWsDestroy()); 1428 #endif 1429 1430 #if defined(PETSC_HAVE_X) 1431 flg1 = PETSC_FALSE; 1432 PetscCall(PetscOptionsGetBool(NULL, NULL, "-x_virtual", &flg1, NULL)); 1433 if (flg1) { 1434 /* this is a crude hack, but better than nothing */ 1435 PetscCall(PetscPOpen(PETSC_COMM_WORLD, NULL, "pkill -9 Xvfb", "r", NULL)); 1436 } 1437 #endif 1438 1439 #if !defined(PETSC_HAVE_THREADSAFETY) 1440 PetscCall(PetscOptionsGetBool(NULL, NULL, "-memory_view", &flg2, NULL)); 1441 if (flg2) PetscCall(PetscMemoryView(PETSC_VIEWER_STDOUT_WORLD, "Summary of Memory Usage in PETSc\n")); 1442 #endif 1443 1444 #if defined(PETSC_USE_LOG) 1445 flg1 = PETSC_FALSE; 1446 PetscCall(PetscOptionsGetBool(NULL, NULL, "-get_total_flops", &flg1, NULL)); 1447 if (flg1) { 1448 PetscLogDouble flops = 0; 1449 PetscCallMPI(MPI_Reduce(&petsc_TotalFlops, &flops, 1, MPI_DOUBLE, MPI_SUM, 0, PETSC_COMM_WORLD)); 1450 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Total flops over all processors %g\n", flops)); 1451 } 1452 #endif 1453 1454 #if defined(PETSC_USE_LOG) 1455 #if defined(PETSC_HAVE_MPE) 1456 mname[0] = 0; 1457 PetscCall(PetscOptionsGetString(NULL, NULL, "-log_mpe", mname, sizeof(mname), &flg1)); 1458 if (flg1) { 1459 if (mname[0]) PetscCall(PetscLogMPEDump(mname)); 1460 else PetscCall(PetscLogMPEDump(0)); 1461 } 1462 #endif 1463 #endif 1464 1465 /* 1466 Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_(). 1467 */ 1468 PetscCall(PetscObjectRegisterDestroyAll()); 1469 1470 #if defined(PETSC_USE_LOG) 1471 PetscCall(PetscOptionsPushGetViewerOff(PETSC_FALSE)); 1472 PetscCall(PetscLogViewFromOptions()); 1473 PetscCall(PetscOptionsPopGetViewerOff()); 1474 1475 /* 1476 Free any objects created by the last block of code. 1477 */ 1478 PetscCall(PetscObjectRegisterDestroyAll()); 1479 1480 mname[0] = 0; 1481 PetscCall(PetscOptionsGetString(NULL, NULL, "-log_all", mname, sizeof(mname), &flg1)); 1482 PetscCall(PetscOptionsGetString(NULL, NULL, "-log", mname, sizeof(mname), &flg2)); 1483 if (flg1 || flg2) PetscCall(PetscLogDump(mname)); 1484 #endif 1485 1486 flg1 = PETSC_FALSE; 1487 PetscCall(PetscOptionsGetBool(NULL, NULL, "-no_signal_handler", &flg1, NULL)); 1488 if (!flg1) PetscCall(PetscPopSignalHandler()); 1489 flg1 = PETSC_FALSE; 1490 PetscCall(PetscOptionsGetBool(NULL, NULL, "-mpidump", &flg1, NULL)); 1491 if (flg1) PetscCall(PetscMPIDump(stdout)); 1492 flg1 = PETSC_FALSE; 1493 flg2 = PETSC_FALSE; 1494 /* preemptive call to avoid listing this option in options table as unused */ 1495 PetscCall(PetscOptionsHasName(NULL, NULL, "-malloc_dump", &flg1)); 1496 PetscCall(PetscOptionsHasName(NULL, NULL, "-objects_dump", &flg1)); 1497 PetscCall(PetscOptionsGetBool(NULL, NULL, "-options_view", &flg2, NULL)); 1498 1499 if (flg2) { 1500 PetscViewer viewer; 1501 PetscCall(PetscViewerCreate(PETSC_COMM_WORLD, &viewer)); 1502 PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII)); 1503 PetscCall(PetscOptionsView(NULL, viewer)); 1504 PetscCall(PetscViewerDestroy(&viewer)); 1505 } 1506 1507 /* to prevent PETSc -options_left from warning */ 1508 PetscCall(PetscOptionsHasName(NULL, NULL, "-nox", &flg1)); 1509 PetscCall(PetscOptionsHasName(NULL, NULL, "-nox_warning", &flg1)); 1510 1511 flg3 = PETSC_FALSE; /* default value is required */ 1512 PetscCall(PetscOptionsGetBool(NULL, NULL, "-options_left", &flg3, &flg1)); 1513 if (!flg1) flg3 = PETSC_TRUE; 1514 if (flg3) { 1515 if (!flg2 && flg1) { /* have not yet printed the options */ 1516 PetscViewer viewer; 1517 PetscCall(PetscViewerCreate(PETSC_COMM_WORLD, &viewer)); 1518 PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII)); 1519 PetscCall(PetscOptionsView(NULL, viewer)); 1520 PetscCall(PetscViewerDestroy(&viewer)); 1521 } 1522 PetscCall(PetscOptionsAllUsed(NULL, &nopt)); 1523 if (nopt) { 1524 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "WARNING! There are options you set that were not used!\n")); 1525 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "WARNING! could be spelling mistake, etc!\n")); 1526 if (nopt == 1) { 1527 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "There is one unused database option. It is:\n")); 1528 } else { 1529 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "There are %" PetscInt_FMT " unused database options. They are:\n", nopt)); 1530 } 1531 } else if (flg3 && flg1) { 1532 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "There are no unused options.\n")); 1533 } 1534 PetscCall(PetscOptionsLeft(NULL)); 1535 } 1536 1537 #if defined(PETSC_HAVE_SAWS) 1538 if (!PetscGlobalRank) { 1539 PetscCall(PetscStackSAWsViewOff()); 1540 PetscCallSAWs(SAWs_Finalize, ()); 1541 } 1542 #endif 1543 1544 #if defined(PETSC_USE_LOG) 1545 /* 1546 List all objects the user may have forgot to free 1547 */ 1548 if (PetscObjectsLog) { 1549 PetscCall(PetscOptionsHasName(NULL, NULL, "-objects_dump", &flg1)); 1550 if (flg1) { 1551 MPI_Comm local_comm; 1552 char string[64]; 1553 1554 PetscCall(PetscOptionsGetString(NULL, NULL, "-objects_dump", string, sizeof(string), NULL)); 1555 PetscCallMPI(MPI_Comm_dup(PETSC_COMM_WORLD, &local_comm)); 1556 PetscCall(PetscSequentialPhaseBegin_Private(local_comm, 1)); 1557 PetscCall(PetscObjectsDump(stdout, (string[0] == 'a') ? PETSC_TRUE : PETSC_FALSE)); 1558 PetscCall(PetscSequentialPhaseEnd_Private(local_comm, 1)); 1559 PetscCallMPI(MPI_Comm_free(&local_comm)); 1560 } 1561 } 1562 #endif 1563 1564 #if defined(PETSC_USE_LOG) 1565 PetscObjectsCounts = 0; 1566 PetscObjectsMaxCounts = 0; 1567 PetscCall(PetscFree(PetscObjects)); 1568 #endif 1569 1570 /* 1571 Destroy any packages that registered a finalize 1572 */ 1573 PetscCall(PetscRegisterFinalizeAll()); 1574 1575 #if defined(PETSC_USE_LOG) 1576 PetscCall(PetscLogFinalize()); 1577 #endif 1578 1579 /* 1580 Print PetscFunctionLists that have not been properly freed 1581 */ 1582 if (PetscPrintFunctionList) PetscCall(PetscFunctionListPrintAll()); 1583 1584 if (petsc_history) { 1585 PetscCall(PetscCloseHistoryFile(&petsc_history)); 1586 petsc_history = NULL; 1587 } 1588 PetscCall(PetscOptionsHelpPrintedDestroy(&PetscOptionsHelpPrintedSingleton)); 1589 PetscCall(PetscInfoDestroy()); 1590 1591 #if !defined(PETSC_HAVE_THREADSAFETY) 1592 if (!(PETSC_RUNNING_ON_VALGRIND)) { 1593 char fname[PETSC_MAX_PATH_LEN]; 1594 char sname[PETSC_MAX_PATH_LEN]; 1595 FILE *fd; 1596 int err; 1597 1598 flg2 = PETSC_FALSE; 1599 flg3 = PETSC_FALSE; 1600 if (PetscDefined(USE_DEBUG)) PetscCall(PetscOptionsGetBool(NULL, NULL, "-malloc_test", &flg2, NULL)); 1601 PetscCall(PetscOptionsGetBool(NULL, NULL, "-malloc_debug", &flg3, NULL)); 1602 fname[0] = 0; 1603 PetscCall(PetscOptionsGetString(NULL, NULL, "-malloc_dump", fname, sizeof(fname), &flg1)); 1604 if (flg1 && fname[0]) { 1605 PetscCall(PetscSNPrintf(sname, sizeof(sname), "%s_%d", fname, rank)); 1606 fd = fopen(sname, "w"); 1607 PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Cannot open log file: %s", sname); 1608 PetscCall(PetscMallocDump(fd)); 1609 err = fclose(fd); 1610 PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file"); 1611 } else if (flg1 || flg2 || flg3) { 1612 MPI_Comm local_comm; 1613 1614 PetscCallMPI(MPI_Comm_dup(PETSC_COMM_WORLD, &local_comm)); 1615 PetscCall(PetscSequentialPhaseBegin_Private(local_comm, 1)); 1616 PetscCall(PetscMallocDump(stdout)); 1617 PetscCall(PetscSequentialPhaseEnd_Private(local_comm, 1)); 1618 PetscCallMPI(MPI_Comm_free(&local_comm)); 1619 } 1620 fname[0] = 0; 1621 PetscCall(PetscOptionsGetString(NULL, NULL, "-malloc_view", fname, sizeof(fname), &flg1)); 1622 if (flg1 && fname[0]) { 1623 PetscCall(PetscSNPrintf(sname, sizeof(sname), "%s_%d", fname, rank)); 1624 fd = fopen(sname, "w"); 1625 PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Cannot open log file: %s", sname); 1626 PetscCall(PetscMallocView(fd)); 1627 err = fclose(fd); 1628 PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file"); 1629 } else if (flg1) { 1630 MPI_Comm local_comm; 1631 1632 PetscCallMPI(MPI_Comm_dup(PETSC_COMM_WORLD, &local_comm)); 1633 PetscCall(PetscSequentialPhaseBegin_Private(local_comm, 1)); 1634 PetscCall(PetscMallocView(stdout)); 1635 PetscCall(PetscSequentialPhaseEnd_Private(local_comm, 1)); 1636 PetscCallMPI(MPI_Comm_free(&local_comm)); 1637 } 1638 } 1639 #endif 1640 1641 /* 1642 Close any open dynamic libraries 1643 */ 1644 PetscCall(PetscFinalize_DynamicLibraries()); 1645 1646 /* Can be destroyed only after all the options are used */ 1647 PetscCall(PetscOptionsDestroyDefault()); 1648 1649 PetscGlobalArgc = 0; 1650 PetscGlobalArgs = NULL; 1651 1652 #if defined(PETSC_HAVE_KOKKOS) 1653 if (PetscBeganKokkos) { 1654 PetscCall(PetscKokkosFinalize_Private()); 1655 PetscBeganKokkos = PETSC_FALSE; 1656 PetscKokkosInitialized = PETSC_FALSE; 1657 } 1658 #endif 1659 1660 #if defined(PETSC_HAVE_NVSHMEM) 1661 if (PetscBeganNvshmem) { 1662 PetscCall(PetscNvshmemFinalize()); 1663 PetscBeganNvshmem = PETSC_FALSE; 1664 } 1665 #endif 1666 1667 PetscCall(PetscFreeMPIResources()); 1668 1669 /* 1670 Destroy any known inner MPI_Comm's and attributes pointing to them 1671 Note this will not destroy any new communicators the user has created. 1672 1673 If all PETSc objects were not destroyed those left over objects will have hanging references to 1674 the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again 1675 */ 1676 { 1677 PetscCommCounter *counter; 1678 PetscMPIInt flg; 1679 MPI_Comm icomm; 1680 union 1681 { 1682 MPI_Comm comm; 1683 void *ptr; 1684 } ucomm; 1685 PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_SELF, Petsc_InnerComm_keyval, &ucomm, &flg)); 1686 if (flg) { 1687 icomm = ucomm.comm; 1688 PetscCallMPI(MPI_Comm_get_attr(icomm, Petsc_Counter_keyval, &counter, &flg)); 1689 PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory"); 1690 1691 PetscCallMPI(MPI_Comm_delete_attr(PETSC_COMM_SELF, Petsc_InnerComm_keyval)); 1692 PetscCallMPI(MPI_Comm_delete_attr(icomm, Petsc_Counter_keyval)); 1693 PetscCallMPI(MPI_Comm_free(&icomm)); 1694 } 1695 PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_WORLD, Petsc_InnerComm_keyval, &ucomm, &flg)); 1696 if (flg) { 1697 icomm = ucomm.comm; 1698 PetscCallMPI(MPI_Comm_get_attr(icomm, Petsc_Counter_keyval, &counter, &flg)); 1699 PetscCheck(flg, PETSC_COMM_WORLD, PETSC_ERR_ARG_CORRUPT, "Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory"); 1700 1701 PetscCallMPI(MPI_Comm_delete_attr(PETSC_COMM_WORLD, Petsc_InnerComm_keyval)); 1702 PetscCallMPI(MPI_Comm_delete_attr(icomm, Petsc_Counter_keyval)); 1703 PetscCallMPI(MPI_Comm_free(&icomm)); 1704 } 1705 } 1706 1707 PetscCallMPI(MPI_Comm_free_keyval(&Petsc_Counter_keyval)); 1708 PetscCallMPI(MPI_Comm_free_keyval(&Petsc_InnerComm_keyval)); 1709 PetscCallMPI(MPI_Comm_free_keyval(&Petsc_OuterComm_keyval)); 1710 PetscCallMPI(MPI_Comm_free_keyval(&Petsc_ShmComm_keyval)); 1711 PetscCallMPI(MPI_Comm_free_keyval(&Petsc_CreationIdx_keyval)); 1712 PetscCallMPI(MPI_Comm_free_keyval(&Petsc_Garbage_HMap_keyval)); 1713 1714 // Free keyvals which may be silently created by some routines 1715 if (Petsc_SharedWD_keyval != MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_free_keyval(&Petsc_SharedWD_keyval)); 1716 if (Petsc_SharedTmp_keyval != MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_free_keyval(&Petsc_SharedTmp_keyval)); 1717 1718 PetscCall(PetscSpinlockDestroy(&PetscViewerASCIISpinLockOpen)); 1719 PetscCall(PetscSpinlockDestroy(&PetscViewerASCIISpinLockStdout)); 1720 PetscCall(PetscSpinlockDestroy(&PetscViewerASCIISpinLockStderr)); 1721 PetscCall(PetscSpinlockDestroy(&PetscCommSpinLock)); 1722 1723 if (PetscBeganMPI) { 1724 PetscMPIInt flag; 1725 PetscCallMPI(MPI_Finalized(&flag)); 1726 PetscCheck(!flag, PETSC_COMM_SELF, PETSC_ERR_LIB, "MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()"); 1727 /* wait until the very last moment to disable error handling */ 1728 PetscErrorHandlingInitialized = PETSC_FALSE; 1729 PetscCallMPI(MPI_Finalize()); 1730 } else PetscErrorHandlingInitialized = PETSC_FALSE; 1731 1732 /* 1733 1734 Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because 1735 the communicator has some outstanding requests on it. Specifically if the 1736 flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See 1737 src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate() 1738 is never freed as it should be. Thus one may obtain messages of the form 1739 [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the 1740 memory was not freed. 1741 1742 */ 1743 PetscCall(PetscMallocClear()); 1744 PetscCall(PetscStackReset()); 1745 1746 PetscInitializeCalled = PETSC_FALSE; 1747 PetscFinalizeCalled = PETSC_TRUE; 1748 #if defined(PETSC_USE_COVERAGE) 1749 /* 1750 flush gcov, otherwise during CI the flushing continues into the next pipeline resulting in git not being able to delete directories since the 1751 gcov files are still being added to the directories as git tries to remove the directories. 1752 */ 1753 __gcov_flush(); 1754 #endif 1755 /* To match PetscFunctionBegin() at the beginning of this function */ 1756 PetscStackClearTop; 1757 return PETSC_SUCCESS; 1758 } 1759 1760 #if defined(PETSC_MISSING_LAPACK_lsame_) 1761 PETSC_EXTERN int lsame_(char *a, char *b) 1762 { 1763 if (*a == *b) return 1; 1764 if (*a + 32 == *b) return 1; 1765 if (*a - 32 == *b) return 1; 1766 return 0; 1767 } 1768 #endif 1769 1770 #if defined(PETSC_MISSING_LAPACK_lsame) 1771 PETSC_EXTERN int lsame(char *a, char *b) 1772 { 1773 if (*a == *b) return 1; 1774 if (*a + 32 == *b) return 1; 1775 if (*a - 32 == *b) return 1; 1776 return 0; 1777 } 1778 #endif 1779