1 2 /* 3 This file defines the initialization of PETSc, including PetscInitialize() 4 */ 5 #define PETSC_DESIRE_COMPLEX 6 #include <petsc-private/petscimpl.h> /*I "petscsys.h" I*/ 7 #include <petscviewer.h> 8 9 #if defined(PETSC_HAVE_CUDA) 10 #include <cublas.h> 11 #endif 12 13 #include <petscthreadcomm.h> 14 15 #if defined(PETSC_USE_LOG) 16 extern PetscErrorCode PetscLogBegin_Private(void); 17 #endif 18 extern PetscBool PetscHMPIWorker; 19 20 21 #if defined(PETSC_SERIALIZE_FUNCTIONS) 22 PetscFPT PetscFPTData = 0; 23 #endif 24 25 /* -----------------------------------------------------------------------------------------*/ 26 27 extern FILE *petsc_history; 28 29 extern PetscErrorCode PetscInitialize_DynamicLibraries(void); 30 extern PetscErrorCode PetscFinalize_DynamicLibraries(void); 31 extern PetscErrorCode PetscFunctionListPrintAll(void); 32 extern PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm,int); 33 extern PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm,int); 34 extern PetscErrorCode PetscCloseHistoryFile(FILE**); 35 36 /* user may set this BEFORE calling PetscInitialize() */ 37 MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL; 38 39 PetscMPIInt Petsc_Counter_keyval = MPI_KEYVAL_INVALID; 40 PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID; 41 PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID; 42 43 /* 44 Declare and set all the string names of the PETSc enums 45 */ 46 const char *const PetscBools[] = {"FALSE","TRUE","PetscBool","PETSC_",0}; 47 const char *const PetscCopyModes[] = {"COPY_VALUES","OWN_POINTER","USE_POINTER","PetscCopyMode","PETSC_",0}; 48 const char *const PetscDataTypes[] = {"INT","DOUBLE","COMPLEX","LONG","SHORT","FLOAT", 49 "CHAR","LOGICAL","ENUM","BOOL","LONGDOUBLE","OBJECT","FUNCTION","PetscDataType","PETSC_",0}; 50 51 PetscBool PetscPreLoadingUsed = PETSC_FALSE; 52 PetscBool PetscPreLoadingOn = PETSC_FALSE; 53 54 /* pthread_key for PetscStack */ 55 #if defined(PETSC_HAVE_PTHREADCLASSES) && !defined(PETSC_PTHREAD_LOCAL) 56 pthread_key_t petscstack; 57 #endif 58 59 /* 60 Checks the options database for initializations related to the 61 PETSc components 62 */ 63 #undef __FUNCT__ 64 #define __FUNCT__ "PetscOptionsCheckInitial_Components" 65 PetscErrorCode PetscOptionsCheckInitial_Components(void) 66 { 67 PetscBool flg1; 68 PetscErrorCode ierr; 69 70 PetscFunctionBegin; 71 ierr = PetscOptionsHasName(NULL,"-help",&flg1);CHKERRQ(ierr); 72 if (flg1) { 73 #if defined(PETSC_USE_LOG) 74 MPI_Comm comm = PETSC_COMM_WORLD; 75 ierr = (*PetscHelpPrintf)(comm,"------Additional PETSc component options--------\n");CHKERRQ(ierr); 76 ierr = (*PetscHelpPrintf)(comm," -log_summary_exclude: <vec,mat,pc.ksp,snes>\n");CHKERRQ(ierr); 77 ierr = (*PetscHelpPrintf)(comm," -info_exclude: <null,vec,mat,pc,ksp,snes,ts>\n");CHKERRQ(ierr); 78 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 79 #endif 80 } 81 PetscFunctionReturn(0); 82 } 83 84 #undef __FUNCT__ 85 #define __FUNCT__ "PetscInitializeNoPointers" 86 /* 87 PetscInitializeNoPointers - Calls PetscInitialize() from C/C++ without the pointers to argc and args 88 89 Collective 90 91 Level: advanced 92 93 Notes: this is called only by the PETSc MATLAB and Julia interface. Even though it might start MPI it sets the flag to 94 indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to 95 be called multiple times from MATLAB and Julia without the problem of trying to initialize MPI more than once. 96 97 Turns off PETSc signal handling because that can interact with MATLAB's signal handling causing random crashes. 98 99 .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments() 100 */ 101 PetscErrorCode PetscInitializeNoPointers(int argc,char **args,const char *filename,const char *help) 102 { 103 PetscErrorCode ierr; 104 int myargc = argc; 105 char **myargs = args; 106 107 PetscFunctionBegin; 108 ierr = PetscInitialize(&myargc,&myargs,filename,help);CHKERRQ(ierr); 109 ierr = PetscPopSignalHandler();CHKERRQ(ierr); 110 PetscBeganMPI = PETSC_FALSE; 111 PetscFunctionReturn(ierr); 112 } 113 114 #undef __FUNCT__ 115 #define __FUNCT__ "PetscGetPETSC_COMM_SELF" 116 /* 117 Used by MATLAB and Julia interface to get communicator 118 */ 119 PetscErrorCode PetscGetPETSC_COMM_SELF(MPI_Comm *comm) 120 { 121 PetscFunctionBegin; 122 *comm = PETSC_COMM_SELF; 123 PetscFunctionReturn(0); 124 } 125 126 #undef __FUNCT__ 127 #define __FUNCT__ "PetscInitializeNoArguments" 128 /*@C 129 PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without 130 the command line arguments. 131 132 Collective 133 134 Level: advanced 135 136 .seealso: PetscInitialize(), PetscInitializeFortran() 137 @*/ 138 PetscErrorCode PetscInitializeNoArguments(void) 139 { 140 PetscErrorCode ierr; 141 int argc = 0; 142 char **args = 0; 143 144 PetscFunctionBegin; 145 ierr = PetscInitialize(&argc,&args,NULL,NULL); 146 PetscFunctionReturn(ierr); 147 } 148 149 #undef __FUNCT__ 150 #define __FUNCT__ "PetscInitialized" 151 /*@ 152 PetscInitialized - Determine whether PETSc is initialized. 153 154 Level: beginner 155 156 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran() 157 @*/ 158 PetscErrorCode PetscInitialized(PetscBool *isInitialized) 159 { 160 *isInitialized = PetscInitializeCalled; 161 return 0; 162 } 163 164 #undef __FUNCT__ 165 #define __FUNCT__ "PetscFinalized" 166 /*@ 167 PetscFinalized - Determine whether PetscFinalize() has been called yet 168 169 Level: developer 170 171 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran() 172 @*/ 173 PetscErrorCode PetscFinalized(PetscBool *isFinalized) 174 { 175 *isFinalized = PetscFinalizeCalled; 176 return 0; 177 } 178 179 extern PetscErrorCode PetscOptionsCheckInitial_Private(void); 180 181 /* 182 This function is the MPI reduction operation used to compute the sum of the 183 first half of the datatype and the max of the second half. 184 */ 185 MPI_Op PetscMaxSum_Op = 0; 186 187 #undef __FUNCT__ 188 #define __FUNCT__ "PetscMaxSum_Local" 189 PETSC_EXTERN void MPIAPI PetscMaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype) 190 { 191 PetscInt *xin = (PetscInt*)in,*xout = (PetscInt*)out,i,count = *cnt; 192 193 PetscFunctionBegin; 194 if (*datatype != MPIU_2INT) { 195 (*PetscErrorPrintf)("Can only handle MPIU_2INT data types"); 196 MPI_Abort(MPI_COMM_WORLD,1); 197 } 198 199 for (i=0; i<count; i++) { 200 xout[2*i] = PetscMax(xout[2*i],xin[2*i]); 201 xout[2*i+1] += xin[2*i+1]; 202 } 203 PetscFunctionReturnVoid(); 204 } 205 206 /* 207 Returns the max of the first entry owned by this processor and the 208 sum of the second entry. 209 210 The reason nprocs[2*i] contains lengths nprocs[2*i+1] contains flag of 1 if length is nonzero 211 is so that the PetscMaxSum_Op() can set TWO values, if we passed in only nprocs[i] with lengths 212 there would be no place to store the both needed results. 213 */ 214 #undef __FUNCT__ 215 #define __FUNCT__ "PetscMaxSum" 216 PetscErrorCode PetscMaxSum(MPI_Comm comm,const PetscInt nprocs[],PetscInt *max,PetscInt *sum) 217 { 218 PetscMPIInt size,rank; 219 struct {PetscInt max,sum;} *work; 220 PetscErrorCode ierr; 221 222 PetscFunctionBegin; 223 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 224 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 225 ierr = PetscMalloc(size*sizeof(*work),&work);CHKERRQ(ierr); 226 ierr = MPI_Allreduce((void*)nprocs,work,size,MPIU_2INT,PetscMaxSum_Op,comm);CHKERRQ(ierr); 227 *max = work[rank].max; 228 *sum = work[rank].sum; 229 ierr = PetscFree(work);CHKERRQ(ierr); 230 PetscFunctionReturn(0); 231 } 232 233 /* ----------------------------------------------------------------------------*/ 234 MPI_Op PetscADMax_Op = 0; 235 236 #undef __FUNCT__ 237 #define __FUNCT__ "PetscADMax_Local" 238 PETSC_EXTERN void MPIAPI PetscADMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 239 { 240 PetscScalar *xin = (PetscScalar*)in,*xout = (PetscScalar*)out; 241 PetscInt i,count = *cnt; 242 243 PetscFunctionBegin; 244 if (*datatype != MPIU_2SCALAR) { 245 (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types"); 246 MPI_Abort(MPI_COMM_WORLD,1); 247 } 248 249 for (i=0; i<count; i++) { 250 if (PetscRealPart(xout[2*i]) < PetscRealPart(xin[2*i])) { 251 xout[2*i] = xin[2*i]; 252 xout[2*i+1] = xin[2*i+1]; 253 } 254 } 255 PetscFunctionReturnVoid(); 256 } 257 258 MPI_Op PetscADMin_Op = 0; 259 260 #undef __FUNCT__ 261 #define __FUNCT__ "PetscADMin_Local" 262 PETSC_EXTERN void MPIAPI PetscADMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 263 { 264 PetscScalar *xin = (PetscScalar*)in,*xout = (PetscScalar*)out; 265 PetscInt i,count = *cnt; 266 267 PetscFunctionBegin; 268 if (*datatype != MPIU_2SCALAR) { 269 (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types"); 270 MPI_Abort(MPI_COMM_WORLD,1); 271 } 272 273 for (i=0; i<count; i++) { 274 if (PetscRealPart(xout[2*i]) > PetscRealPart(xin[2*i])) { 275 xout[2*i] = xin[2*i]; 276 xout[2*i+1] = xin[2*i+1]; 277 } 278 } 279 PetscFunctionReturnVoid(); 280 } 281 /* ---------------------------------------------------------------------------------------*/ 282 283 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 284 MPI_Op MPIU_SUM = 0; 285 286 #undef __FUNCT__ 287 #define __FUNCT__ "PetscSum_Local" 288 PETSC_EXTERN void PetscSum_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 289 { 290 PetscInt i,count = *cnt; 291 292 PetscFunctionBegin; 293 if (*datatype == MPIU_REAL) { 294 PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out; 295 for (i=0; i<count; i++) xout[i] += xin[i]; 296 } 297 #if defined(PETSC_HAVE_COMPLEX) 298 else if (*datatype == MPIU_COMPLEX) { 299 PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out; 300 for (i=0; i<count; i++) xout[i] += xin[i]; 301 } 302 #endif 303 else { 304 (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types"); 305 MPI_Abort(MPI_COMM_WORLD,1); 306 } 307 PetscFunctionReturnVoid(); 308 } 309 #endif 310 311 #if defined(PETSC_USE_REAL___FLOAT128) 312 MPI_Op MPIU_MAX = 0; 313 MPI_Op MPIU_MIN = 0; 314 315 #undef __FUNCT__ 316 #define __FUNCT__ "PetscMax_Local" 317 PETSC_EXTERN void PetscMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 318 { 319 PetscInt i,count = *cnt; 320 321 PetscFunctionBegin; 322 if (*datatype == MPIU_REAL) { 323 PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out; 324 for (i=0; i<count; i++) xout[i] = PetscMax(xout[i],xin[i]); 325 } 326 #if defined(PETSC_HAVE_COMPLEX) 327 else if (*datatype == MPIU_COMPLEX) { 328 PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out; 329 for (i=0; i<count; i++) { 330 xout[i] = PetscRealPartComplex(xout[i])<PetscRealPartComplex(xin[i]) ? xin[i] : xout[i]; 331 } 332 } 333 #endif 334 else { 335 (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types"); 336 MPI_Abort(MPI_COMM_WORLD,1); 337 } 338 PetscFunctionReturnVoid(); 339 } 340 341 #undef __FUNCT__ 342 #define __FUNCT__ "PetscMin_Local" 343 PETSC_EXTERN void PetscMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 344 { 345 PetscInt i,count = *cnt; 346 347 PetscFunctionBegin; 348 if (*datatype == MPIU_REAL) { 349 PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out; 350 for (i=0; i<count; i++) xout[i] = PetscMin(xout[i],xin[i]); 351 } 352 #if defined(PETSC_HAVE_COMPLEX) 353 else if (*datatype == MPIU_COMPLEX) { 354 PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out; 355 for (i=0; i<count; i++) { 356 xout[i] = PetscRealPartComplex(xout[i])>PetscRealPartComplex(xin[i]) ? xin[i] : xout[i]; 357 } 358 } 359 #endif 360 else { 361 (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types"); 362 MPI_Abort(MPI_COMM_WORLD,1); 363 } 364 PetscFunctionReturnVoid(); 365 } 366 #endif 367 368 #undef __FUNCT__ 369 #define __FUNCT__ "Petsc_DelCounter" 370 /* 371 Private routine to delete internal tag/name counter storage when a communicator is freed. 372 373 This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this data as an attribute is freed. 374 375 Note: this is declared extern "C" because it is passed to MPI_Keyval_create() 376 377 */ 378 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelCounter(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state) 379 { 380 PetscErrorCode ierr; 381 382 PetscFunctionBegin; 383 ierr = PetscInfo1(0,"Deleting counter data in an MPI_Comm %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 384 ierr = PetscFree(count_val);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 385 PetscFunctionReturn(MPI_SUCCESS); 386 } 387 388 #undef __FUNCT__ 389 #define __FUNCT__ "Petsc_DelComm" 390 /* 391 This does not actually free anything, it simply marks when a reference count to an internal or external MPI_Comm reaches zero and the 392 the external MPI_Comm drops its reference to the internal or external MPI_Comm 393 394 This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator. 395 396 Note: this is declared extern "C" because it is passed to MPI_Keyval_create() 397 398 */ 399 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelComm(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state) 400 { 401 PetscErrorCode ierr; 402 PetscMPIInt flg; 403 union {MPI_Comm comm; void *ptr;} icomm,ocomm; 404 405 PetscFunctionBegin; 406 ierr = MPI_Attr_get(comm,Petsc_InnerComm_keyval,&icomm,&flg);CHKERRQ(ierr); 407 if (flg) { 408 ierr = MPI_Attr_get(icomm.comm,Petsc_OuterComm_keyval,&ocomm,&flg);CHKERRQ(ierr); 409 if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected reference to outer comm"); 410 ierr = MPI_Attr_delete(icomm.comm,Petsc_OuterComm_keyval);CHKERRQ(ierr); 411 ierr = PetscInfo1(0,"User MPI_Comm m %ld is being freed, removing reference from inner PETSc comm to this outer comm\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 412 } else { 413 ierr = PetscInfo1(0,"Removing reference to PETSc communicator imbedded in a user MPI_Comm m %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 414 } 415 PetscFunctionReturn(MPI_SUCCESS); 416 } 417 418 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32) 419 #if !defined(PETSC_WORDS_BIGENDIAN) 420 PETSC_EXTERN PetscMPIInt PetscDataRep_extent_fn(MPI_Datatype,MPI_Aint*,void*); 421 PETSC_EXTERN PetscMPIInt PetscDataRep_read_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*); 422 PETSC_EXTERN PetscMPIInt PetscDataRep_write_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*); 423 #endif 424 #endif 425 426 int PetscGlobalArgc = 0; 427 char **PetscGlobalArgs = 0; 428 429 #undef __FUNCT__ 430 #define __FUNCT__ "PetscGetArgs" 431 /*@C 432 PetscGetArgs - Allows you to access the raw command line arguments anywhere 433 after PetscInitialize() is called but before PetscFinalize(). 434 435 Not Collective 436 437 Output Parameters: 438 + argc - count of number of command line arguments 439 - args - the command line arguments 440 441 Level: intermediate 442 443 Notes: 444 This is usually used to pass the command line arguments into other libraries 445 that are called internally deep in PETSc or the application. 446 447 The first argument contains the program name as is normal for C arguments. 448 449 Concepts: command line arguments 450 451 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArguments() 452 453 @*/ 454 PetscErrorCode PetscGetArgs(int *argc,char ***args) 455 { 456 PetscFunctionBegin; 457 if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()"); 458 *argc = PetscGlobalArgc; 459 *args = PetscGlobalArgs; 460 PetscFunctionReturn(0); 461 } 462 463 #undef __FUNCT__ 464 #define __FUNCT__ "PetscGetArguments" 465 /*@C 466 PetscGetArguments - Allows you to access the command line arguments anywhere 467 after PetscInitialize() is called but before PetscFinalize(). 468 469 Not Collective 470 471 Output Parameters: 472 . args - the command line arguments 473 474 Level: intermediate 475 476 Notes: 477 This does NOT start with the program name and IS null terminated (final arg is void) 478 479 Concepts: command line arguments 480 481 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscFreeArguments() 482 483 @*/ 484 PetscErrorCode PetscGetArguments(char ***args) 485 { 486 PetscInt i,argc = PetscGlobalArgc; 487 PetscErrorCode ierr; 488 489 PetscFunctionBegin; 490 if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()"); 491 if (!argc) {*args = 0; PetscFunctionReturn(0);} 492 ierr = PetscMalloc(argc*sizeof(char*),args);CHKERRQ(ierr); 493 for (i=0; i<argc-1; i++) { 494 ierr = PetscStrallocpy(PetscGlobalArgs[i+1],&(*args)[i]);CHKERRQ(ierr); 495 } 496 (*args)[argc-1] = 0; 497 PetscFunctionReturn(0); 498 } 499 500 #undef __FUNCT__ 501 #define __FUNCT__ "PetscFreeArguments" 502 /*@C 503 PetscFreeArguments - Frees the memory obtained with PetscGetArguments() 504 505 Not Collective 506 507 Output Parameters: 508 . args - the command line arguments 509 510 Level: intermediate 511 512 Concepts: command line arguments 513 514 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscGetArguments() 515 516 @*/ 517 PetscErrorCode PetscFreeArguments(char **args) 518 { 519 PetscInt i = 0; 520 PetscErrorCode ierr; 521 522 PetscFunctionBegin; 523 if (!args) PetscFunctionReturn(0); 524 while (args[i]) { 525 ierr = PetscFree(args[i]);CHKERRQ(ierr); 526 i++; 527 } 528 ierr = PetscFree(args);CHKERRQ(ierr); 529 PetscFunctionReturn(0); 530 } 531 532 #undef __FUNCT__ 533 #define __FUNCT__ "PetscInitialize" 534 /*@C 535 PetscInitialize - Initializes the PETSc database and MPI. 536 PetscInitialize() calls MPI_Init() if that has yet to be called, 537 so this routine should always be called near the beginning of 538 your program -- usually the very first line! 539 540 Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set 541 542 Input Parameters: 543 + argc - count of number of command line arguments 544 . args - the command line arguments 545 . file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL to not check for 546 code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files 547 - help - [optional] Help message to print, use NULL for no message 548 549 If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that 550 communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a 551 four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not, 552 then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even 553 if different subcommunicators of the job are doing different things with PETSc. 554 555 Options Database Keys: 556 + -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger 557 . -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected 558 . -on_error_emacs <machinename> causes emacsclient to jump to error file 559 . -on_error_abort calls abort() when error detected (no traceback) 560 . -on_error_mpiabort calls MPI_abort() when error detected 561 . -error_output_stderr prints error messages to stderr instead of the default stdout 562 . -error_output_none does not print the error messages (but handles errors in the same way as if this was not called) 563 . -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger 564 . -debugger_pause [sleeptime] (in seconds) - Pauses debugger 565 . -stop_for_debugger - Print message on how to attach debugger manually to 566 process and wait (-debugger_pause) seconds for attachment 567 . -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries) 568 . -malloc no - Indicates not to use error-checking malloc 569 . -malloc_debug - check for memory corruption at EVERY malloc or free 570 . -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds 571 . -fp_trap - Stops on floating point exceptions (Note that on the 572 IBM RS6000 this slows code by at least a factor of 10.) 573 . -no_signal_handler - Indicates not to trap error signals 574 . -shared_tmp - indicates /tmp directory is shared by all processors 575 . -not_shared_tmp - each processor has own /tmp 576 . -tmp - alternative name of /tmp directory 577 . -get_total_flops - returns total flops done by all processors 578 . -memory_info - Print memory usage at end of run 579 - -server <port> - start PETSc webserver (default port is 8080) 580 581 Options Database Keys for Profiling: 582 See the <a href="../../docs/manual.pdf#nameddest=ch_profiling">profiling chapter of the users manual</a> for details. 583 + -info <optional filename> - Prints verbose information to the screen 584 . -info_exclude <null,vec,mat,pc,ksp,snes,ts> - Excludes some of the verbose messages 585 . -log_sync - Log the synchronization in scatters, inner products and norms 586 . -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program 587 hangs without running in the debugger). See PetscLogTraceBegin(). 588 . -log_summary [filename] - Prints summary of flop and timing information to screen. If the filename is specified the 589 summary is written to the file. See PetscLogView(). 590 . -log_summary_python [filename] - Prints data on of flop and timing usage to a file or screen. See PetscLogPrintSViewPython(). 591 . -log_all [filename] - Logs extensive profiling information See PetscLogDump(). 592 . -log [filename] - Logs basic profiline information See PetscLogDump(). 593 - -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution) 594 595 Only one of -log_trace, -log_summary, -log_all, -log, or -log_mpe may be used at a time 596 597 Environmental Variables: 598 + PETSC_TMP - alternative tmp directory 599 . PETSC_SHARED_TMP - tmp is shared by all processes 600 . PETSC_NOT_SHARED_TMP - each process has its own private tmp 601 . PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer 602 - PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to 603 604 605 Level: beginner 606 607 Notes: 608 If for some reason you must call MPI_Init() separately, call 609 it before PetscInitialize(). 610 611 Fortran Version: 612 In Fortran this routine has the format 613 $ call PetscInitialize(file,ierr) 614 615 + ierr - error return code 616 - file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL_CHARACTER to not check for 617 code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files 618 619 Important Fortran Note: 620 In Fortran, you MUST use NULL_CHARACTER to indicate a 621 null character string; you CANNOT just use NULL as 622 in the C version. See the <a href="../../docs/manual.pdf">users manual</a> for details. 623 624 If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after 625 calling PetscInitialize(). 626 627 Concepts: initializing PETSc 628 629 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments() 630 631 @*/ 632 PetscErrorCode PetscInitialize(int *argc,char ***args,const char file[],const char help[]) 633 { 634 PetscErrorCode ierr; 635 PetscMPIInt flag, size; 636 PetscInt nodesize; 637 PetscBool flg; 638 char hostname[256]; 639 640 PetscFunctionBegin; 641 if (PetscInitializeCalled) PetscFunctionReturn(0); 642 643 /* these must be initialized in a routine, not as a constant declaration*/ 644 PETSC_STDOUT = stdout; 645 PETSC_STDERR = stderr; 646 647 ierr = PetscOptionsCreate();CHKERRQ(ierr); 648 649 /* 650 We initialize the program name here (before MPI_Init()) because MPICH has a bug in 651 it that it sets args[0] on all processors to be args[0] on the first processor. 652 */ 653 if (argc && *argc) { 654 ierr = PetscSetProgramName(**args);CHKERRQ(ierr); 655 } else { 656 ierr = PetscSetProgramName("Unknown Name");CHKERRQ(ierr); 657 } 658 659 ierr = MPI_Initialized(&flag);CHKERRQ(ierr); 660 if (!flag) { 661 if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first"); 662 #if defined(PETSC_HAVE_MPI_INIT_THREAD) 663 { 664 PetscMPIInt provided; 665 ierr = MPI_Init_thread(argc,args,MPI_THREAD_FUNNELED,&provided);CHKERRQ(ierr); 666 } 667 #else 668 ierr = MPI_Init(argc,args);CHKERRQ(ierr); 669 #endif 670 PetscBeganMPI = PETSC_TRUE; 671 } 672 if (argc && args) { 673 PetscGlobalArgc = *argc; 674 PetscGlobalArgs = *args; 675 } 676 PetscFinalizeCalled = PETSC_FALSE; 677 678 if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD; 679 ierr = MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);CHKERRQ(ierr); 680 681 /* Done after init due to a bug in MPICH-GM? */ 682 ierr = PetscErrorPrintfInitialize();CHKERRQ(ierr); 683 684 ierr = MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);CHKERRQ(ierr); 685 ierr = MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);CHKERRQ(ierr); 686 687 MPIU_BOOL = MPI_INT; 688 MPIU_ENUM = MPI_INT; 689 690 /* 691 Initialized the global complex variable; this is because with 692 shared libraries the constructors for global variables 693 are not called; at least on IRIX. 694 */ 695 #if defined(PETSC_HAVE_COMPLEX) 696 { 697 #if defined(PETSC_CLANGUAGE_CXX) 698 PetscComplex ic(0.0,1.0); 699 PETSC_i = ic; 700 #elif defined(PETSC_CLANGUAGE_C) 701 PETSC_i = _Complex_I; 702 #endif 703 } 704 705 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 706 ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr); 707 ierr = MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr); 708 ierr = MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX);CHKERRQ(ierr); 709 ierr = MPI_Type_commit(&MPIU_C_COMPLEX);CHKERRQ(ierr); 710 #endif 711 #endif /* PETSC_HAVE_COMPLEX */ 712 713 /* 714 Create the PETSc MPI reduction operator that sums of the first 715 half of the entries and maxes the second half. 716 */ 717 ierr = MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);CHKERRQ(ierr); 718 719 #if defined(PETSC_USE_REAL___FLOAT128) 720 ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);CHKERRQ(ierr); 721 ierr = MPI_Type_commit(&MPIU___FLOAT128);CHKERRQ(ierr); 722 #if defined(PETSC_HAVE_COMPLEX) 723 ierr = MPI_Type_contiguous(4,MPI_DOUBLE,&MPIU___COMPLEX128);CHKERRQ(ierr); 724 ierr = MPI_Type_commit(&MPIU___COMPLEX128);CHKERRQ(ierr); 725 #endif 726 ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);CHKERRQ(ierr); 727 ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);CHKERRQ(ierr); 728 #endif 729 730 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 731 ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);CHKERRQ(ierr); 732 #endif 733 734 ierr = MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);CHKERRQ(ierr); 735 ierr = MPI_Type_commit(&MPIU_2SCALAR);CHKERRQ(ierr); 736 ierr = MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op);CHKERRQ(ierr); 737 ierr = MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op);CHKERRQ(ierr); 738 739 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 740 ierr = MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);CHKERRQ(ierr); 741 ierr = MPI_Type_commit(&MPIU_2INT);CHKERRQ(ierr); 742 #endif 743 744 /* 745 Attributes to be set on PETSc communicators 746 */ 747 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0);CHKERRQ(ierr); 748 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_InnerComm_keyval,(void*)0);CHKERRQ(ierr); 749 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_OuterComm_keyval,(void*)0);CHKERRQ(ierr); 750 751 /* 752 Build the options database 753 */ 754 ierr = PetscOptionsInsert(argc,args,file);CHKERRQ(ierr); 755 756 757 /* 758 Print main application help message 759 */ 760 ierr = PetscOptionsHasName(NULL,"-help",&flg);CHKERRQ(ierr); 761 if (help && flg) { 762 ierr = PetscPrintf(PETSC_COMM_WORLD,help);CHKERRQ(ierr); 763 } 764 ierr = PetscOptionsCheckInitial_Private();CHKERRQ(ierr); 765 766 /* SHOULD PUT IN GUARDS: Make sure logging is initialized, even if we do not print it out */ 767 #if defined(PETSC_USE_LOG) 768 ierr = PetscLogBegin_Private();CHKERRQ(ierr); 769 #endif 770 771 /* 772 Load the dynamic libraries (on machines that support them), this registers all 773 the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes) 774 */ 775 ierr = PetscInitialize_DynamicLibraries();CHKERRQ(ierr); 776 777 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 778 ierr = PetscInfo1(0,"PETSc successfully started: number of processors = %d\n",size);CHKERRQ(ierr); 779 ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr); 780 ierr = PetscInfo1(0,"Running on machine: %s\n",hostname);CHKERRQ(ierr); 781 782 ierr = PetscOptionsCheckInitial_Components();CHKERRQ(ierr); 783 /* Check the options database for options related to the options database itself */ 784 ierr = PetscOptionsSetFromOptions();CHKERRQ(ierr); 785 786 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32) 787 /* 788 Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI 789 790 Currently not used because it is not supported by MPICH. 791 */ 792 #if !defined(PETSC_WORDS_BIGENDIAN) 793 ierr = MPI_Register_datarep((char*)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,NULL);CHKERRQ(ierr); 794 #endif 795 #endif 796 797 ierr = PetscOptionsGetInt(NULL,"-hmpi_spawn_size",&nodesize,&flg);CHKERRQ(ierr); 798 if (flg) { 799 #if defined(PETSC_HAVE_MPI_COMM_SPAWN) 800 ierr = PetscHMPISpawn((PetscMPIInt) nodesize);CHKERRQ(ierr); /* worker nodes never return from here; they go directly to PetscEnd() */ 801 #else 802 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead"); 803 #endif 804 } else { 805 ierr = PetscOptionsGetInt(NULL,"-hmpi_merge_size",&nodesize,&flg);CHKERRQ(ierr); 806 if (flg) { 807 ierr = PetscHMPIMerge((PetscMPIInt) nodesize,NULL,NULL);CHKERRQ(ierr); 808 if (PetscHMPIWorker) { /* if worker then never enter user code */ 809 PetscInitializeCalled = PETSC_TRUE; 810 PetscEnd(); 811 } 812 } 813 } 814 815 #if defined(PETSC_HAVE_CUDA) 816 { 817 PetscMPIInt p; 818 for (p = 0; p < PetscGlobalSize; ++p) { 819 if (p == PetscGlobalRank) cublasInit(); 820 ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); 821 } 822 } 823 #endif 824 825 ierr = PetscOptionsHasName(NULL,"-python",&flg);CHKERRQ(ierr); 826 if (flg) { 827 PetscInitializeCalled = PETSC_TRUE; 828 ierr = PetscPythonInitialize(NULL,NULL);CHKERRQ(ierr); 829 } 830 831 ierr = PetscThreadCommInitializePackage();CHKERRQ(ierr); 832 833 /* 834 Setup building of stack frames for all function calls 835 */ 836 #if defined(PETSC_USE_DEBUG) 837 PetscThreadLocalRegister((PetscThreadKey*)&petscstack); /* Creates petscstack_key if needed */ 838 ierr = PetscStackCreate();CHKERRQ(ierr); 839 #endif 840 841 #if defined(PETSC_SERIALIZE_FUNCTIONS) 842 ierr = PetscFPTCreate(10000);CHKERRQ(ierr); 843 #endif 844 845 /* 846 Once we are completedly initialized then we can set this variables 847 */ 848 PetscInitializeCalled = PETSC_TRUE; 849 PetscFunctionReturn(0); 850 } 851 852 extern PetscObject *PetscObjects; 853 extern PetscInt PetscObjectsCounts, PetscObjectsMaxCounts; 854 855 #undef __FUNCT__ 856 #define __FUNCT__ "PetscFinalize" 857 /*@C 858 PetscFinalize - Checks for options to be called at the conclusion 859 of the program. MPI_Finalize() is called only if the user had not 860 called MPI_Init() before calling PetscInitialize(). 861 862 Collective on PETSC_COMM_WORLD 863 864 Options Database Keys: 865 + -options_table - Calls PetscOptionsView() 866 . -options_left - Prints unused options that remain in the database 867 . -objects_dump [all] - Prints list of objects allocated by the user that have not been freed, the option all cause all outstanding objects to be listed 868 . -mpidump - Calls PetscMPIDump() 869 . -malloc_dump - Calls PetscMallocDump() 870 . -malloc_info - Prints total memory usage 871 - -malloc_log - Prints summary of memory usage 872 873 Level: beginner 874 875 Note: 876 See PetscInitialize() for more general runtime options. 877 878 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscEnd() 879 @*/ 880 PetscErrorCode PetscFinalize(void) 881 { 882 PetscErrorCode ierr; 883 PetscMPIInt rank; 884 PetscInt nopt; 885 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE; 886 #if defined(PETSC_HAVE_AMS) 887 PetscBool flg = PETSC_FALSE; 888 #endif 889 #if defined(PETSC_USE_LOG) 890 char mname[PETSC_MAX_PATH_LEN]; 891 #endif 892 893 PetscFunctionBegin; 894 if (!PetscInitializeCalled) { 895 printf("PetscInitialize() must be called before PetscFinalize()\n"); 896 PetscFunctionReturn(PETSC_ERR_ARG_WRONGSTATE); 897 } 898 ierr = PetscInfo(NULL,"PetscFinalize() called\n");CHKERRQ(ierr); 899 900 #if defined(PETSC_SERIALIZE_FUNCTIONS) 901 ierr = PetscFPTDestroy();CHKERRQ(ierr); 902 #endif 903 904 905 #if defined(PETSC_HAVE_AMS) 906 ierr = PetscOptionsGetBool(NULL,"-options_gui",&flg,NULL);CHKERRQ(ierr); 907 if (flg) { 908 ierr = PetscOptionsAMSDestroy();CHKERRQ(ierr); 909 } 910 #endif 911 912 #if defined(PETSC_HAVE_SERVER) 913 flg1 = PETSC_FALSE; 914 ierr = PetscOptionsGetBool(NULL,"-server",&flg1,NULL);CHKERRQ(ierr); 915 if (flg1) { 916 /* this is a crude hack, but better than nothing */ 917 ierr = PetscPOpen(PETSC_COMM_WORLD,NULL,"pkill -9 petscwebserver","r",NULL);CHKERRQ(ierr); 918 } 919 #endif 920 921 ierr = PetscHMPIFinalize();CHKERRQ(ierr); 922 923 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 924 ierr = PetscOptionsGetBool(NULL,"-malloc_info",&flg2,NULL);CHKERRQ(ierr); 925 if (!flg2) { 926 flg2 = PETSC_FALSE; 927 ierr = PetscOptionsGetBool(NULL,"-memory_info",&flg2,NULL);CHKERRQ(ierr); 928 } 929 if (flg2) { 930 ierr = PetscMemoryShowUsage(PETSC_VIEWER_STDOUT_WORLD,"Summary of Memory Usage in PETSc\n");CHKERRQ(ierr); 931 } 932 933 #if defined(PETSC_USE_LOG) 934 flg1 = PETSC_FALSE; 935 ierr = PetscOptionsGetBool(NULL,"-get_total_flops",&flg1,NULL);CHKERRQ(ierr); 936 if (flg1) { 937 PetscLogDouble flops = 0; 938 ierr = MPI_Reduce(&petsc_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);CHKERRQ(ierr); 939 ierr = PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %g\n",flops);CHKERRQ(ierr); 940 } 941 #endif 942 943 944 #if defined(PETSC_USE_LOG) 945 #if defined(PETSC_HAVE_MPE) 946 mname[0] = 0; 947 948 ierr = PetscOptionsGetString(NULL,"-log_mpe",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 949 if (flg1) { 950 if (mname[0]) {ierr = PetscLogMPEDump(mname);CHKERRQ(ierr);} 951 else {ierr = PetscLogMPEDump(0);CHKERRQ(ierr);} 952 } 953 #endif 954 mname[0] = 0; 955 956 ierr = PetscOptionsGetString(NULL,"-log_summary",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 957 if (flg1) { 958 PetscViewer viewer; 959 if (mname[0]) { 960 ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr); 961 ierr = PetscLogView(viewer);CHKERRQ(ierr); 962 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 963 } else { 964 viewer = PETSC_VIEWER_STDOUT_WORLD; 965 ierr = PetscLogView(viewer);CHKERRQ(ierr); 966 } 967 } 968 969 mname[0] = 0; 970 971 ierr = PetscOptionsGetString(NULL,"-log_summary_python",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 972 if (flg1) { 973 PetscViewer viewer; 974 if (mname[0]) { 975 ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr); 976 ierr = PetscLogViewPython(viewer);CHKERRQ(ierr); 977 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 978 } else { 979 viewer = PETSC_VIEWER_STDOUT_WORLD; 980 ierr = PetscLogViewPython(viewer);CHKERRQ(ierr); 981 } 982 } 983 984 ierr = PetscOptionsGetString(NULL,"-log_detailed",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 985 if (flg1) { 986 if (mname[0]) {ierr = PetscLogPrintDetailed(PETSC_COMM_WORLD,mname);CHKERRQ(ierr);} 987 else {ierr = PetscLogPrintDetailed(PETSC_COMM_WORLD,0);CHKERRQ(ierr);} 988 } 989 990 mname[0] = 0; 991 992 ierr = PetscOptionsGetString(NULL,"-log_all",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 993 ierr = PetscOptionsGetString(NULL,"-log",mname,PETSC_MAX_PATH_LEN,&flg2);CHKERRQ(ierr); 994 if (flg1 || flg2) { 995 if (mname[0]) PetscLogDump(mname); 996 else PetscLogDump(0); 997 } 998 #endif 999 1000 /* 1001 Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_(). 1002 */ 1003 ierr = PetscObjectRegisterDestroyAll();CHKERRQ(ierr); 1004 1005 ierr = PetscStackDestroy();CHKERRQ(ierr); 1006 1007 flg1 = PETSC_FALSE; 1008 ierr = PetscOptionsGetBool(NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 1009 if (!flg1) { ierr = PetscPopSignalHandler();CHKERRQ(ierr);} 1010 flg1 = PETSC_FALSE; 1011 ierr = PetscOptionsGetBool(NULL,"-mpidump",&flg1,NULL);CHKERRQ(ierr); 1012 if (flg1) { 1013 ierr = PetscMPIDump(stdout);CHKERRQ(ierr); 1014 } 1015 flg1 = PETSC_FALSE; 1016 flg2 = PETSC_FALSE; 1017 /* preemptive call to avoid listing this option in options table as unused */ 1018 ierr = PetscOptionsHasName(NULL,"-malloc_dump",&flg1);CHKERRQ(ierr); 1019 ierr = PetscOptionsHasName(NULL,"-objects_dump",&flg1);CHKERRQ(ierr); 1020 ierr = PetscOptionsGetBool(NULL,"-options_table",&flg2,NULL);CHKERRQ(ierr); 1021 1022 if (flg2) { 1023 PetscViewer viewer; 1024 ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr); 1025 ierr = PetscOptionsView(viewer);CHKERRQ(ierr); 1026 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 1027 } 1028 1029 /* to prevent PETSc -options_left from warning */ 1030 ierr = PetscOptionsHasName(NULL,"-nox",&flg1);CHKERRQ(ierr); 1031 ierr = PetscOptionsHasName(NULL,"-nox_warning",&flg1);CHKERRQ(ierr); 1032 1033 if (!PetscHMPIWorker) { /* worker processes skip this because they do not usually process options */ 1034 flg3 = PETSC_FALSE; /* default value is required */ 1035 ierr = PetscOptionsGetBool(NULL,"-options_left",&flg3,&flg1);CHKERRQ(ierr); 1036 ierr = PetscOptionsAllUsed(&nopt);CHKERRQ(ierr); 1037 if (flg3) { 1038 if (!flg2) { /* have not yet printed the options */ 1039 PetscViewer viewer; 1040 ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr); 1041 ierr = PetscOptionsView(viewer);CHKERRQ(ierr); 1042 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 1043 } 1044 if (!nopt) { 1045 ierr = PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");CHKERRQ(ierr); 1046 } else if (nopt == 1) { 1047 ierr = PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");CHKERRQ(ierr); 1048 } else { 1049 ierr = PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);CHKERRQ(ierr); 1050 } 1051 } 1052 #if defined(PETSC_USE_DEBUG) 1053 if (nopt && !flg3 && !flg1) { 1054 ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!\n");CHKERRQ(ierr); 1055 ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!\n");CHKERRQ(ierr); 1056 ierr = PetscOptionsLeft();CHKERRQ(ierr); 1057 } else if (nopt && flg3) { 1058 #else 1059 if (nopt && flg3) { 1060 #endif 1061 ierr = PetscOptionsLeft();CHKERRQ(ierr); 1062 } 1063 } 1064 1065 { 1066 PetscThreadComm tcomm_world; 1067 ierr = PetscGetThreadCommWorld(&tcomm_world);CHKERRQ(ierr); 1068 /* Free global thread communicator */ 1069 ierr = PetscThreadCommDestroy(&tcomm_world);CHKERRQ(ierr); 1070 } 1071 1072 /* 1073 List all objects the user may have forgot to free 1074 */ 1075 ierr = PetscOptionsHasName(NULL,"-objects_dump",&flg1);CHKERRQ(ierr); 1076 if (flg1) { 1077 MPI_Comm local_comm; 1078 char string[64]; 1079 1080 ierr = PetscOptionsGetString(NULL,"-objects_dump",string,64,NULL);CHKERRQ(ierr); 1081 ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRQ(ierr); 1082 ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr); 1083 ierr = PetscObjectsDump(stdout,(string[0] == 'a') ? PETSC_TRUE : PETSC_FALSE);CHKERRQ(ierr); 1084 ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr); 1085 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 1086 } 1087 PetscObjectsCounts = 0; 1088 PetscObjectsMaxCounts = 0; 1089 1090 ierr = PetscFree(PetscObjects);CHKERRQ(ierr); 1091 1092 #if defined(PETSC_USE_LOG) 1093 ierr = PetscLogDestroy();CHKERRQ(ierr); 1094 #endif 1095 1096 /* 1097 Destroy any packages that registered a finalize 1098 */ 1099 ierr = PetscRegisterFinalizeAll();CHKERRQ(ierr); 1100 1101 /* 1102 Destroy all the function registration lists created 1103 */ 1104 ierr = PetscFinalize_DynamicLibraries();CHKERRQ(ierr); 1105 1106 /* 1107 Print PetscFunctionLists that have not been properly freed 1108 1109 ierr = PetscFunctionListPrintAll();CHKERRQ(ierr); 1110 */ 1111 1112 if (petsc_history) { 1113 ierr = PetscCloseHistoryFile(&petsc_history);CHKERRQ(ierr); 1114 petsc_history = 0; 1115 } 1116 1117 ierr = PetscInfoAllow(PETSC_FALSE,NULL);CHKERRQ(ierr); 1118 1119 { 1120 char fname[PETSC_MAX_PATH_LEN]; 1121 FILE *fd; 1122 int err; 1123 1124 fname[0] = 0; 1125 1126 ierr = PetscOptionsGetString(NULL,"-malloc_dump",fname,250,&flg1);CHKERRQ(ierr); 1127 flg2 = PETSC_FALSE; 1128 ierr = PetscOptionsGetBool(NULL,"-malloc_test",&flg2,NULL);CHKERRQ(ierr); 1129 #if defined(PETSC_USE_DEBUG) 1130 if (PETSC_RUNNING_ON_VALGRIND) flg2 = PETSC_FALSE; 1131 #else 1132 flg2 = PETSC_FALSE; /* Skip reporting for optimized builds regardless of -malloc_test */ 1133 #endif 1134 if (flg1 && fname[0]) { 1135 char sname[PETSC_MAX_PATH_LEN]; 1136 1137 sprintf(sname,"%s_%d",fname,rank); 1138 fd = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname); 1139 ierr = PetscMallocDump(fd);CHKERRQ(ierr); 1140 err = fclose(fd); 1141 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 1142 } else if (flg1 || flg2) { 1143 MPI_Comm local_comm; 1144 1145 ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRQ(ierr); 1146 ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr); 1147 ierr = PetscMallocDump(stdout);CHKERRQ(ierr); 1148 ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr); 1149 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 1150 } 1151 } 1152 1153 { 1154 char fname[PETSC_MAX_PATH_LEN]; 1155 FILE *fd = NULL; 1156 1157 fname[0] = 0; 1158 1159 ierr = PetscOptionsGetString(NULL,"-malloc_log",fname,250,&flg1);CHKERRQ(ierr); 1160 ierr = PetscOptionsHasName(NULL,"-malloc_log_threshold",&flg2);CHKERRQ(ierr); 1161 if (flg1 && fname[0]) { 1162 int err; 1163 1164 if (!rank) { 1165 fd = fopen(fname,"w"); 1166 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",fname); 1167 } 1168 ierr = PetscMallocDumpLog(fd);CHKERRQ(ierr); 1169 if (fd) { 1170 err = fclose(fd); 1171 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 1172 } 1173 } else if (flg1 || flg2) { 1174 ierr = PetscMallocDumpLog(stdout);CHKERRQ(ierr); 1175 } 1176 } 1177 /* Can be destroyed only after all the options are used */ 1178 ierr = PetscOptionsDestroy();CHKERRQ(ierr); 1179 1180 PetscGlobalArgc = 0; 1181 PetscGlobalArgs = 0; 1182 1183 #if defined(PETSC_USE_REAL___FLOAT128) 1184 ierr = MPI_Type_free(&MPIU___FLOAT128);CHKERRQ(ierr); 1185 #if defined(PETSC_HAVE_COMPLEX) 1186 ierr = MPI_Type_free(&MPIU___COMPLEX128);CHKERRQ(ierr); 1187 #endif 1188 ierr = MPI_Op_free(&MPIU_MAX);CHKERRQ(ierr); 1189 ierr = MPI_Op_free(&MPIU_MIN);CHKERRQ(ierr); 1190 #endif 1191 1192 #if defined(PETSC_HAVE_COMPLEX) 1193 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 1194 ierr = MPI_Type_free(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr); 1195 ierr = MPI_Type_free(&MPIU_C_COMPLEX);CHKERRQ(ierr); 1196 #endif 1197 #endif 1198 1199 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 1200 ierr = MPI_Op_free(&MPIU_SUM);CHKERRQ(ierr); 1201 #endif 1202 1203 ierr = MPI_Type_free(&MPIU_2SCALAR);CHKERRQ(ierr); 1204 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 1205 ierr = MPI_Type_free(&MPIU_2INT);CHKERRQ(ierr); 1206 #endif 1207 ierr = MPI_Op_free(&PetscMaxSum_Op);CHKERRQ(ierr); 1208 ierr = MPI_Op_free(&PetscADMax_Op);CHKERRQ(ierr); 1209 ierr = MPI_Op_free(&PetscADMin_Op);CHKERRQ(ierr); 1210 1211 /* 1212 Destroy any known inner MPI_Comm's and attributes pointing to them 1213 Note this will not destroy any new communicators the user has created. 1214 1215 If all PETSc objects were not destroyed those left over objects will have hanging references to 1216 the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again 1217 */ 1218 { 1219 PetscCommCounter *counter; 1220 PetscMPIInt flg; 1221 MPI_Comm icomm; 1222 union {MPI_Comm comm; void *ptr;} ucomm; 1223 ierr = MPI_Attr_get(PETSC_COMM_SELF,Petsc_InnerComm_keyval,&ucomm,&flg);CHKERRQ(ierr); 1224 if (flg) { 1225 icomm = ucomm.comm; 1226 ierr = MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRQ(ierr); 1227 if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory"); 1228 1229 ierr = MPI_Attr_delete(PETSC_COMM_SELF,Petsc_InnerComm_keyval);CHKERRQ(ierr); 1230 ierr = MPI_Attr_delete(icomm,Petsc_Counter_keyval);CHKERRQ(ierr); 1231 ierr = MPI_Comm_free(&icomm);CHKERRQ(ierr); 1232 } 1233 ierr = MPI_Attr_get(PETSC_COMM_WORLD,Petsc_InnerComm_keyval,&ucomm,&flg);CHKERRQ(ierr); 1234 if (flg) { 1235 icomm = ucomm.comm; 1236 ierr = MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRQ(ierr); 1237 if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory"); 1238 1239 ierr = MPI_Attr_delete(PETSC_COMM_WORLD,Petsc_InnerComm_keyval);CHKERRQ(ierr); 1240 ierr = MPI_Attr_delete(icomm,Petsc_Counter_keyval);CHKERRQ(ierr); 1241 ierr = MPI_Comm_free(&icomm);CHKERRQ(ierr); 1242 } 1243 } 1244 1245 ierr = MPI_Keyval_free(&Petsc_Counter_keyval);CHKERRQ(ierr); 1246 ierr = MPI_Keyval_free(&Petsc_InnerComm_keyval);CHKERRQ(ierr); 1247 ierr = MPI_Keyval_free(&Petsc_OuterComm_keyval);CHKERRQ(ierr); 1248 1249 #if defined(PETSC_HAVE_CUDA) 1250 { 1251 PetscInt p; 1252 for (p = 0; p < PetscGlobalSize; ++p) { 1253 if (p == PetscGlobalRank) cublasShutdown(); 1254 ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); 1255 } 1256 } 1257 #endif 1258 1259 if (PetscBeganMPI) { 1260 #if defined(PETSC_HAVE_MPI_FINALIZED) 1261 PetscMPIInt flag; 1262 ierr = MPI_Finalized(&flag);CHKERRQ(ierr); 1263 if (flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()"); 1264 #endif 1265 ierr = MPI_Finalize();CHKERRQ(ierr); 1266 } 1267 /* 1268 1269 Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because 1270 the communicator has some outstanding requests on it. Specifically if the 1271 flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See 1272 src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate() 1273 is never freed as it should be. Thus one may obtain messages of the form 1274 [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the 1275 memory was not freed. 1276 1277 */ 1278 ierr = PetscMallocClear();CHKERRQ(ierr); 1279 1280 PetscInitializeCalled = PETSC_FALSE; 1281 PetscFinalizeCalled = PETSC_TRUE; 1282 PetscFunctionReturn(ierr); 1283 } 1284 1285 #if defined(PETSC_MISSING_LAPACK_lsame_) 1286 PETSC_EXTERN int lsame_(char *a,char *b) 1287 { 1288 if (*a == *b) return 1; 1289 if (*a + 32 == *b) return 1; 1290 if (*a - 32 == *b) return 1; 1291 return 0; 1292 } 1293 #endif 1294 1295 #if defined(PETSC_MISSING_LAPACK_lsame) 1296 PETSC_EXTERN int lsame(char *a,char *b) 1297 { 1298 if (*a == *b) return 1; 1299 if (*a + 32 == *b) return 1; 1300 if (*a - 32 == *b) return 1; 1301 return 0; 1302 } 1303 #endif 1304