1 2 /* 3 This file defines the initialization of PETSc, including PetscInitialize() 4 */ 5 #define PETSC_DESIRE_COMPLEX 6 #include <petsc-private/petscimpl.h> /*I "petscsys.h" I*/ 7 #include <petscviewer.h> 8 9 #if defined(PETSC_HAVE_CUDA) 10 #include <cublas.h> 11 #endif 12 13 #include <petscthreadcomm.h> 14 15 #if defined(PETSC_USE_LOG) 16 extern PetscErrorCode PetscLogBegin_Private(void); 17 #endif 18 extern PetscBool PetscHMPIWorker; 19 20 21 #if defined(PETSC_SERIALIZE_FUNCTIONS) 22 PetscFPT PetscFPTData = 0; 23 #endif 24 25 /* -----------------------------------------------------------------------------------------*/ 26 27 extern FILE *petsc_history; 28 29 extern PetscErrorCode PetscInitialize_DynamicLibraries(void); 30 extern PetscErrorCode PetscFinalize_DynamicLibraries(void); 31 extern PetscErrorCode PetscFunctionListPrintAll(void); 32 extern PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm,int); 33 extern PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm,int); 34 extern PetscErrorCode PetscCloseHistoryFile(FILE**); 35 36 /* user may set this BEFORE calling PetscInitialize() */ 37 MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL; 38 39 PetscMPIInt Petsc_Counter_keyval = MPI_KEYVAL_INVALID; 40 PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID; 41 PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID; 42 43 /* 44 Declare and set all the string names of the PETSc enums 45 */ 46 const char *const PetscBools[] = {"FALSE","TRUE","PetscBool","PETSC_",0}; 47 const char *const PetscCopyModes[] = {"COPY_VALUES","OWN_POINTER","USE_POINTER","PetscCopyMode","PETSC_",0}; 48 const char *const PetscDataTypes[] = {"INT","DOUBLE","COMPLEX","LONG","SHORT","FLOAT", 49 "CHAR","LOGICAL","ENUM","BOOL","LONGDOUBLE","OBJECT","FUNCTION","PetscDataType","PETSC_",0}; 50 51 PetscBool PetscPreLoadingUsed = PETSC_FALSE; 52 PetscBool PetscPreLoadingOn = PETSC_FALSE; 53 54 /* pthread_key for PetscStack */ 55 #if defined(PETSC_HAVE_PTHREADCLASSES) && !defined(PETSC_PTHREAD_LOCAL) 56 pthread_key_t petscstack; 57 #endif 58 59 /* 60 Checks the options database for initializations related to the 61 PETSc components 62 */ 63 #undef __FUNCT__ 64 #define __FUNCT__ "PetscOptionsCheckInitial_Components" 65 PetscErrorCode PetscOptionsCheckInitial_Components(void) 66 { 67 PetscBool flg1; 68 PetscErrorCode ierr; 69 70 PetscFunctionBegin; 71 ierr = PetscOptionsHasName(NULL,"-help",&flg1);CHKERRQ(ierr); 72 if (flg1) { 73 #if defined(PETSC_USE_LOG) 74 MPI_Comm comm = PETSC_COMM_WORLD; 75 ierr = (*PetscHelpPrintf)(comm,"------Additional PETSc component options--------\n");CHKERRQ(ierr); 76 ierr = (*PetscHelpPrintf)(comm," -log_summary_exclude: <vec,mat,pc.ksp,snes>\n");CHKERRQ(ierr); 77 ierr = (*PetscHelpPrintf)(comm," -info_exclude: <null,vec,mat,pc,ksp,snes,ts>\n");CHKERRQ(ierr); 78 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 79 #endif 80 } 81 PetscFunctionReturn(0); 82 } 83 84 #undef __FUNCT__ 85 #define __FUNCT__ "PetscInitializeNoPointers" 86 /* 87 PetscInitializeNoPointers - Calls PetscInitialize() from C/C++ without the pointers to argc and args 88 89 Collective 90 91 Level: advanced 92 93 Notes: this is called only by the PETSc MATLAB and Julia interface. Even though it might start MPI it sets the flag to 94 indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to 95 be called multiple times from MATLAB and Julia without the problem of trying to initialize MPI more than once. 96 97 Turns off PETSc signal handling because that can interact with MATLAB's signal handling causing random crashes. 98 99 .seealso: PetscInitialize(), PetscInitializeFortran(), PetscInitializeNoArguments() 100 */ 101 PetscErrorCode PetscInitializeNoPointers(int argc,char **args,const char *filename,const char *help) 102 { 103 PetscErrorCode ierr; 104 int myargc = argc; 105 char **myargs = args; 106 107 PetscFunctionBegin; 108 ierr = PetscInitialize(&myargc,&myargs,filename,help);CHKERRQ(ierr); 109 ierr = PetscPopSignalHandler();CHKERRQ(ierr); 110 PetscBeganMPI = PETSC_FALSE; 111 PetscFunctionReturn(ierr); 112 } 113 114 #undef __FUNCT__ 115 #define __FUNCT__ "PetscGetPETSC_COMM_SELF" 116 /* 117 Used by MATLAB and Julia interface to get communicator 118 */ 119 PetscErrorCode PetscGetPETSC_COMM_SELF(MPI_Comm *comm) 120 { 121 PetscFunctionBegin; 122 *comm = PETSC_COMM_SELF; 123 PetscFunctionReturn(0); 124 } 125 126 #undef __FUNCT__ 127 #define __FUNCT__ "PetscInitializeNoArguments" 128 /*@C 129 PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without 130 the command line arguments. 131 132 Collective 133 134 Level: advanced 135 136 .seealso: PetscInitialize(), PetscInitializeFortran() 137 @*/ 138 PetscErrorCode PetscInitializeNoArguments(void) 139 { 140 PetscErrorCode ierr; 141 int argc = 0; 142 char **args = 0; 143 144 PetscFunctionBegin; 145 ierr = PetscInitialize(&argc,&args,NULL,NULL); 146 PetscFunctionReturn(ierr); 147 } 148 149 #undef __FUNCT__ 150 #define __FUNCT__ "PetscInitialized" 151 /*@ 152 PetscInitialized - Determine whether PETSc is initialized. 153 154 7 Level: beginner 155 156 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran() 157 @*/ 158 PetscErrorCode PetscInitialized(PetscBool *isInitialized) 159 { 160 PetscFunctionBegin; 161 PetscValidPointer(isInitialized, 1); 162 *isInitialized = PetscInitializeCalled; 163 PetscFunctionReturn(0); 164 } 165 166 #undef __FUNCT__ 167 #define __FUNCT__ "PetscFinalized" 168 /*@ 169 PetscFinalized - Determine whether PetscFinalize() has been called yet 170 171 Level: developer 172 173 .seealso: PetscInitialize(), PetscInitializeNoArguments(), PetscInitializeFortran() 174 @*/ 175 PetscErrorCode PetscFinalized(PetscBool *isFinalized) 176 { 177 PetscFunctionBegin; 178 PetscValidPointer(isFinalized, 1); 179 *isFinalized = PetscFinalizeCalled; 180 PetscFunctionReturn(0); 181 } 182 183 extern PetscErrorCode PetscOptionsCheckInitial_Private(void); 184 185 /* 186 This function is the MPI reduction operation used to compute the sum of the 187 first half of the datatype and the max of the second half. 188 */ 189 MPI_Op PetscMaxSum_Op = 0; 190 191 #undef __FUNCT__ 192 #define __FUNCT__ "PetscMaxSum_Local" 193 PETSC_EXTERN void MPIAPI PetscMaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype) 194 { 195 PetscInt *xin = (PetscInt*)in,*xout = (PetscInt*)out,i,count = *cnt; 196 197 PetscFunctionBegin; 198 if (*datatype != MPIU_2INT) { 199 (*PetscErrorPrintf)("Can only handle MPIU_2INT data types"); 200 MPI_Abort(MPI_COMM_WORLD,1); 201 } 202 203 for (i=0; i<count; i++) { 204 xout[2*i] = PetscMax(xout[2*i],xin[2*i]); 205 xout[2*i+1] += xin[2*i+1]; 206 } 207 PetscFunctionReturnVoid(); 208 } 209 210 /* 211 Returns the max of the first entry owned by this processor and the 212 sum of the second entry. 213 214 The reason nprocs[2*i] contains lengths nprocs[2*i+1] contains flag of 1 if length is nonzero 215 is so that the PetscMaxSum_Op() can set TWO values, if we passed in only nprocs[i] with lengths 216 there would be no place to store the both needed results. 217 */ 218 #undef __FUNCT__ 219 #define __FUNCT__ "PetscMaxSum" 220 PetscErrorCode PetscMaxSum(MPI_Comm comm,const PetscInt nprocs[],PetscInt *max,PetscInt *sum) 221 { 222 PetscMPIInt size,rank; 223 struct {PetscInt max,sum;} *work; 224 PetscErrorCode ierr; 225 226 PetscFunctionBegin; 227 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 228 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 229 ierr = PetscMalloc(size*sizeof(*work),&work);CHKERRQ(ierr); 230 ierr = MPI_Allreduce((void*)nprocs,work,size,MPIU_2INT,PetscMaxSum_Op,comm);CHKERRQ(ierr); 231 *max = work[rank].max; 232 *sum = work[rank].sum; 233 ierr = PetscFree(work);CHKERRQ(ierr); 234 PetscFunctionReturn(0); 235 } 236 237 /* ----------------------------------------------------------------------------*/ 238 MPI_Op PetscADMax_Op = 0; 239 240 #undef __FUNCT__ 241 #define __FUNCT__ "PetscADMax_Local" 242 PETSC_EXTERN void MPIAPI PetscADMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 243 { 244 PetscScalar *xin = (PetscScalar*)in,*xout = (PetscScalar*)out; 245 PetscInt i,count = *cnt; 246 247 PetscFunctionBegin; 248 if (*datatype != MPIU_2SCALAR) { 249 (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types"); 250 MPI_Abort(MPI_COMM_WORLD,1); 251 } 252 253 for (i=0; i<count; i++) { 254 if (PetscRealPart(xout[2*i]) < PetscRealPart(xin[2*i])) { 255 xout[2*i] = xin[2*i]; 256 xout[2*i+1] = xin[2*i+1]; 257 } 258 } 259 PetscFunctionReturnVoid(); 260 } 261 262 MPI_Op PetscADMin_Op = 0; 263 264 #undef __FUNCT__ 265 #define __FUNCT__ "PetscADMin_Local" 266 PETSC_EXTERN void MPIAPI PetscADMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 267 { 268 PetscScalar *xin = (PetscScalar*)in,*xout = (PetscScalar*)out; 269 PetscInt i,count = *cnt; 270 271 PetscFunctionBegin; 272 if (*datatype != MPIU_2SCALAR) { 273 (*PetscErrorPrintf)("Can only handle MPIU_2SCALAR data (i.e. double or complex) types"); 274 MPI_Abort(MPI_COMM_WORLD,1); 275 } 276 277 for (i=0; i<count; i++) { 278 if (PetscRealPart(xout[2*i]) > PetscRealPart(xin[2*i])) { 279 xout[2*i] = xin[2*i]; 280 xout[2*i+1] = xin[2*i+1]; 281 } 282 } 283 PetscFunctionReturnVoid(); 284 } 285 /* ---------------------------------------------------------------------------------------*/ 286 287 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 288 MPI_Op MPIU_SUM = 0; 289 290 #undef __FUNCT__ 291 #define __FUNCT__ "PetscSum_Local" 292 PETSC_EXTERN void PetscSum_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 293 { 294 PetscInt i,count = *cnt; 295 296 PetscFunctionBegin; 297 if (*datatype == MPIU_REAL) { 298 PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out; 299 for (i=0; i<count; i++) xout[i] += xin[i]; 300 } 301 #if defined(PETSC_HAVE_COMPLEX) 302 else if (*datatype == MPIU_COMPLEX) { 303 PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out; 304 for (i=0; i<count; i++) xout[i] += xin[i]; 305 } 306 #endif 307 else { 308 (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types"); 309 MPI_Abort(MPI_COMM_WORLD,1); 310 } 311 PetscFunctionReturnVoid(); 312 } 313 #endif 314 315 #if defined(PETSC_USE_REAL___FLOAT128) 316 MPI_Op MPIU_MAX = 0; 317 MPI_Op MPIU_MIN = 0; 318 319 #undef __FUNCT__ 320 #define __FUNCT__ "PetscMax_Local" 321 PETSC_EXTERN void PetscMax_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 322 { 323 PetscInt i,count = *cnt; 324 325 PetscFunctionBegin; 326 if (*datatype == MPIU_REAL) { 327 PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out; 328 for (i=0; i<count; i++) xout[i] = PetscMax(xout[i],xin[i]); 329 } 330 #if defined(PETSC_HAVE_COMPLEX) 331 else if (*datatype == MPIU_COMPLEX) { 332 PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out; 333 for (i=0; i<count; i++) { 334 xout[i] = PetscRealPartComplex(xout[i])<PetscRealPartComplex(xin[i]) ? xin[i] : xout[i]; 335 } 336 } 337 #endif 338 else { 339 (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types"); 340 MPI_Abort(MPI_COMM_WORLD,1); 341 } 342 PetscFunctionReturnVoid(); 343 } 344 345 #undef __FUNCT__ 346 #define __FUNCT__ "PetscMin_Local" 347 PETSC_EXTERN void PetscMin_Local(void *in,void *out,PetscMPIInt *cnt,MPI_Datatype *datatype) 348 { 349 PetscInt i,count = *cnt; 350 351 PetscFunctionBegin; 352 if (*datatype == MPIU_REAL) { 353 PetscReal *xin = (PetscReal*)in,*xout = (PetscReal*)out; 354 for (i=0; i<count; i++) xout[i] = PetscMin(xout[i],xin[i]); 355 } 356 #if defined(PETSC_HAVE_COMPLEX) 357 else if (*datatype == MPIU_COMPLEX) { 358 PetscComplex *xin = (PetscComplex*)in,*xout = (PetscComplex*)out; 359 for (i=0; i<count; i++) { 360 xout[i] = PetscRealPartComplex(xout[i])>PetscRealPartComplex(xin[i]) ? xin[i] : xout[i]; 361 } 362 } 363 #endif 364 else { 365 (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types"); 366 MPI_Abort(MPI_COMM_WORLD,1); 367 } 368 PetscFunctionReturnVoid(); 369 } 370 #endif 371 372 #undef __FUNCT__ 373 #define __FUNCT__ "Petsc_DelCounter" 374 /* 375 Private routine to delete internal tag/name counter storage when a communicator is freed. 376 377 This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this data as an attribute is freed. 378 379 Note: this is declared extern "C" because it is passed to MPI_Keyval_create() 380 381 */ 382 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelCounter(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state) 383 { 384 PetscErrorCode ierr; 385 386 PetscFunctionBegin; 387 ierr = PetscInfo1(0,"Deleting counter data in an MPI_Comm %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 388 ierr = PetscFree(count_val);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 389 PetscFunctionReturn(MPI_SUCCESS); 390 } 391 392 #undef __FUNCT__ 393 #define __FUNCT__ "Petsc_DelComm" 394 /* 395 This does not actually free anything, it simply marks when a reference count to an internal or external MPI_Comm reaches zero and the 396 the external MPI_Comm drops its reference to the internal or external MPI_Comm 397 398 This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator. 399 400 Note: this is declared extern "C" because it is passed to MPI_Keyval_create() 401 402 */ 403 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelComm(MPI_Comm comm,PetscMPIInt keyval,void *attr_val,void *extra_state) 404 { 405 PetscErrorCode ierr; 406 PetscMPIInt flg; 407 MPI_Comm icomm; 408 void *ptr; 409 410 PetscFunctionBegin; 411 ierr = MPI_Attr_get(comm,Petsc_InnerComm_keyval,&ptr,&flg);CHKERRQ(ierr); 412 if (flg) { 413 /* Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers */ 414 ierr = PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));CHKERRQ(ierr); 415 ierr = MPI_Attr_get(icomm,Petsc_OuterComm_keyval,&ptr,&flg);CHKERRQ(ierr); 416 if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected reference to outer comm"); 417 ierr = MPI_Attr_delete(icomm,Petsc_OuterComm_keyval);CHKERRQ(ierr); 418 ierr = PetscInfo1(0,"User MPI_Comm m %ld is being freed, removing reference from inner PETSc comm to this outer comm\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 419 } else { 420 ierr = PetscInfo1(0,"Removing reference to PETSc communicator imbedded in a user MPI_Comm m %ld\n",(long)comm);if (ierr) PetscFunctionReturn((PetscMPIInt)ierr); 421 } 422 PetscFunctionReturn(MPI_SUCCESS); 423 } 424 425 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32) 426 #if !defined(PETSC_WORDS_BIGENDIAN) 427 PETSC_EXTERN PetscMPIInt PetscDataRep_extent_fn(MPI_Datatype,MPI_Aint*,void*); 428 PETSC_EXTERN PetscMPIInt PetscDataRep_read_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*); 429 PETSC_EXTERN PetscMPIInt PetscDataRep_write_conv_fn(void*, MPI_Datatype,PetscMPIInt,void*,MPI_Offset,void*); 430 #endif 431 #endif 432 433 int PetscGlobalArgc = 0; 434 char **PetscGlobalArgs = 0; 435 436 #undef __FUNCT__ 437 #define __FUNCT__ "PetscGetArgs" 438 /*@C 439 PetscGetArgs - Allows you to access the raw command line arguments anywhere 440 after PetscInitialize() is called but before PetscFinalize(). 441 442 Not Collective 443 444 Output Parameters: 445 + argc - count of number of command line arguments 446 - args - the command line arguments 447 448 Level: intermediate 449 450 Notes: 451 This is usually used to pass the command line arguments into other libraries 452 that are called internally deep in PETSc or the application. 453 454 The first argument contains the program name as is normal for C arguments. 455 456 Concepts: command line arguments 457 458 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArguments() 459 460 @*/ 461 PetscErrorCode PetscGetArgs(int *argc,char ***args) 462 { 463 PetscFunctionBegin; 464 if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()"); 465 *argc = PetscGlobalArgc; 466 *args = PetscGlobalArgs; 467 PetscFunctionReturn(0); 468 } 469 470 #undef __FUNCT__ 471 #define __FUNCT__ "PetscGetArguments" 472 /*@C 473 PetscGetArguments - Allows you to access the command line arguments anywhere 474 after PetscInitialize() is called but before PetscFinalize(). 475 476 Not Collective 477 478 Output Parameters: 479 . args - the command line arguments 480 481 Level: intermediate 482 483 Notes: 484 This does NOT start with the program name and IS null terminated (final arg is void) 485 486 Concepts: command line arguments 487 488 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscFreeArguments() 489 490 @*/ 491 PetscErrorCode PetscGetArguments(char ***args) 492 { 493 PetscInt i,argc = PetscGlobalArgc; 494 PetscErrorCode ierr; 495 496 PetscFunctionBegin; 497 if (!PetscInitializeCalled && PetscFinalizeCalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ORDER,"You must call after PetscInitialize() but before PetscFinalize()"); 498 if (!argc) {*args = 0; PetscFunctionReturn(0);} 499 ierr = PetscMalloc(argc*sizeof(char*),args);CHKERRQ(ierr); 500 for (i=0; i<argc-1; i++) { 501 ierr = PetscStrallocpy(PetscGlobalArgs[i+1],&(*args)[i]);CHKERRQ(ierr); 502 } 503 (*args)[argc-1] = 0; 504 PetscFunctionReturn(0); 505 } 506 507 #undef __FUNCT__ 508 #define __FUNCT__ "PetscFreeArguments" 509 /*@C 510 PetscFreeArguments - Frees the memory obtained with PetscGetArguments() 511 512 Not Collective 513 514 Output Parameters: 515 . args - the command line arguments 516 517 Level: intermediate 518 519 Concepts: command line arguments 520 521 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscGetArguments() 522 523 @*/ 524 PetscErrorCode PetscFreeArguments(char **args) 525 { 526 PetscInt i = 0; 527 PetscErrorCode ierr; 528 529 PetscFunctionBegin; 530 if (!args) PetscFunctionReturn(0); 531 while (args[i]) { 532 ierr = PetscFree(args[i]);CHKERRQ(ierr); 533 i++; 534 } 535 ierr = PetscFree(args);CHKERRQ(ierr); 536 PetscFunctionReturn(0); 537 } 538 539 #undef __FUNCT__ 540 #define __FUNCT__ "PetscInitialize" 541 /*@C 542 PetscInitialize - Initializes the PETSc database and MPI. 543 PetscInitialize() calls MPI_Init() if that has yet to be called, 544 so this routine should always be called near the beginning of 545 your program -- usually the very first line! 546 547 Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set 548 549 Input Parameters: 550 + argc - count of number of command line arguments 551 . args - the command line arguments 552 . file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL to not check for 553 code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files 554 - help - [optional] Help message to print, use NULL for no message 555 556 If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that 557 communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a 558 four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not, 559 then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even 560 if different subcommunicators of the job are doing different things with PETSc. 561 562 Options Database Keys: 563 + -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger 564 . -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected 565 . -on_error_emacs <machinename> causes emacsclient to jump to error file 566 . -on_error_abort calls abort() when error detected (no traceback) 567 . -on_error_mpiabort calls MPI_abort() when error detected 568 . -error_output_stderr prints error messages to stderr instead of the default stdout 569 . -error_output_none does not print the error messages (but handles errors in the same way as if this was not called) 570 . -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger 571 . -debugger_pause [sleeptime] (in seconds) - Pauses debugger 572 . -stop_for_debugger - Print message on how to attach debugger manually to 573 process and wait (-debugger_pause) seconds for attachment 574 . -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries) 575 . -malloc no - Indicates not to use error-checking malloc 576 . -malloc_debug - check for memory corruption at EVERY malloc or free 577 . -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds 578 . -fp_trap - Stops on floating point exceptions (Note that on the 579 IBM RS6000 this slows code by at least a factor of 10.) 580 . -no_signal_handler - Indicates not to trap error signals 581 . -shared_tmp - indicates /tmp directory is shared by all processors 582 . -not_shared_tmp - each processor has own /tmp 583 . -tmp - alternative name of /tmp directory 584 . -get_total_flops - returns total flops done by all processors 585 . -memory_info - Print memory usage at end of run 586 - -server <port> - start PETSc webserver (default port is 8080) 587 588 Options Database Keys for Profiling: 589 See the <a href="../../docs/manual.pdf#nameddest=ch_profiling">profiling chapter of the users manual</a> for details. 590 + -info <optional filename> - Prints verbose information to the screen 591 . -info_exclude <null,vec,mat,pc,ksp,snes,ts> - Excludes some of the verbose messages 592 . -log_sync - Log the synchronization in scatters, inner products and norms 593 . -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program 594 hangs without running in the debugger). See PetscLogTraceBegin(). 595 . -log_summary [filename] - Prints summary of flop and timing information to screen. If the filename is specified the 596 summary is written to the file. See PetscLogView(). 597 . -log_summary_python [filename] - Prints data on of flop and timing usage to a file or screen. See PetscLogPrintSViewPython(). 598 . -log_all [filename] - Logs extensive profiling information See PetscLogDump(). 599 . -log [filename] - Logs basic profiline information See PetscLogDump(). 600 - -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution) 601 602 Only one of -log_trace, -log_summary, -log_all, -log, or -log_mpe may be used at a time 603 604 Environmental Variables: 605 + PETSC_TMP - alternative tmp directory 606 . PETSC_SHARED_TMP - tmp is shared by all processes 607 . PETSC_NOT_SHARED_TMP - each process has its own private tmp 608 . PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer 609 - PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to 610 611 612 Level: beginner 613 614 Notes: 615 If for some reason you must call MPI_Init() separately, call 616 it before PetscInitialize(). 617 618 Fortran Version: 619 In Fortran this routine has the format 620 $ call PetscInitialize(file,ierr) 621 622 + ierr - error return code 623 - file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL_CHARACTER to not check for 624 code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files 625 626 Important Fortran Note: 627 In Fortran, you MUST use NULL_CHARACTER to indicate a 628 null character string; you CANNOT just use NULL as 629 in the C version. See the <a href="../../docs/manual.pdf">users manual</a> for details. 630 631 If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after 632 calling PetscInitialize(). 633 634 Concepts: initializing PETSc 635 636 .seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments() 637 638 @*/ 639 PetscErrorCode PetscInitialize(int *argc,char ***args,const char file[],const char help[]) 640 { 641 PetscErrorCode ierr; 642 PetscMPIInt flag, size; 643 PetscInt nodesize; 644 PetscBool flg; 645 char hostname[256]; 646 647 PetscFunctionBegin; 648 if (PetscInitializeCalled) PetscFunctionReturn(0); 649 650 /* these must be initialized in a routine, not as a constant declaration*/ 651 PETSC_STDOUT = stdout; 652 PETSC_STDERR = stderr; 653 654 ierr = PetscOptionsCreate();CHKERRQ(ierr); 655 656 /* 657 We initialize the program name here (before MPI_Init()) because MPICH has a bug in 658 it that it sets args[0] on all processors to be args[0] on the first processor. 659 */ 660 if (argc && *argc) { 661 ierr = PetscSetProgramName(**args);CHKERRQ(ierr); 662 } else { 663 ierr = PetscSetProgramName("Unknown Name");CHKERRQ(ierr); 664 } 665 666 ierr = MPI_Initialized(&flag);CHKERRQ(ierr); 667 if (!flag) { 668 if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first"); 669 #if defined(PETSC_HAVE_MPI_INIT_THREAD) 670 { 671 PetscMPIInt provided; 672 ierr = MPI_Init_thread(argc,args,MPI_THREAD_FUNNELED,&provided);CHKERRQ(ierr); 673 } 674 #else 675 ierr = MPI_Init(argc,args);CHKERRQ(ierr); 676 #endif 677 PetscBeganMPI = PETSC_TRUE; 678 } 679 if (argc && args) { 680 PetscGlobalArgc = *argc; 681 PetscGlobalArgs = *args; 682 } 683 PetscFinalizeCalled = PETSC_FALSE; 684 685 if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD; 686 ierr = MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);CHKERRQ(ierr); 687 688 /* Done after init due to a bug in MPICH-GM? */ 689 ierr = PetscErrorPrintfInitialize();CHKERRQ(ierr); 690 691 ierr = MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);CHKERRQ(ierr); 692 ierr = MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);CHKERRQ(ierr); 693 694 MPIU_BOOL = MPI_INT; 695 MPIU_ENUM = MPI_INT; 696 697 /* 698 Initialized the global complex variable; this is because with 699 shared libraries the constructors for global variables 700 are not called; at least on IRIX. 701 */ 702 #if defined(PETSC_HAVE_COMPLEX) 703 { 704 #if defined(PETSC_CLANGUAGE_CXX) 705 PetscComplex ic(0.0,1.0); 706 PETSC_i = ic; 707 #elif defined(PETSC_CLANGUAGE_C) 708 PETSC_i = _Complex_I; 709 #endif 710 } 711 712 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 713 ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr); 714 ierr = MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr); 715 ierr = MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX);CHKERRQ(ierr); 716 ierr = MPI_Type_commit(&MPIU_C_COMPLEX);CHKERRQ(ierr); 717 #endif 718 #endif /* PETSC_HAVE_COMPLEX */ 719 720 /* 721 Create the PETSc MPI reduction operator that sums of the first 722 half of the entries and maxes the second half. 723 */ 724 ierr = MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);CHKERRQ(ierr); 725 726 #if defined(PETSC_USE_REAL___FLOAT128) 727 ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);CHKERRQ(ierr); 728 ierr = MPI_Type_commit(&MPIU___FLOAT128);CHKERRQ(ierr); 729 #if defined(PETSC_HAVE_COMPLEX) 730 ierr = MPI_Type_contiguous(4,MPI_DOUBLE,&MPIU___COMPLEX128);CHKERRQ(ierr); 731 ierr = MPI_Type_commit(&MPIU___COMPLEX128);CHKERRQ(ierr); 732 #endif 733 ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);CHKERRQ(ierr); 734 ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);CHKERRQ(ierr); 735 #endif 736 737 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 738 ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);CHKERRQ(ierr); 739 #endif 740 741 ierr = MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);CHKERRQ(ierr); 742 ierr = MPI_Type_commit(&MPIU_2SCALAR);CHKERRQ(ierr); 743 ierr = MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op);CHKERRQ(ierr); 744 ierr = MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op);CHKERRQ(ierr); 745 746 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 747 ierr = MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);CHKERRQ(ierr); 748 ierr = MPI_Type_commit(&MPIU_2INT);CHKERRQ(ierr); 749 #endif 750 751 /* 752 Attributes to be set on PETSc communicators 753 */ 754 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0);CHKERRQ(ierr); 755 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_InnerComm_keyval,(void*)0);CHKERRQ(ierr); 756 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm,&Petsc_OuterComm_keyval,(void*)0);CHKERRQ(ierr); 757 758 /* 759 Build the options database 760 */ 761 ierr = PetscOptionsInsert(argc,args,file);CHKERRQ(ierr); 762 763 764 /* 765 Print main application help message 766 */ 767 ierr = PetscOptionsHasName(NULL,"-help",&flg);CHKERRQ(ierr); 768 if (help && flg) { 769 ierr = PetscPrintf(PETSC_COMM_WORLD,help);CHKERRQ(ierr); 770 } 771 ierr = PetscOptionsCheckInitial_Private();CHKERRQ(ierr); 772 773 /* SHOULD PUT IN GUARDS: Make sure logging is initialized, even if we do not print it out */ 774 #if defined(PETSC_USE_LOG) 775 ierr = PetscLogBegin_Private();CHKERRQ(ierr); 776 #endif 777 778 /* 779 Load the dynamic libraries (on machines that support them), this registers all 780 the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes) 781 */ 782 ierr = PetscInitialize_DynamicLibraries();CHKERRQ(ierr); 783 784 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 785 ierr = PetscInfo1(0,"PETSc successfully started: number of processors = %d\n",size);CHKERRQ(ierr); 786 ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr); 787 ierr = PetscInfo1(0,"Running on machine: %s\n",hostname);CHKERRQ(ierr); 788 789 ierr = PetscOptionsCheckInitial_Components();CHKERRQ(ierr); 790 /* Check the options database for options related to the options database itself */ 791 ierr = PetscOptionsSetFromOptions();CHKERRQ(ierr); 792 793 #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32) 794 /* 795 Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI 796 797 Currently not used because it is not supported by MPICH. 798 */ 799 #if !defined(PETSC_WORDS_BIGENDIAN) 800 ierr = MPI_Register_datarep((char*)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,NULL);CHKERRQ(ierr); 801 #endif 802 #endif 803 804 ierr = PetscOptionsGetInt(NULL,"-hmpi_spawn_size",&nodesize,&flg);CHKERRQ(ierr); 805 if (flg) { 806 #if defined(PETSC_HAVE_MPI_COMM_SPAWN) 807 ierr = PetscHMPISpawn((PetscMPIInt) nodesize);CHKERRQ(ierr); /* worker nodes never return from here; they go directly to PetscEnd() */ 808 #else 809 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead"); 810 #endif 811 } else { 812 ierr = PetscOptionsGetInt(NULL,"-hmpi_merge_size",&nodesize,&flg);CHKERRQ(ierr); 813 if (flg) { 814 ierr = PetscHMPIMerge((PetscMPIInt) nodesize,NULL,NULL);CHKERRQ(ierr); 815 if (PetscHMPIWorker) { /* if worker then never enter user code */ 816 PetscInitializeCalled = PETSC_TRUE; 817 PetscEnd(); 818 } 819 } 820 } 821 822 #if defined(PETSC_HAVE_CUDA) 823 { 824 PetscMPIInt p; 825 for (p = 0; p < PetscGlobalSize; ++p) { 826 if (p == PetscGlobalRank) cublasInit(); 827 ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); 828 } 829 } 830 #endif 831 832 ierr = PetscOptionsHasName(NULL,"-python",&flg);CHKERRQ(ierr); 833 if (flg) { 834 PetscInitializeCalled = PETSC_TRUE; 835 ierr = PetscPythonInitialize(NULL,NULL);CHKERRQ(ierr); 836 } 837 838 ierr = PetscThreadCommInitializePackage();CHKERRQ(ierr); 839 840 /* 841 Setup building of stack frames for all function calls 842 */ 843 #if defined(PETSC_USE_DEBUG) 844 PetscThreadLocalRegister((PetscThreadKey*)&petscstack); /* Creates petscstack_key if needed */ 845 ierr = PetscStackCreate();CHKERRQ(ierr); 846 #endif 847 848 #if defined(PETSC_SERIALIZE_FUNCTIONS) 849 ierr = PetscFPTCreate(10000);CHKERRQ(ierr); 850 #endif 851 852 /* 853 Once we are completedly initialized then we can set this variables 854 */ 855 PetscInitializeCalled = PETSC_TRUE; 856 PetscFunctionReturn(0); 857 } 858 859 extern PetscObject *PetscObjects; 860 extern PetscInt PetscObjectsCounts, PetscObjectsMaxCounts; 861 862 #undef __FUNCT__ 863 #define __FUNCT__ "PetscFinalize" 864 /*@C 865 PetscFinalize - Checks for options to be called at the conclusion 866 of the program. MPI_Finalize() is called only if the user had not 867 called MPI_Init() before calling PetscInitialize(). 868 869 Collective on PETSC_COMM_WORLD 870 871 Options Database Keys: 872 + -options_table - Calls PetscOptionsView() 873 . -options_left - Prints unused options that remain in the database 874 . -objects_dump [all] - Prints list of objects allocated by the user that have not been freed, the option all cause all outstanding objects to be listed 875 . -mpidump - Calls PetscMPIDump() 876 . -malloc_dump - Calls PetscMallocDump() 877 . -malloc_info - Prints total memory usage 878 - -malloc_log - Prints summary of memory usage 879 880 Level: beginner 881 882 Note: 883 See PetscInitialize() for more general runtime options. 884 885 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscEnd() 886 @*/ 887 PetscErrorCode PetscFinalize(void) 888 { 889 PetscErrorCode ierr; 890 PetscMPIInt rank; 891 PetscInt nopt; 892 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE; 893 #if defined(PETSC_HAVE_AMS) 894 PetscBool flg = PETSC_FALSE; 895 #endif 896 #if defined(PETSC_USE_LOG) 897 char mname[PETSC_MAX_PATH_LEN]; 898 #endif 899 900 PetscFunctionBegin; 901 if (!PetscInitializeCalled) { 902 printf("PetscInitialize() must be called before PetscFinalize()\n"); 903 PetscFunctionReturn(PETSC_ERR_ARG_WRONGSTATE); 904 } 905 ierr = PetscInfo(NULL,"PetscFinalize() called\n");CHKERRQ(ierr); 906 907 #if defined(PETSC_SERIALIZE_FUNCTIONS) 908 ierr = PetscFPTDestroy();CHKERRQ(ierr); 909 #endif 910 911 912 #if defined(PETSC_HAVE_AMS) 913 ierr = PetscOptionsGetBool(NULL,"-options_gui",&flg,NULL);CHKERRQ(ierr); 914 if (flg) { 915 ierr = PetscOptionsAMSDestroy();CHKERRQ(ierr); 916 } 917 #endif 918 919 #if defined(PETSC_HAVE_SERVER) 920 flg1 = PETSC_FALSE; 921 ierr = PetscOptionsGetBool(NULL,"-server",&flg1,NULL);CHKERRQ(ierr); 922 if (flg1) { 923 /* this is a crude hack, but better than nothing */ 924 ierr = PetscPOpen(PETSC_COMM_WORLD,NULL,"pkill -9 petscwebserver","r",NULL);CHKERRQ(ierr); 925 } 926 #endif 927 928 ierr = PetscHMPIFinalize();CHKERRQ(ierr); 929 930 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 931 ierr = PetscOptionsGetBool(NULL,"-malloc_info",&flg2,NULL);CHKERRQ(ierr); 932 if (!flg2) { 933 flg2 = PETSC_FALSE; 934 ierr = PetscOptionsGetBool(NULL,"-memory_info",&flg2,NULL);CHKERRQ(ierr); 935 } 936 if (flg2) { 937 ierr = PetscMemoryShowUsage(PETSC_VIEWER_STDOUT_WORLD,"Summary of Memory Usage in PETSc\n");CHKERRQ(ierr); 938 } 939 940 #if defined(PETSC_USE_LOG) 941 flg1 = PETSC_FALSE; 942 ierr = PetscOptionsGetBool(NULL,"-get_total_flops",&flg1,NULL);CHKERRQ(ierr); 943 if (flg1) { 944 PetscLogDouble flops = 0; 945 ierr = MPI_Reduce(&petsc_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);CHKERRQ(ierr); 946 ierr = PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %g\n",flops);CHKERRQ(ierr); 947 } 948 #endif 949 950 951 #if defined(PETSC_USE_LOG) 952 #if defined(PETSC_HAVE_MPE) 953 mname[0] = 0; 954 955 ierr = PetscOptionsGetString(NULL,"-log_mpe",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 956 if (flg1) { 957 if (mname[0]) {ierr = PetscLogMPEDump(mname);CHKERRQ(ierr);} 958 else {ierr = PetscLogMPEDump(0);CHKERRQ(ierr);} 959 } 960 #endif 961 mname[0] = 0; 962 963 ierr = PetscOptionsGetString(NULL,"-log_summary",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 964 if (flg1) { 965 PetscViewer viewer; 966 if (mname[0]) { 967 ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr); 968 ierr = PetscLogView(viewer);CHKERRQ(ierr); 969 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 970 } else { 971 viewer = PETSC_VIEWER_STDOUT_WORLD; 972 ierr = PetscLogView(viewer);CHKERRQ(ierr); 973 } 974 } 975 976 mname[0] = 0; 977 978 ierr = PetscOptionsGetString(NULL,"-log_summary_python",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 979 if (flg1) { 980 PetscViewer viewer; 981 if (mname[0]) { 982 ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,mname,&viewer);CHKERRQ(ierr); 983 ierr = PetscLogViewPython(viewer);CHKERRQ(ierr); 984 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 985 } else { 986 viewer = PETSC_VIEWER_STDOUT_WORLD; 987 ierr = PetscLogViewPython(viewer);CHKERRQ(ierr); 988 } 989 } 990 991 ierr = PetscOptionsGetString(NULL,"-log_detailed",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 992 if (flg1) { 993 if (mname[0]) {ierr = PetscLogPrintDetailed(PETSC_COMM_WORLD,mname);CHKERRQ(ierr);} 994 else {ierr = PetscLogPrintDetailed(PETSC_COMM_WORLD,0);CHKERRQ(ierr);} 995 } 996 997 mname[0] = 0; 998 999 ierr = PetscOptionsGetString(NULL,"-log_all",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 1000 ierr = PetscOptionsGetString(NULL,"-log",mname,PETSC_MAX_PATH_LEN,&flg2);CHKERRQ(ierr); 1001 if (flg1 || flg2) { 1002 if (mname[0]) PetscLogDump(mname); 1003 else PetscLogDump(0); 1004 } 1005 #endif 1006 1007 /* 1008 Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_(). 1009 */ 1010 ierr = PetscObjectRegisterDestroyAll();CHKERRQ(ierr); 1011 1012 { 1013 PetscThreadComm tcomm_world; 1014 ierr = PetscGetThreadCommWorld(&tcomm_world);CHKERRQ(ierr); 1015 /* Free global thread communicator */ 1016 ierr = PetscThreadCommDestroy(&tcomm_world);CHKERRQ(ierr); 1017 } 1018 1019 #if defined(PETSC_USE_DEBUG) 1020 ierr = PetscStackDestroy();CHKERRQ(ierr); 1021 #endif 1022 1023 flg1 = PETSC_FALSE; 1024 ierr = PetscOptionsGetBool(NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 1025 if (!flg1) { ierr = PetscPopSignalHandler();CHKERRQ(ierr);} 1026 flg1 = PETSC_FALSE; 1027 ierr = PetscOptionsGetBool(NULL,"-mpidump",&flg1,NULL);CHKERRQ(ierr); 1028 if (flg1) { 1029 ierr = PetscMPIDump(stdout);CHKERRQ(ierr); 1030 } 1031 flg1 = PETSC_FALSE; 1032 flg2 = PETSC_FALSE; 1033 /* preemptive call to avoid listing this option in options table as unused */ 1034 ierr = PetscOptionsHasName(NULL,"-malloc_dump",&flg1);CHKERRQ(ierr); 1035 ierr = PetscOptionsHasName(NULL,"-objects_dump",&flg1);CHKERRQ(ierr); 1036 ierr = PetscOptionsGetBool(NULL,"-options_table",&flg2,NULL);CHKERRQ(ierr); 1037 1038 if (flg2) { 1039 PetscViewer viewer; 1040 ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr); 1041 ierr = PetscOptionsView(viewer);CHKERRQ(ierr); 1042 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 1043 } 1044 1045 /* to prevent PETSc -options_left from warning */ 1046 ierr = PetscOptionsHasName(NULL,"-nox",&flg1);CHKERRQ(ierr); 1047 ierr = PetscOptionsHasName(NULL,"-nox_warning",&flg1);CHKERRQ(ierr); 1048 1049 if (!PetscHMPIWorker) { /* worker processes skip this because they do not usually process options */ 1050 flg3 = PETSC_FALSE; /* default value is required */ 1051 ierr = PetscOptionsGetBool(NULL,"-options_left",&flg3,&flg1);CHKERRQ(ierr); 1052 ierr = PetscOptionsAllUsed(&nopt);CHKERRQ(ierr); 1053 if (flg3) { 1054 if (!flg2) { /* have not yet printed the options */ 1055 PetscViewer viewer; 1056 ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr); 1057 ierr = PetscOptionsView(viewer);CHKERRQ(ierr); 1058 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 1059 } 1060 if (!nopt) { 1061 ierr = PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.\n");CHKERRQ(ierr); 1062 } else if (nopt == 1) { 1063 ierr = PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:\n");CHKERRQ(ierr); 1064 } else { 1065 ierr = PetscPrintf(PETSC_COMM_WORLD,"There are %D unused database options. They are:\n",nopt);CHKERRQ(ierr); 1066 } 1067 } 1068 #if defined(PETSC_USE_DEBUG) 1069 if (nopt && !flg3 && !flg1) { 1070 ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!\n");CHKERRQ(ierr); 1071 ierr = PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!\n");CHKERRQ(ierr); 1072 ierr = PetscOptionsLeft();CHKERRQ(ierr); 1073 } else if (nopt && flg3) { 1074 #else 1075 if (nopt && flg3) { 1076 #endif 1077 ierr = PetscOptionsLeft();CHKERRQ(ierr); 1078 } 1079 } 1080 1081 /* 1082 List all objects the user may have forgot to free 1083 */ 1084 ierr = PetscOptionsHasName(NULL,"-objects_dump",&flg1);CHKERRQ(ierr); 1085 if (flg1) { 1086 MPI_Comm local_comm; 1087 char string[64]; 1088 1089 ierr = PetscOptionsGetString(NULL,"-objects_dump",string,64,NULL);CHKERRQ(ierr); 1090 ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRQ(ierr); 1091 ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr); 1092 ierr = PetscObjectsDump(stdout,(string[0] == 'a') ? PETSC_TRUE : PETSC_FALSE);CHKERRQ(ierr); 1093 ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr); 1094 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 1095 } 1096 PetscObjectsCounts = 0; 1097 PetscObjectsMaxCounts = 0; 1098 1099 ierr = PetscFree(PetscObjects);CHKERRQ(ierr); 1100 1101 #if defined(PETSC_USE_LOG) 1102 ierr = PetscLogDestroy();CHKERRQ(ierr); 1103 #endif 1104 1105 /* 1106 Destroy any packages that registered a finalize 1107 */ 1108 ierr = PetscRegisterFinalizeAll();CHKERRQ(ierr); 1109 1110 /* 1111 Destroy all the function registration lists created 1112 */ 1113 ierr = PetscFinalize_DynamicLibraries();CHKERRQ(ierr); 1114 1115 /* 1116 Print PetscFunctionLists that have not been properly freed 1117 1118 ierr = PetscFunctionListPrintAll();CHKERRQ(ierr); 1119 */ 1120 1121 if (petsc_history) { 1122 ierr = PetscCloseHistoryFile(&petsc_history);CHKERRQ(ierr); 1123 petsc_history = 0; 1124 } 1125 1126 ierr = PetscInfoAllow(PETSC_FALSE,NULL);CHKERRQ(ierr); 1127 1128 { 1129 char fname[PETSC_MAX_PATH_LEN]; 1130 FILE *fd; 1131 int err; 1132 1133 fname[0] = 0; 1134 1135 ierr = PetscOptionsGetString(NULL,"-malloc_dump",fname,250,&flg1);CHKERRQ(ierr); 1136 flg2 = PETSC_FALSE; 1137 ierr = PetscOptionsGetBool(NULL,"-malloc_test",&flg2,NULL);CHKERRQ(ierr); 1138 #if defined(PETSC_USE_DEBUG) 1139 if (PETSC_RUNNING_ON_VALGRIND) flg2 = PETSC_FALSE; 1140 #else 1141 flg2 = PETSC_FALSE; /* Skip reporting for optimized builds regardless of -malloc_test */ 1142 #endif 1143 if (flg1 && fname[0]) { 1144 char sname[PETSC_MAX_PATH_LEN]; 1145 1146 sprintf(sname,"%s_%d",fname,rank); 1147 fd = fopen(sname,"w"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",sname); 1148 ierr = PetscMallocDump(fd);CHKERRQ(ierr); 1149 err = fclose(fd); 1150 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 1151 } else if (flg1 || flg2) { 1152 MPI_Comm local_comm; 1153 1154 ierr = MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);CHKERRQ(ierr); 1155 ierr = PetscSequentialPhaseBegin_Private(local_comm,1);CHKERRQ(ierr); 1156 ierr = PetscMallocDump(stdout);CHKERRQ(ierr); 1157 ierr = PetscSequentialPhaseEnd_Private(local_comm,1);CHKERRQ(ierr); 1158 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 1159 } 1160 } 1161 1162 { 1163 char fname[PETSC_MAX_PATH_LEN]; 1164 FILE *fd = NULL; 1165 1166 fname[0] = 0; 1167 1168 ierr = PetscOptionsGetString(NULL,"-malloc_log",fname,250,&flg1);CHKERRQ(ierr); 1169 ierr = PetscOptionsHasName(NULL,"-malloc_log_threshold",&flg2);CHKERRQ(ierr); 1170 if (flg1 && fname[0]) { 1171 int err; 1172 1173 if (!rank) { 1174 fd = fopen(fname,"w"); 1175 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open log file: %s",fname); 1176 } 1177 ierr = PetscMallocDumpLog(fd);CHKERRQ(ierr); 1178 if (fd) { 1179 err = fclose(fd); 1180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 1181 } 1182 } else if (flg1 || flg2) { 1183 ierr = PetscMallocDumpLog(stdout);CHKERRQ(ierr); 1184 } 1185 } 1186 /* Can be destroyed only after all the options are used */ 1187 ierr = PetscOptionsDestroy();CHKERRQ(ierr); 1188 1189 PetscGlobalArgc = 0; 1190 PetscGlobalArgs = 0; 1191 1192 #if defined(PETSC_USE_REAL___FLOAT128) 1193 ierr = MPI_Type_free(&MPIU___FLOAT128);CHKERRQ(ierr); 1194 #if defined(PETSC_HAVE_COMPLEX) 1195 ierr = MPI_Type_free(&MPIU___COMPLEX128);CHKERRQ(ierr); 1196 #endif 1197 ierr = MPI_Op_free(&MPIU_MAX);CHKERRQ(ierr); 1198 ierr = MPI_Op_free(&MPIU_MIN);CHKERRQ(ierr); 1199 #endif 1200 1201 #if defined(PETSC_HAVE_COMPLEX) 1202 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 1203 ierr = MPI_Type_free(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr); 1204 ierr = MPI_Type_free(&MPIU_C_COMPLEX);CHKERRQ(ierr); 1205 #endif 1206 #endif 1207 1208 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 1209 ierr = MPI_Op_free(&MPIU_SUM);CHKERRQ(ierr); 1210 #endif 1211 1212 ierr = MPI_Type_free(&MPIU_2SCALAR);CHKERRQ(ierr); 1213 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 1214 ierr = MPI_Type_free(&MPIU_2INT);CHKERRQ(ierr); 1215 #endif 1216 ierr = MPI_Op_free(&PetscMaxSum_Op);CHKERRQ(ierr); 1217 ierr = MPI_Op_free(&PetscADMax_Op);CHKERRQ(ierr); 1218 ierr = MPI_Op_free(&PetscADMin_Op);CHKERRQ(ierr); 1219 1220 /* 1221 Destroy any known inner MPI_Comm's and attributes pointing to them 1222 Note this will not destroy any new communicators the user has created. 1223 1224 If all PETSc objects were not destroyed those left over objects will have hanging references to 1225 the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again 1226 */ 1227 { 1228 PetscCommCounter *counter; 1229 PetscMPIInt flg; 1230 MPI_Comm icomm; 1231 void *ptr; 1232 ierr = MPI_Attr_get(PETSC_COMM_SELF,Petsc_InnerComm_keyval,&ptr,&flg);CHKERRQ(ierr); 1233 if (flg) { 1234 /* Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers */ 1235 ierr = PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));CHKERRQ(ierr); 1236 ierr = MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRQ(ierr); 1237 if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory"); 1238 1239 ierr = MPI_Attr_delete(PETSC_COMM_SELF,Petsc_InnerComm_keyval);CHKERRQ(ierr); 1240 ierr = MPI_Attr_delete(icomm,Petsc_Counter_keyval);CHKERRQ(ierr); 1241 ierr = MPI_Comm_free(&icomm);CHKERRQ(ierr); 1242 } 1243 ierr = MPI_Attr_get(PETSC_COMM_WORLD,Petsc_InnerComm_keyval,&ptr,&flg);CHKERRQ(ierr); 1244 if (flg) { 1245 /* Use PetscMemcpy() because casting from pointer to integer of different size is not allowed with some compilers */ 1246 ierr = PetscMemcpy(&icomm,&ptr,sizeof(MPI_Comm));CHKERRQ(ierr); 1247 ierr = MPI_Attr_get(icomm,Petsc_Counter_keyval,&counter,&flg);CHKERRQ(ierr); 1248 if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_CORRUPT,"Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory"); 1249 1250 ierr = MPI_Attr_delete(PETSC_COMM_WORLD,Petsc_InnerComm_keyval);CHKERRQ(ierr); 1251 ierr = MPI_Attr_delete(icomm,Petsc_Counter_keyval);CHKERRQ(ierr); 1252 ierr = MPI_Comm_free(&icomm);CHKERRQ(ierr); 1253 } 1254 } 1255 1256 ierr = MPI_Keyval_free(&Petsc_Counter_keyval);CHKERRQ(ierr); 1257 ierr = MPI_Keyval_free(&Petsc_InnerComm_keyval);CHKERRQ(ierr); 1258 ierr = MPI_Keyval_free(&Petsc_OuterComm_keyval);CHKERRQ(ierr); 1259 1260 #if defined(PETSC_HAVE_CUDA) 1261 { 1262 PetscInt p; 1263 for (p = 0; p < PetscGlobalSize; ++p) { 1264 if (p == PetscGlobalRank) cublasShutdown(); 1265 ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); 1266 } 1267 } 1268 #endif 1269 1270 if (PetscBeganMPI) { 1271 #if defined(PETSC_HAVE_MPI_FINALIZED) 1272 PetscMPIInt flag; 1273 ierr = MPI_Finalized(&flag);CHKERRQ(ierr); 1274 if (flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()"); 1275 #endif 1276 ierr = MPI_Finalize();CHKERRQ(ierr); 1277 } 1278 /* 1279 1280 Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because 1281 the communicator has some outstanding requests on it. Specifically if the 1282 flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See 1283 src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate() 1284 is never freed as it should be. Thus one may obtain messages of the form 1285 [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the 1286 memory was not freed. 1287 1288 */ 1289 ierr = PetscMallocClear();CHKERRQ(ierr); 1290 1291 PetscInitializeCalled = PETSC_FALSE; 1292 PetscFinalizeCalled = PETSC_TRUE; 1293 PetscFunctionReturn(ierr); 1294 } 1295 1296 #if defined(PETSC_MISSING_LAPACK_lsame_) 1297 PETSC_EXTERN int lsame_(char *a,char *b) 1298 { 1299 if (*a == *b) return 1; 1300 if (*a + 32 == *b) return 1; 1301 if (*a - 32 == *b) return 1; 1302 return 0; 1303 } 1304 #endif 1305 1306 #if defined(PETSC_MISSING_LAPACK_lsame) 1307 PETSC_EXTERN int lsame(char *a,char *b) 1308 { 1309 if (*a == *b) return 1; 1310 if (*a + 32 == *b) return 1; 1311 if (*a - 32 == *b) return 1; 1312 return 0; 1313 } 1314 #endif 1315