1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 58 59 #if defined(PETSC_HAVE_COMPLEX) 60 #if defined(PETSC_COMPLEX_INSTANTIATE) 61 template <> class std::complex<double>; /* instantiate complex template class */ 62 #endif 63 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 64 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 65 MPI_Datatype MPIU_C_COMPLEX; 66 #endif 67 68 /*MC 69 PETSC_i - the imaginary number i 70 71 Synopsis: 72 #include <petscsys.h> 73 PetscComplex PETSC_i; 74 75 Level: beginner 76 77 Note: 78 Complex numbers are automatically available if PETSc located a working complex implementation 79 80 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 81 M*/ 82 PetscComplex PETSC_i; 83 #endif 84 #if defined(PETSC_USE_REAL___FLOAT128) 85 MPI_Datatype MPIU___FLOAT128 = 0; 86 #if defined(PETSC_HAVE_COMPLEX) 87 MPI_Datatype MPIU___COMPLEX128 = 0; 88 #endif 89 #elif defined(PETSC_USE_REAL___FP16) 90 MPI_Datatype MPIU___FP16 = 0; 91 #endif 92 MPI_Datatype MPIU_2SCALAR = 0; 93 #if defined(PETSC_USE_64BIT_INDICES) 94 MPI_Datatype MPIU_2INT = 0; 95 #endif 96 MPI_Datatype MPIU_BOOL; 97 MPI_Datatype MPIU_ENUM; 98 MPI_Datatype MPIU_FORTRANADDR; 99 MPI_Datatype MPIU_SIZE_T; 100 101 /* 102 Function that is called to display all error messages 103 */ 104 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 105 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 106 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 107 /* 108 This is needed to turn on/off GPU synchronization 109 */ 110 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 111 112 /* ------------------------------------------------------------------------------*/ 113 /* 114 Optional file where all PETSc output from various prints is saved 115 */ 116 PETSC_INTERN FILE *petsc_history; 117 FILE *petsc_history = NULL; 118 119 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 120 { 121 PetscErrorCode ierr; 122 PetscMPIInt rank,size; 123 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 124 char version[256]; 125 126 PetscFunctionBegin; 127 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 128 if (!rank) { 129 char arch[10]; 130 int err; 131 132 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 133 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 134 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 135 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 136 if (filename) { 137 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 138 } else { 139 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 140 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 141 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 142 } 143 144 *fd = fopen(fname,"a"); 145 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 146 147 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 148 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 149 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 150 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 153 err = fflush(*fd); 154 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 155 } 156 PetscFunctionReturn(0); 157 } 158 159 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 160 { 161 PetscErrorCode ierr; 162 PetscMPIInt rank; 163 char date[64]; 164 int err; 165 166 PetscFunctionBegin; 167 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 168 if (!rank) { 169 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 170 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 171 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 172 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 173 err = fflush(*fd); 174 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 175 err = fclose(*fd); 176 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 177 } 178 PetscFunctionReturn(0); 179 } 180 181 /* ------------------------------------------------------------------------------*/ 182 183 /* 184 This is ugly and probably belongs somewhere else, but I want to 185 be able to put a true MPI abort error handler with command line args. 186 187 This is so MPI errors in the debugger will leave all the stack 188 frames. The default MP_Abort() cleans up and exits thus providing no useful information 189 in the debugger hence we call abort() instead of MPI_Abort(). 190 */ 191 192 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 193 { 194 PetscFunctionBegin; 195 (*PetscErrorPrintf)("MPI error %d\n",*flag); 196 abort(); 197 } 198 199 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 200 { 201 PetscErrorCode ierr; 202 203 PetscFunctionBegin; 204 (*PetscErrorPrintf)("MPI error %d\n",*flag); 205 ierr = PetscAttachDebugger(); 206 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 207 } 208 209 /*@C 210 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 211 wishes a clean exit somewhere deep in the program. 212 213 Collective on PETSC_COMM_WORLD 214 215 Options Database Keys are the same as for PetscFinalize() 216 217 Level: advanced 218 219 Note: 220 See PetscInitialize() for more general runtime options. 221 222 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 223 @*/ 224 PetscErrorCode PetscEnd(void) 225 { 226 PetscFunctionBegin; 227 PetscFinalize(); 228 exit(0); 229 return 0; 230 } 231 232 PetscBool PetscOptionsPublish = PETSC_FALSE; 233 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 234 PETSC_INTERN PetscBool petscsetmallocvisited; 235 static char emacsmachinename[256]; 236 237 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 238 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 239 240 /*@C 241 PetscSetHelpVersionFunctions - Sets functions that print help and version information 242 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 243 This routine enables a "higher-level" package that uses PETSc to print its messages first. 244 245 Input Parameter: 246 + help - the help function (may be NULL) 247 - version - the version function (may be NULL) 248 249 Level: developer 250 251 @*/ 252 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 253 { 254 PetscFunctionBegin; 255 PetscExternalHelpFunction = help; 256 PetscExternalVersionFunction = version; 257 PetscFunctionReturn(0); 258 } 259 260 #if defined(PETSC_USE_LOG) 261 PETSC_INTERN PetscBool PetscObjectsLog; 262 #endif 263 264 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 265 { 266 if (PetscUnlikely(*err)) { 267 PetscMPIInt len; 268 char errstring[MPI_MAX_ERROR_STRING]; 269 270 MPI_Error_string(*err,errstring,&len); 271 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 272 } 273 return; 274 } 275 276 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 277 Use the following macros to define CUDA/HIP initialization related vars/routines. 278 */ 279 #if defined(PETSC_HAVE_CUDA) 280 typedef cudaError_t cupmError_t; 281 typedef struct cudaDeviceProp cupmDeviceProp; 282 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 283 #define cupmGetDevice(x) cudaGetDevice(x) 284 #define cupmSetDevice(x) cudaSetDevice(x) 285 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 286 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 287 #define cupmGetLastError() cudaGetLastError() 288 #define cupmDeviceMapHost cudaDeviceMapHost 289 #define cupmSuccess cudaSuccess 290 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 291 #define CHKERRCUPM(x) CHKERRCUDA(x) 292 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 293 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 294 #define PetscCUPMInitialize PetscCUDAInitialize 295 #define PetscCUPMInitialized PetscCUDAInitialized 296 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 297 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 298 #define PetscCUPMSynchronize PetscCUDASynchronize 299 #define PetscNotUseCUPM PetscNotUseCUDA 300 #define cupmOptionsStr "CUDA options" 301 #define cupmSetDeviceStr "-cuda_set_device" 302 #define cupmViewStr "-cuda_view" 303 #define cupmSynchronizeStr "-cuda_synchronize" 304 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 305 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 306 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 307 #include "cupminit.inc" 308 #endif 309 310 #if defined(PETSC_HAVE_HIP) 311 typedef hipError_t cupmError_t; 312 typedef hipDeviceProp_t cupmDeviceProp; 313 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 314 #define cupmGetDevice(x) hipGetDevice(x) 315 #define cupmSetDevice(x) hipSetDevice(x) 316 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 317 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 318 #define cupmGetLastError() hipGetLastError() 319 #define cupmDeviceMapHost hipDeviceMapHost 320 #define cupmSuccess hipSuccess 321 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 322 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 323 #define PetscCUPMBLASInitializeHandle() 0 324 #define PetscCUPMSOLVERDnInitializeHandle() 0 325 #define PetscCUPMInitialize PetscHIPInitialize 326 #define PetscCUPMInitialized PetscHIPInitialized 327 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 328 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 329 #define PetscCUPMSynchronize PetscHIPSynchronize 330 #define PetscNotUseCUPM PetscNotUseHIP 331 #define cupmOptionsStr "HIP options" 332 #define cupmSetDeviceStr "-hip_set_device" 333 #define cupmViewStr "-hip_view" 334 #define cupmSynchronizeStr "-hip_synchronize" 335 #define PetscCUPMInitializeStr "PetscHIPInitialize" 336 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 337 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 338 #include "cupminit.inc" 339 #endif 340 341 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 342 { 343 char string[64]; 344 MPI_Comm comm = PETSC_COMM_WORLD; 345 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 346 PetscErrorCode ierr; 347 PetscReal si; 348 PetscInt intensity; 349 int i; 350 PetscMPIInt rank; 351 char version[256]; 352 #if defined(PETSC_USE_LOG) 353 char mname[PETSC_MAX_PATH_LEN]; 354 PetscViewerFormat format; 355 PetscBool flg4 = PETSC_FALSE; 356 #endif 357 358 PetscFunctionBegin; 359 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 360 361 #if !defined(PETSC_HAVE_THREADSAFETY) 362 if (!(PETSC_RUNNING_ON_VALGRIND)) { 363 /* 364 Setup the memory management; support for tracing malloc() usage 365 */ 366 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 367 368 if (PetscDefined(USE_DEBUG)) { 369 mdebug = PETSC_TRUE; 370 initializenan = PETSC_TRUE; 371 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 372 } else { 373 /* don't warn about unused option */ 374 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 375 flg1 = PETSC_FALSE; 376 } 377 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 378 if (flg1 || flg2) { 379 mdebug = PETSC_TRUE; 380 eachcall = PETSC_TRUE; 381 initializenan = PETSC_TRUE; 382 } else if (flg3 && !flg2) { 383 mdebug = PETSC_FALSE; 384 eachcall = PETSC_FALSE; 385 initializenan = PETSC_FALSE; 386 } 387 388 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 389 if (mlog) { 390 mdebug = PETSC_TRUE; 391 } 392 /* the next line is deprecated */ 393 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 394 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 395 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 396 if (mdebug) { 397 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 398 } 399 if (mlog) { 400 PetscReal logthreshold = 0; 401 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 402 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 403 } 404 #if defined(PETSC_USE_LOG) 405 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 406 #endif 407 } 408 409 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 410 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 411 flg1 = PETSC_FALSE; 412 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 413 /* ignore this option if malloc is already set */ 414 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 415 416 flg1 = PETSC_FALSE; 417 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 418 if (!flg1) { 419 flg1 = PETSC_FALSE; 420 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 421 } 422 if (flg1) { 423 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 424 } 425 #endif 426 427 #if defined(PETSC_USE_LOG) 428 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 429 #endif 430 431 /* 432 Set the display variable for graphics 433 */ 434 ierr = PetscSetDisplay();CHKERRQ(ierr); 435 436 /* 437 Print main application help message 438 */ 439 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 440 if (help && hasHelp) { 441 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 442 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 443 } 444 445 /* 446 Print the PETSc version information 447 */ 448 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 449 if (flg1 || hasHelp) { 450 /* 451 Print "higher-level" package version message 452 */ 453 if (PetscExternalVersionFunction) { 454 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 455 } 456 457 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 458 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 459 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 460 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 461 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 462 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 463 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 464 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 465 } 466 467 /* 468 Print "higher-level" package help message 469 */ 470 if (hasHelp) { 471 PetscBool hasHelpIntro; 472 473 if (PetscExternalHelpFunction) { 474 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 475 } 476 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 477 if (hasHelpIntro) { 478 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 479 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 480 ierr = MPI_Finalize();CHKERRQ(ierr); 481 exit(0); 482 } 483 } 484 485 /* 486 Setup the error handling 487 */ 488 flg1 = PETSC_FALSE; 489 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 490 if (flg1) { 491 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 492 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 493 } 494 flg1 = PETSC_FALSE; 495 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 496 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 497 flg1 = PETSC_FALSE; 498 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 499 if (flg1) { 500 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 501 } 502 /* experimental */ 503 flg1 = PETSC_FALSE; 504 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 505 if (flg1) { 506 MPI_Errhandler eh; 507 508 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 509 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 510 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 511 } 512 flg1 = PETSC_FALSE; 513 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 514 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 515 516 /* 517 Setup debugger information 518 */ 519 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 520 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 521 if (flg1) { 522 MPI_Errhandler err_handler; 523 524 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 525 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 526 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 527 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 528 } 529 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 530 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 531 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 532 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 533 if (flg1 || flg2) { 534 PetscMPIInt size; 535 PetscInt lsize,*nodes; 536 MPI_Errhandler err_handler; 537 /* 538 we have to make sure that all processors have opened 539 connections to all other processors, otherwise once the 540 debugger has stated it is likely to receive a SIGUSR1 541 and kill the program. 542 */ 543 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 544 if (size > 2) { 545 PetscMPIInt dummy = 0; 546 MPI_Status status; 547 for (i=0; i<size; i++) { 548 if (rank != i) { 549 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 550 } 551 } 552 for (i=0; i<size; i++) { 553 if (rank != i) { 554 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 555 } 556 } 557 } 558 /* check if this processor node should be in debugger */ 559 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 560 lsize = size; 561 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 562 if (flag) { 563 for (i=0; i<lsize; i++) { 564 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 565 } 566 } 567 if (!flag) { 568 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 569 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 570 if (flg1) { 571 ierr = PetscAttachDebugger();CHKERRQ(ierr); 572 } else { 573 ierr = PetscStopForDebugger();CHKERRQ(ierr); 574 } 575 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 576 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 577 } 578 ierr = PetscFree(nodes);CHKERRQ(ierr); 579 } 580 581 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 582 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 583 584 /* 585 Setup profiling and logging 586 */ 587 #if defined(PETSC_USE_INFO) 588 { 589 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 590 } 591 #endif 592 ierr = PetscDetermineInitialFPTrap(); 593 flg1 = PETSC_FALSE; 594 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 595 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 596 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 597 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 598 #if defined(PETSC_USE_LOG) 599 mname[0] = 0; 600 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 601 if (flg1) { 602 if (mname[0]) { 603 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 604 } else { 605 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 606 } 607 } 608 609 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 610 611 #if defined(PETSC_HAVE_MPE) 612 flg1 = PETSC_FALSE; 613 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 614 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 615 #endif 616 flg1 = PETSC_FALSE; 617 flg3 = PETSC_FALSE; 618 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 619 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 620 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 621 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 622 623 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 624 if (flg1) { 625 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 626 FILE *file; 627 if (mname[0]) { 628 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 629 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 630 file = fopen(fname,"w"); 631 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 632 } else file = PETSC_STDOUT; 633 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 634 } 635 636 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 637 if (flg4) { 638 if (format == PETSC_VIEWER_ASCII_XML) { 639 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 640 } else { 641 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 642 } 643 } 644 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 645 PetscReal threshold = PetscRealConstant(0.01); 646 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 647 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 648 } 649 #endif 650 651 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 652 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 653 /* 654 If collecting logging information, by default, wait for device to complete its operations 655 before returning to the CPU in order to get accurate timings of each event 656 */ 657 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 658 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 659 660 #if defined(PETSC_HAVE_CUDA) 661 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 662 #endif 663 664 #if defined(PETSC_HAVE_HIP) 665 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 666 #endif 667 668 /* 669 Print basic help message 670 */ 671 if (hasHelp) { 672 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 673 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 674 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 675 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 676 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 677 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 678 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 679 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 680 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 681 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 682 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 683 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 684 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 685 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 686 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 687 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 688 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 689 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 690 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 693 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 694 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 707 #if defined(PETSC_USE_LOG) 708 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 712 #if defined(PETSC_HAVE_MPE) 713 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 714 #endif 715 #endif 716 #if defined(PETSC_USE_INFO) 717 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 718 #endif 719 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 723 } 724 725 #if defined(PETSC_HAVE_POPEN) 726 { 727 char machine[128]; 728 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 729 if (flg1) { 730 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 731 } 732 } 733 #endif 734 735 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 736 if (flg1) { 737 ierr = PetscSleep(si);CHKERRQ(ierr); 738 } 739 740 #if defined(PETSC_HAVE_VIENNACL) 741 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 742 if (!flg3) { 743 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 744 } 745 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 746 PetscViennaCLSynchronize = flg3; 747 ierr = PetscViennaCLInit();CHKERRQ(ierr); 748 #endif 749 750 /* 751 Creates the logging data structures; this is enabled even if logging is not turned on 752 This is the last thing we do before returning to the user code to prevent having the 753 logging numbers contaminated by any startup time associated with MPI and the GPUs 754 */ 755 #if defined(PETSC_USE_LOG) 756 ierr = PetscLogInitialize();CHKERRQ(ierr); 757 #endif 758 759 PetscFunctionReturn(0); 760 } 761