1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 63 #if defined(PETSC_HAVE_COMPLEX) 64 #if defined(PETSC_COMPLEX_INSTANTIATE) 65 template <> class std::complex<double>; /* instantiate complex template class */ 66 #endif 67 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 68 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 69 MPI_Datatype MPIU_C_COMPLEX; 70 #endif 71 72 /*MC 73 PETSC_i - the imaginary number i 74 75 Synopsis: 76 #include <petscsys.h> 77 PetscComplex PETSC_i; 78 79 Level: beginner 80 81 Note: 82 Complex numbers are automatically available if PETSc located a working complex implementation 83 84 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 85 M*/ 86 PetscComplex PETSC_i; 87 #endif 88 #if defined(PETSC_USE_REAL___FLOAT128) 89 MPI_Datatype MPIU___FLOAT128 = 0; 90 #if defined(PETSC_HAVE_COMPLEX) 91 MPI_Datatype MPIU___COMPLEX128 = 0; 92 #endif 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 #if defined(PETSC_USE_64BIT_INDICES) 98 MPI_Datatype MPIU_2INT = 0; 99 #endif 100 MPI_Datatype MPIU_BOOL; 101 MPI_Datatype MPIU_ENUM; 102 MPI_Datatype MPIU_FORTRANADDR; 103 MPI_Datatype MPIU_SIZE_T; 104 105 /* 106 Function that is called to display all error messages 107 */ 108 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 109 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 110 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 111 /* 112 This is needed to turn on/off GPU synchronization 113 */ 114 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 115 116 /* ------------------------------------------------------------------------------*/ 117 /* 118 Optional file where all PETSc output from various prints is saved 119 */ 120 PETSC_INTERN FILE *petsc_history; 121 FILE *petsc_history = NULL; 122 123 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 124 { 125 PetscErrorCode ierr; 126 PetscMPIInt rank,size; 127 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 128 char version[256]; 129 130 PetscFunctionBegin; 131 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 132 if (!rank) { 133 char arch[10]; 134 int err; 135 136 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 137 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 138 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 139 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 140 if (filename) { 141 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 142 } else { 143 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 146 } 147 148 *fd = fopen(fname,"a"); 149 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 150 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 153 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 157 err = fflush(*fd); 158 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 159 } 160 PetscFunctionReturn(0); 161 } 162 163 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 164 { 165 PetscErrorCode ierr; 166 PetscMPIInt rank; 167 char date[64]; 168 int err; 169 170 PetscFunctionBegin; 171 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 172 if (!rank) { 173 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 err = fflush(*fd); 178 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 179 err = fclose(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 181 } 182 PetscFunctionReturn(0); 183 } 184 185 /* ------------------------------------------------------------------------------*/ 186 187 /* 188 This is ugly and probably belongs somewhere else, but I want to 189 be able to put a true MPI abort error handler with command line args. 190 191 This is so MPI errors in the debugger will leave all the stack 192 frames. The default MP_Abort() cleans up and exits thus providing no useful information 193 in the debugger hence we call abort() instead of MPI_Abort(). 194 */ 195 196 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 197 { 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 abort(); 201 } 202 203 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 204 { 205 PetscErrorCode ierr; 206 207 PetscFunctionBegin; 208 (*PetscErrorPrintf)("MPI error %d\n",*flag); 209 ierr = PetscAttachDebugger(); 210 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 211 } 212 213 /*@C 214 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 215 wishes a clean exit somewhere deep in the program. 216 217 Collective on PETSC_COMM_WORLD 218 219 Options Database Keys are the same as for PetscFinalize() 220 221 Level: advanced 222 223 Note: 224 See PetscInitialize() for more general runtime options. 225 226 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 227 @*/ 228 PetscErrorCode PetscEnd(void) 229 { 230 PetscFunctionBegin; 231 PetscFinalize(); 232 exit(0); 233 return 0; 234 } 235 236 PetscBool PetscOptionsPublish = PETSC_FALSE; 237 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 238 PETSC_INTERN PetscBool petscsetmallocvisited; 239 static char emacsmachinename[256]; 240 241 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 242 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 243 244 /*@C 245 PetscSetHelpVersionFunctions - Sets functions that print help and version information 246 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 247 This routine enables a "higher-level" package that uses PETSc to print its messages first. 248 249 Input Parameter: 250 + help - the help function (may be NULL) 251 - version - the version function (may be NULL) 252 253 Level: developer 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #if defined(PETSC_USE_LOG) 265 PETSC_INTERN PetscBool PetscObjectsLog; 266 #endif 267 268 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 269 { 270 if (PetscUnlikely(*err)) { 271 PetscMPIInt len; 272 char errstring[MPI_MAX_ERROR_STRING]; 273 274 MPI_Error_string(*err,errstring,&len); 275 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 276 } 277 return; 278 } 279 280 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 281 Use the following macros to define CUDA/HIP initialization related vars/routines. 282 */ 283 #if defined(PETSC_HAVE_CUDA) 284 typedef cudaError_t cupmError_t; 285 typedef struct cudaDeviceProp cupmDeviceProp; 286 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 287 #define cupmGetDevice(x) cudaGetDevice(x) 288 #define cupmSetDevice(x) cudaSetDevice(x) 289 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 290 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 291 #define cupmGetLastError() cudaGetLastError() 292 #define cupmDeviceMapHost cudaDeviceMapHost 293 #define cupmSuccess cudaSuccess 294 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 295 #define CHKERRCUPM(x) CHKERRCUDA(x) 296 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 297 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 298 #define PetscCUPMInitialize PetscCUDAInitialize 299 #define PetscCUPMInitialized PetscCUDAInitialized 300 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 301 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 302 #define PetscCUPMSynchronize PetscCUDASynchronize 303 #define PetscNotUseCUPM PetscNotUseCUDA 304 #define cupmOptionsStr "CUDA options" 305 #define cupmSetDeviceStr "-cuda_device" 306 #define cupmViewStr "-cuda_view" 307 #define cupmSynchronizeStr "-cuda_synchronize" 308 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 309 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 310 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 311 #include "cupminit.inc" 312 #endif 313 314 #if defined(PETSC_HAVE_HIP) 315 typedef hipError_t cupmError_t; 316 typedef hipDeviceProp_t cupmDeviceProp; 317 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 318 #define cupmGetDevice(x) hipGetDevice(x) 319 #define cupmSetDevice(x) hipSetDevice(x) 320 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 321 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 322 #define cupmGetLastError() hipGetLastError() 323 #define cupmDeviceMapHost hipDeviceMapHost 324 #define cupmSuccess hipSuccess 325 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 326 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 327 #define PetscCUPMBLASInitializeHandle() 0 328 #define PetscCUPMSOLVERDnInitializeHandle() 0 329 #define PetscCUPMInitialize PetscHIPInitialize 330 #define PetscCUPMInitialized PetscHIPInitialized 331 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 332 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 333 #define PetscCUPMSynchronize PetscHIPSynchronize 334 #define PetscNotUseCUPM PetscNotUseHIP 335 #define cupmOptionsStr "HIP options" 336 #define cupmSetDeviceStr "-hip_device" 337 #define cupmViewStr "-hip_view" 338 #define cupmSynchronizeStr "-hip_synchronize" 339 #define PetscCUPMInitializeStr "PetscHIPInitialize" 340 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 341 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 342 #include "cupminit.inc" 343 #endif 344 345 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 346 { 347 char string[64]; 348 MPI_Comm comm = PETSC_COMM_WORLD; 349 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 350 PetscErrorCode ierr; 351 PetscReal si; 352 PetscInt intensity; 353 int i; 354 PetscMPIInt rank; 355 char version[256]; 356 #if defined(PETSC_USE_LOG) 357 char mname[PETSC_MAX_PATH_LEN]; 358 PetscViewerFormat format; 359 PetscBool flg4 = PETSC_FALSE; 360 #endif 361 362 PetscFunctionBegin; 363 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 364 365 #if !defined(PETSC_HAVE_THREADSAFETY) 366 if (!(PETSC_RUNNING_ON_VALGRIND)) { 367 /* 368 Setup the memory management; support for tracing malloc() usage 369 */ 370 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 371 372 if (PetscDefined(USE_DEBUG)) { 373 mdebug = PETSC_TRUE; 374 initializenan = PETSC_TRUE; 375 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 376 } else { 377 /* don't warn about unused option */ 378 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 379 flg1 = PETSC_FALSE; 380 } 381 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 382 if (flg1 || flg2) { 383 mdebug = PETSC_TRUE; 384 eachcall = PETSC_TRUE; 385 initializenan = PETSC_TRUE; 386 } else if (flg3 && !flg2) { 387 mdebug = PETSC_FALSE; 388 eachcall = PETSC_FALSE; 389 initializenan = PETSC_FALSE; 390 } 391 392 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 393 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 394 395 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 396 if (mlog) { 397 mdebug = PETSC_TRUE; 398 } 399 /* the next line is deprecated */ 400 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 401 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 402 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 403 if (mdebug) { 404 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 405 } 406 if (mlog) { 407 PetscReal logthreshold = 0; 408 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 409 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 410 } 411 #if defined(PETSC_USE_LOG) 412 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 413 #endif 414 } 415 416 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 417 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 418 flg1 = PETSC_FALSE; 419 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 420 /* ignore this option if malloc is already set */ 421 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 422 423 flg1 = PETSC_FALSE; 424 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 425 if (!flg1) { 426 flg1 = PETSC_FALSE; 427 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 428 } 429 if (flg1) { 430 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 431 } 432 #endif 433 434 #if defined(PETSC_USE_LOG) 435 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 436 #endif 437 438 /* 439 Set the display variable for graphics 440 */ 441 ierr = PetscSetDisplay();CHKERRQ(ierr); 442 443 /* 444 Print main application help message 445 */ 446 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 447 if (help && hasHelp) { 448 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 449 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 450 } 451 452 /* 453 Print the PETSc version information 454 */ 455 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 456 if (flg1 || hasHelp) { 457 /* 458 Print "higher-level" package version message 459 */ 460 if (PetscExternalVersionFunction) { 461 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 462 } 463 464 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 465 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 472 } 473 474 /* 475 Print "higher-level" package help message 476 */ 477 if (hasHelp) { 478 PetscBool hasHelpIntro; 479 480 if (PetscExternalHelpFunction) { 481 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 482 } 483 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 484 if (hasHelpIntro) { 485 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 486 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 487 ierr = MPI_Finalize();CHKERRQ(ierr); 488 exit(0); 489 } 490 } 491 492 /* 493 Setup the error handling 494 */ 495 flg1 = PETSC_FALSE; 496 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 497 if (flg1) { 498 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 499 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 500 } 501 flg1 = PETSC_FALSE; 502 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 503 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 504 flg1 = PETSC_FALSE; 505 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 506 if (flg1) { 507 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 508 } 509 /* experimental */ 510 flg1 = PETSC_FALSE; 511 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 512 if (flg1) { 513 MPI_Errhandler eh; 514 515 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 516 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 517 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 518 } 519 flg1 = PETSC_FALSE; 520 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 521 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 522 523 /* 524 Setup debugger information 525 */ 526 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 527 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 528 if (flg1) { 529 MPI_Errhandler err_handler; 530 531 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 532 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 533 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 534 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 535 } 536 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 537 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 538 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 539 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 540 if (flg1 || flg2) { 541 PetscMPIInt size; 542 PetscInt lsize,*nodes; 543 MPI_Errhandler err_handler; 544 /* 545 we have to make sure that all processors have opened 546 connections to all other processors, otherwise once the 547 debugger has stated it is likely to receive a SIGUSR1 548 and kill the program. 549 */ 550 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 551 if (size > 2) { 552 PetscMPIInt dummy = 0; 553 MPI_Status status; 554 for (i=0; i<size; i++) { 555 if (rank != i) { 556 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 557 } 558 } 559 for (i=0; i<size; i++) { 560 if (rank != i) { 561 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 562 } 563 } 564 } 565 /* check if this processor node should be in debugger */ 566 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 567 lsize = size; 568 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 569 if (flag) { 570 for (i=0; i<lsize; i++) { 571 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 572 } 573 } 574 if (!flag) { 575 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 576 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 577 if (flg1) { 578 ierr = PetscAttachDebugger();CHKERRQ(ierr); 579 } else { 580 ierr = PetscStopForDebugger();CHKERRQ(ierr); 581 } 582 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 583 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 584 } 585 ierr = PetscFree(nodes);CHKERRQ(ierr); 586 } 587 588 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 589 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 590 591 /* 592 Setup profiling and logging 593 */ 594 #if defined(PETSC_USE_INFO) 595 { 596 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 597 } 598 #endif 599 ierr = PetscDetermineInitialFPTrap(); 600 flg1 = PETSC_FALSE; 601 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 602 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 603 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 604 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 605 #if defined(PETSC_USE_LOG) 606 mname[0] = 0; 607 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 608 if (flg1) { 609 if (mname[0]) { 610 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 611 } else { 612 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 613 } 614 } 615 616 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 617 618 #if defined(PETSC_HAVE_MPE) 619 flg1 = PETSC_FALSE; 620 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 621 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 622 #endif 623 flg1 = PETSC_FALSE; 624 flg3 = PETSC_FALSE; 625 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 626 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 627 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 628 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 629 630 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 631 if (flg1) { 632 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 633 FILE *file; 634 if (mname[0]) { 635 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 636 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 637 file = fopen(fname,"w"); 638 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 639 } else file = PETSC_STDOUT; 640 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 641 } 642 643 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 644 if (flg4) { 645 if (format == PETSC_VIEWER_ASCII_XML) { 646 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 647 } else { 648 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 649 } 650 } 651 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 652 PetscReal threshold = PetscRealConstant(0.01); 653 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 654 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 655 } 656 #endif 657 658 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 659 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 660 /* 661 If collecting logging information, by default, wait for device to complete its operations 662 before returning to the CPU in order to get accurate timings of each event 663 */ 664 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 665 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 666 667 #if defined(PETSC_HAVE_CUDA) 668 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 669 #endif 670 671 #if defined(PETSC_HAVE_HIP) 672 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 673 #endif 674 675 /* 676 Print basic help message 677 */ 678 if (hasHelp) { 679 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 680 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 681 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 682 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 683 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 684 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 685 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 686 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 687 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 688 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 689 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 690 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 693 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 694 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 714 #if defined(PETSC_USE_LOG) 715 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 719 #if defined(PETSC_HAVE_MPE) 720 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 721 #endif 722 #endif 723 #if defined(PETSC_USE_INFO) 724 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 725 #endif 726 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 730 } 731 732 #if defined(PETSC_HAVE_POPEN) 733 { 734 char machine[128]; 735 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 736 if (flg1) { 737 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 738 } 739 } 740 #endif 741 742 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 743 if (flg1) { 744 ierr = PetscSleep(si);CHKERRQ(ierr); 745 } 746 747 #if defined(PETSC_HAVE_VIENNACL) 748 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 749 if (!flg3) { 750 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 751 } 752 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 753 PetscViennaCLSynchronize = flg3; 754 ierr = PetscViennaCLInit();CHKERRQ(ierr); 755 #endif 756 757 /* 758 Creates the logging data structures; this is enabled even if logging is not turned on 759 This is the last thing we do before returning to the user code to prevent having the 760 logging numbers contaminated by any startup time associated with MPI and the GPUs 761 */ 762 #if defined(PETSC_USE_LOG) 763 ierr = PetscLogInitialize();CHKERRQ(ierr); 764 #endif 765 766 PetscFunctionReturn(0); 767 } 768