1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <cuda_runtime_api.h> 27 #include <cuda_device_runtime_api.h> 28 #include <petsccublas.h> 29 #endif 30 31 #if defined(PETSC_HAVE_HIP) 32 #include <hip/hip_runtime.h> 33 #endif 34 35 #if defined(PETSC_HAVE_DEVICE) 36 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 37 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 38 #endif 39 #endif 40 41 #if defined(PETSC_HAVE_VIENNACL) 42 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 43 #endif 44 45 /* ------------------------Nasty global variables -------------------------------*/ 46 /* 47 Indicates if PETSc started up MPI, or it was 48 already started before PETSc was initialized. 49 */ 50 PetscBool PetscBeganMPI = PETSC_FALSE; 51 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 52 PetscBool PetscInitializeCalled = PETSC_FALSE; 53 PetscBool PetscFinalizeCalled = PETSC_FALSE; 54 55 PetscMPIInt PetscGlobalRank = -1; 56 PetscMPIInt PetscGlobalSize = -1; 57 58 #if defined(PETSC_HAVE_KOKKOS) 59 PetscBool PetscBeganKokkos = PETSC_FALSE; 60 #endif 61 62 #if defined(PETSC_HAVE_NVSHMEM) 63 PetscBool PetscBeganNvshmem = PETSC_FALSE; 64 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 65 #endif 66 67 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 68 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 69 70 #if defined(PETSC_HAVE_COMPLEX) 71 #if defined(PETSC_COMPLEX_INSTANTIATE) 72 template <> class std::complex<double>; /* instantiate complex template class */ 73 #endif 74 75 /*MC 76 PETSC_i - the imaginary number i 77 78 Synopsis: 79 #include <petscsys.h> 80 PetscComplex PETSC_i; 81 82 Level: beginner 83 84 Note: 85 Complex numbers are automatically available if PETSc located a working complex implementation 86 87 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 88 M*/ 89 PetscComplex PETSC_i; 90 MPI_Datatype MPIU___COMPLEX128 = 0; 91 #endif /* PETSC_HAVE_COMPLEX */ 92 #if defined(PETSC_USE_REAL___FLOAT128) 93 MPI_Datatype MPIU___FLOAT128 = 0; 94 #elif defined(PETSC_USE_REAL___FP16) 95 MPI_Datatype MPIU___FP16 = 0; 96 #endif 97 MPI_Datatype MPIU_2SCALAR = 0; 98 #if defined(PETSC_USE_64BIT_INDICES) 99 MPI_Datatype MPIU_2INT = 0; 100 #endif 101 MPI_Datatype MPIU_BOOL; 102 MPI_Datatype MPIU_ENUM; 103 MPI_Datatype MPIU_FORTRANADDR; 104 MPI_Datatype MPIU_SIZE_T; 105 106 /* 107 Function that is called to display all error messages 108 */ 109 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 110 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 111 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 112 /* 113 This is needed to turn on/off GPU synchronization 114 */ 115 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 116 117 /* ------------------------------------------------------------------------------*/ 118 /* 119 Optional file where all PETSc output from various prints is saved 120 */ 121 PETSC_INTERN FILE *petsc_history; 122 FILE *petsc_history = NULL; 123 124 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 125 { 126 PetscErrorCode ierr; 127 PetscMPIInt rank,size; 128 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 129 char version[256]; 130 131 PetscFunctionBegin; 132 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 133 if (!rank) { 134 char arch[10]; 135 int err; 136 137 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 138 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 139 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 140 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 141 if (filename) { 142 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 143 } else { 144 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 146 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 147 } 148 149 *fd = fopen(fname,"a"); 150 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 151 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 153 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 154 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 156 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 157 158 err = fflush(*fd); 159 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 160 } 161 PetscFunctionReturn(0); 162 } 163 164 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 165 { 166 PetscErrorCode ierr; 167 PetscMPIInt rank; 168 char date[64]; 169 int err; 170 171 PetscFunctionBegin; 172 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 173 if (!rank) { 174 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 177 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 178 err = fflush(*fd); 179 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 180 err = fclose(*fd); 181 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 182 } 183 PetscFunctionReturn(0); 184 } 185 186 /* ------------------------------------------------------------------------------*/ 187 188 /* 189 This is ugly and probably belongs somewhere else, but I want to 190 be able to put a true MPI abort error handler with command line args. 191 192 This is so MPI errors in the debugger will leave all the stack 193 frames. The default MP_Abort() cleans up and exits thus providing no useful information 194 in the debugger hence we call abort() instead of MPI_Abort(). 195 */ 196 197 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 198 { 199 PetscFunctionBegin; 200 (*PetscErrorPrintf)("MPI error %d\n",*flag); 201 abort(); 202 } 203 204 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 205 { 206 PetscErrorCode ierr; 207 208 PetscFunctionBegin; 209 (*PetscErrorPrintf)("MPI error %d\n",*flag); 210 ierr = PetscAttachDebugger(); 211 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 212 } 213 214 /*@C 215 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 216 wishes a clean exit somewhere deep in the program. 217 218 Collective on PETSC_COMM_WORLD 219 220 Options Database Keys are the same as for PetscFinalize() 221 222 Level: advanced 223 224 Note: 225 See PetscInitialize() for more general runtime options. 226 227 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 228 @*/ 229 PetscErrorCode PetscEnd(void) 230 { 231 PetscFunctionBegin; 232 PetscFinalize(); 233 exit(0); 234 return 0; 235 } 236 237 PetscBool PetscOptionsPublish = PETSC_FALSE; 238 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 239 PETSC_INTERN PetscBool petscsetmallocvisited; 240 static char emacsmachinename[256]; 241 242 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 243 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 244 245 /*@C 246 PetscSetHelpVersionFunctions - Sets functions that print help and version information 247 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 248 This routine enables a "higher-level" package that uses PETSc to print its messages first. 249 250 Input Parameter: 251 + help - the help function (may be NULL) 252 - version - the version function (may be NULL) 253 254 Level: developer 255 256 @*/ 257 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 258 { 259 PetscFunctionBegin; 260 PetscExternalHelpFunction = help; 261 PetscExternalVersionFunction = version; 262 PetscFunctionReturn(0); 263 } 264 265 #if defined(PETSC_USE_LOG) 266 PETSC_INTERN PetscBool PetscObjectsLog; 267 #endif 268 269 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 270 Use the following macros to define CUDA/HIP initialization related vars/routines. 271 */ 272 #if defined(PETSC_HAVE_CUDA) 273 typedef cudaError_t cupmError_t; 274 typedef struct cudaDeviceProp cupmDeviceProp; 275 typedef cudaStream_t cupmStream_t; 276 typedef cudaEvent_t cupmEvent_t; 277 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 278 #define cupmGetDevice(x) cudaGetDevice(x) 279 #define cupmSetDevice(x) cudaSetDevice(x) 280 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 281 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 282 #define cupmStreamCreate(x) cudaStreamCreate(x) 283 #define cupmGetLastError() cudaGetLastError() 284 #define cupmDeviceMapHost cudaDeviceMapHost 285 #define cupmSuccess cudaSuccess 286 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 287 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 288 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 289 #define CHKERRCUPM(x) CHKERRCUDA(x) 290 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 291 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 292 #define PetscCUPMInitialize PetscCUDAInitialize 293 #define PetscCUPMInitialized PetscCUDAInitialized 294 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 295 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 296 #define PetscCUPMSynchronize PetscCUDASynchronize 297 #define PetscNotUseCUPM PetscNotUseCUDA 298 #define cupmOptionsStr "CUDA options" 299 #define cupmSetDeviceStr "-cuda_device" 300 #define cupmViewStr "-cuda_view" 301 #define cupmSynchronizeStr "-cuda_synchronize" 302 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 303 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 304 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 305 #define PetscDefaultCupmStream PetscDefaultCudaStream 306 #define cupmEventCreate(x) cudaEventCreate(x) 307 #include "cupminit.inc" 308 #endif 309 310 #if defined(PETSC_HAVE_HIP) 311 typedef hipError_t cupmError_t; 312 typedef hipDeviceProp_t cupmDeviceProp; 313 typedef hipStream_t cupmStream_t; 314 typedef hipEvent_t cupmEvent_t; 315 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 316 #define cupmGetDevice(x) hipGetDevice(x) 317 #define cupmSetDevice(x) hipSetDevice(x) 318 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 319 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 320 #define cupmStreamCreate(x) hipStreamCreate(x) 321 #define cupmEventCreate(x) hipEventCreate(x); 322 #define cupmGetLastError() hipGetLastError() 323 #define cupmDeviceMapHost hipDeviceMapHost 324 #define cupmSuccess hipSuccess 325 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 326 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 327 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 328 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 329 #define PetscCUPMBLASInitializeHandle() 0 330 #define PetscCUPMSOLVERDnInitializeHandle() 0 331 #define PetscCUPMInitialize PetscHIPInitialize 332 #define PetscCUPMInitialized PetscHIPInitialized 333 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 334 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 335 #define PetscCUPMSynchronize PetscHIPSynchronize 336 #define PetscNotUseCUPM PetscNotUseHIP 337 #define cupmOptionsStr "HIP options" 338 #define cupmSetDeviceStr "-hip_device" 339 #define cupmViewStr "-hip_view" 340 #define cupmSynchronizeStr "-hip_synchronize" 341 #define PetscCUPMInitializeStr "PetscHIPInitialize" 342 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 343 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 344 #define PetscDefaultCupmStream PetscDefaultHipStream 345 #include "cupminit.inc" 346 #endif 347 348 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 349 { 350 char string[64]; 351 MPI_Comm comm = PETSC_COMM_WORLD; 352 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 353 PetscErrorCode ierr; 354 PetscReal si; 355 PetscInt intensity; 356 int i; 357 PetscMPIInt rank; 358 char version[256]; 359 #if defined(PETSC_USE_LOG) 360 char mname[PETSC_MAX_PATH_LEN]; 361 PetscViewerFormat format; 362 PetscBool flg4 = PETSC_FALSE; 363 #endif 364 365 PetscFunctionBegin; 366 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 367 368 #if !defined(PETSC_HAVE_THREADSAFETY) 369 if (!(PETSC_RUNNING_ON_VALGRIND)) { 370 /* 371 Setup the memory management; support for tracing malloc() usage 372 */ 373 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 374 375 if (PetscDefined(USE_DEBUG)) { 376 mdebug = PETSC_TRUE; 377 initializenan = PETSC_TRUE; 378 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 379 } else { 380 /* don't warn about unused option */ 381 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 382 flg1 = PETSC_FALSE; 383 } 384 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 385 if (flg1 || flg2) { 386 mdebug = PETSC_TRUE; 387 eachcall = PETSC_TRUE; 388 initializenan = PETSC_TRUE; 389 } else if (flg3 && !flg2) { 390 mdebug = PETSC_FALSE; 391 eachcall = PETSC_FALSE; 392 initializenan = PETSC_FALSE; 393 } 394 395 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 396 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 397 398 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 399 if (mlog) { 400 mdebug = PETSC_TRUE; 401 } 402 /* the next line is deprecated */ 403 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 404 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 405 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 406 if (mdebug) { 407 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 408 } 409 if (mlog) { 410 PetscReal logthreshold = 0; 411 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 412 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 413 } 414 #if defined(PETSC_USE_LOG) 415 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 416 #endif 417 } 418 419 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 420 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 421 flg1 = PETSC_FALSE; 422 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 423 /* ignore this option if malloc is already set */ 424 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 425 426 flg1 = PETSC_FALSE; 427 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 428 if (!flg1) { 429 flg1 = PETSC_FALSE; 430 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 431 } 432 if (flg1) { 433 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 434 } 435 #endif 436 437 #if defined(PETSC_USE_LOG) 438 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 439 #endif 440 441 /* 442 Set the display variable for graphics 443 */ 444 ierr = PetscSetDisplay();CHKERRQ(ierr); 445 446 /* 447 Print main application help message 448 */ 449 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 450 if (help && hasHelp) { 451 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 452 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 453 } 454 455 /* 456 Print the PETSc version information 457 */ 458 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 459 if (flg1 || hasHelp) { 460 /* 461 Print "higher-level" package version message 462 */ 463 if (PetscExternalVersionFunction) { 464 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 465 } 466 467 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 475 } 476 477 /* 478 Print "higher-level" package help message 479 */ 480 if (hasHelp) { 481 PetscBool hasHelpIntro; 482 483 if (PetscExternalHelpFunction) { 484 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 485 } 486 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 487 if (hasHelpIntro) { 488 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 489 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 490 ierr = MPI_Finalize();CHKERRMPI(ierr); 491 exit(0); 492 } 493 } 494 495 /* 496 Setup the error handling 497 */ 498 flg1 = PETSC_FALSE; 499 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 500 if (flg1) { 501 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 502 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 503 } 504 flg1 = PETSC_FALSE; 505 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 506 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 507 flg1 = PETSC_FALSE; 508 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 509 if (flg1) { 510 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 511 } 512 flg1 = PETSC_FALSE; 513 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 514 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 515 516 /* 517 Setup debugger information 518 */ 519 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 520 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 521 if (flg1) { 522 MPI_Errhandler err_handler; 523 524 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 525 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 526 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 527 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 528 } 529 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 530 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 531 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 532 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 533 if (flg1 || flg2) { 534 PetscMPIInt size; 535 PetscInt lsize,*ranks; 536 MPI_Errhandler err_handler; 537 /* 538 we have to make sure that all processors have opened 539 connections to all other processors, otherwise once the 540 debugger has stated it is likely to receive a SIGUSR1 541 and kill the program. 542 */ 543 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 544 if (size > 2) { 545 PetscMPIInt dummy = 0; 546 MPI_Status status; 547 for (i=0; i<size; i++) { 548 if (rank != i) { 549 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 550 } 551 } 552 for (i=0; i<size; i++) { 553 if (rank != i) { 554 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 555 } 556 } 557 } 558 /* check if this processor node should be in debugger */ 559 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 560 lsize = size; 561 /* Deprecated in 3.14 */ 562 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 563 if (flag) { 564 const char * const quietopt="-options_suppress_deprecated_warnings"; 565 char msg[4096]; 566 PetscBool quiet = PETSC_FALSE; 567 568 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 569 if (!quiet) { 570 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 571 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 572 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 573 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 576 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 577 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 579 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 580 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 581 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 582 } 583 } else { 584 lsize = size; 585 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 586 } 587 if (flag) { 588 for (i=0; i<lsize; i++) { 589 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 590 } 591 } 592 if (!flag) { 593 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 594 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 595 if (flg1) { 596 ierr = PetscAttachDebugger();CHKERRQ(ierr); 597 } else { 598 ierr = PetscStopForDebugger();CHKERRQ(ierr); 599 } 600 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 601 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 602 } else { 603 ierr = PetscWaitOnError();CHKERRQ(ierr); 604 } 605 ierr = PetscFree(ranks);CHKERRQ(ierr); 606 } 607 608 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 609 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 610 611 /* 612 Setup profiling and logging 613 */ 614 #if defined(PETSC_USE_INFO) 615 { 616 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 617 } 618 #endif 619 ierr = PetscDetermineInitialFPTrap(); 620 flg1 = PETSC_FALSE; 621 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 622 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 623 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 624 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 625 #if defined(PETSC_USE_LOG) 626 mname[0] = 0; 627 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 628 if (flg1) { 629 if (mname[0]) { 630 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 631 } else { 632 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 633 } 634 } 635 636 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 637 638 #if defined(PETSC_HAVE_MPE) 639 flg1 = PETSC_FALSE; 640 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 641 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 642 #endif 643 flg1 = PETSC_FALSE; 644 flg3 = PETSC_FALSE; 645 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 646 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 647 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 648 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 649 650 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 651 if (flg1) { 652 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 653 FILE *file; 654 if (mname[0]) { 655 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 656 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 657 file = fopen(fname,"w"); 658 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 659 } else file = PETSC_STDOUT; 660 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 661 } 662 663 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 664 if (flg4) { 665 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 666 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 667 } else { 668 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 669 } 670 } 671 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 672 PetscReal threshold = PetscRealConstant(0.01); 673 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 674 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 675 } 676 #endif 677 678 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 679 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 680 /* 681 If collecting logging information, by default, wait for device to complete its operations 682 before returning to the CPU in order to get accurate timings of each event 683 */ 684 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 685 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 686 687 #if defined(PETSC_HAVE_CUDA) 688 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 689 #endif 690 691 #if defined(PETSC_HAVE_HIP) 692 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 693 #endif 694 695 /* 696 Print basic help message 697 */ 698 if (hasHelp) { 699 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 734 #if defined(PETSC_USE_LOG) 735 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 739 #if defined(PETSC_HAVE_MPE) 740 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 741 #endif 742 #endif 743 #if defined(PETSC_USE_INFO) 744 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 745 #endif 746 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 747 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 749 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 750 } 751 752 #if defined(PETSC_HAVE_POPEN) 753 { 754 char machine[128]; 755 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 756 if (flg1) { 757 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 758 } 759 } 760 #endif 761 762 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 763 if (flg1) { 764 ierr = PetscSleep(si);CHKERRQ(ierr); 765 } 766 767 #if defined(PETSC_HAVE_VIENNACL) 768 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 769 if (!flg3) { 770 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 771 } 772 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 773 PetscViennaCLSynchronize = flg3; 774 ierr = PetscViennaCLInit();CHKERRQ(ierr); 775 #endif 776 777 /* 778 Creates the logging data structures; this is enabled even if logging is not turned on 779 This is the last thing we do before returning to the user code to prevent having the 780 logging numbers contaminated by any startup time associated with MPI and the GPUs 781 */ 782 #if defined(PETSC_USE_LOG) 783 ierr = PetscLogInitialize();CHKERRQ(ierr); 784 #endif 785 786 PetscFunctionReturn(0); 787 } 788