1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscviewer.h> 12 #if defined(PETSC_USE_LOG) 13 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 14 #endif 15 16 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 17 #include <sys/sysinfo.h> 18 #endif 19 #if defined(PETSC_HAVE_UNISTD_H) 20 #include <unistd.h> 21 #endif 22 23 #if defined(PETSC_HAVE_CUDA) || defined(PETSC_HAVE_HIP) 24 # include <petscdevice.h> 25 #endif 26 27 #if defined(PETSC_HAVE_DEVICE) 28 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 29 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 30 #endif 31 #endif 32 33 #if defined(PETSC_HAVE_VIENNACL) 34 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 35 #endif 36 37 /* ------------------------Nasty global variables -------------------------------*/ 38 /* 39 Indicates if PETSc started up MPI, or it was 40 already started before PETSc was initialized. 41 */ 42 PetscBool PetscBeganMPI = PETSC_FALSE; 43 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 44 PetscBool PetscInitializeCalled = PETSC_FALSE; 45 PetscBool PetscFinalizeCalled = PETSC_FALSE; 46 47 PetscMPIInt PetscGlobalRank = -1; 48 PetscMPIInt PetscGlobalSize = -1; 49 50 #if defined(PETSC_HAVE_KOKKOS) 51 PetscBool PetscBeganKokkos = PETSC_FALSE; 52 #endif 53 54 #if defined(PETSC_HAVE_NVSHMEM) 55 PetscBool PetscBeganNvshmem = PETSC_FALSE; 56 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 57 #endif 58 59 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 60 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 61 62 #if defined(PETSC_HAVE_COMPLEX) 63 #if defined(PETSC_COMPLEX_INSTANTIATE) 64 template <> class std::complex<double>; /* instantiate complex template class */ 65 #endif 66 67 /*MC 68 PETSC_i - the imaginary number i 69 70 Synopsis: 71 #include <petscsys.h> 72 PetscComplex PETSC_i; 73 74 Level: beginner 75 76 Note: 77 Complex numbers are automatically available if PETSc located a working complex implementation 78 79 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 80 M*/ 81 PetscComplex PETSC_i; 82 MPI_Datatype MPIU___COMPLEX128 = 0; 83 #endif /* PETSC_HAVE_COMPLEX */ 84 #if defined(PETSC_USE_REAL___FLOAT128) 85 MPI_Datatype MPIU___FLOAT128 = 0; 86 #elif defined(PETSC_USE_REAL___FP16) 87 MPI_Datatype MPIU___FP16 = 0; 88 #endif 89 MPI_Datatype MPIU_2SCALAR = 0; 90 MPI_Datatype MPIU_REAL_INT = 0; 91 MPI_Datatype MPIU_SCALAR_INT = 0; 92 #if defined(PETSC_USE_64BIT_INDICES) 93 MPI_Datatype MPIU_2INT = 0; 94 #endif 95 MPI_Datatype MPI_4INT = 0; 96 MPI_Datatype MPIU_4INT = 0; 97 MPI_Datatype MPIU_BOOL; 98 MPI_Datatype MPIU_ENUM; 99 MPI_Datatype MPIU_FORTRANADDR; 100 MPI_Datatype MPIU_SIZE_T; 101 102 /* 103 Function that is called to display all error messages 104 */ 105 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 106 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 107 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 108 /* 109 This is needed to turn on/off GPU synchronization 110 */ 111 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 112 113 /* ------------------------------------------------------------------------------*/ 114 /* 115 Optional file where all PETSc output from various prints is saved 116 */ 117 PETSC_INTERN FILE *petsc_history; 118 FILE *petsc_history = NULL; 119 120 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 121 { 122 PetscErrorCode ierr; 123 PetscMPIInt rank,size; 124 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 125 char version[256]; 126 127 PetscFunctionBegin; 128 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 129 if (!rank) { 130 char arch[10]; 131 int err; 132 133 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 134 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 135 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 136 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 137 if (filename) { 138 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 139 } else { 140 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 141 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 142 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 143 } 144 145 *fd = fopen(fname,"a"); 146 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 147 148 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 149 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 150 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 153 154 err = fflush(*fd); 155 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 156 } 157 PetscFunctionReturn(0); 158 } 159 160 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 161 { 162 PetscErrorCode ierr; 163 PetscMPIInt rank; 164 char date[64]; 165 int err; 166 167 PetscFunctionBegin; 168 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 169 if (!rank) { 170 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 171 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 172 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 173 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 174 err = fflush(*fd); 175 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 176 err = fclose(*fd); 177 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 178 } 179 PetscFunctionReturn(0); 180 } 181 182 /* ------------------------------------------------------------------------------*/ 183 184 /* 185 This is ugly and probably belongs somewhere else, but I want to 186 be able to put a true MPI abort error handler with command line args. 187 188 This is so MPI errors in the debugger will leave all the stack 189 frames. The default MP_Abort() cleans up and exits thus providing no useful information 190 in the debugger hence we call abort() instead of MPI_Abort(). 191 */ 192 193 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 194 { 195 PetscFunctionBegin; 196 (*PetscErrorPrintf)("MPI error %d\n",*flag); 197 abort(); 198 } 199 200 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 201 { 202 PetscErrorCode ierr; 203 204 PetscFunctionBegin; 205 (*PetscErrorPrintf)("MPI error %d\n",*flag); 206 ierr = PetscAttachDebugger(); 207 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 208 } 209 210 /*@C 211 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 212 wishes a clean exit somewhere deep in the program. 213 214 Collective on PETSC_COMM_WORLD 215 216 Options Database Keys are the same as for PetscFinalize() 217 218 Level: advanced 219 220 Note: 221 See PetscInitialize() for more general runtime options. 222 223 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 224 @*/ 225 PetscErrorCode PetscEnd(void) 226 { 227 PetscFunctionBegin; 228 PetscFinalize(); 229 exit(0); 230 return 0; 231 } 232 233 PetscBool PetscOptionsPublish = PETSC_FALSE; 234 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 235 PETSC_INTERN PetscBool petscsetmallocvisited; 236 static char emacsmachinename[256]; 237 238 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 239 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 240 241 /*@C 242 PetscSetHelpVersionFunctions - Sets functions that print help and version information 243 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 244 This routine enables a "higher-level" package that uses PETSc to print its messages first. 245 246 Input Parameter: 247 + help - the help function (may be NULL) 248 - version - the version function (may be NULL) 249 250 Level: developer 251 252 @*/ 253 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 254 { 255 PetscFunctionBegin; 256 PetscExternalHelpFunction = help; 257 PetscExternalVersionFunction = version; 258 PetscFunctionReturn(0); 259 } 260 261 #if defined(PETSC_USE_LOG) 262 PETSC_INTERN PetscBool PetscObjectsLog; 263 #endif 264 265 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 266 Use the following macros to define CUDA/HIP initialization related vars/routines. 267 */ 268 #if defined(PETSC_HAVE_CUDA) 269 typedef cudaError_t cupmError_t; 270 typedef struct cudaDeviceProp cupmDeviceProp; 271 typedef cudaStream_t cupmStream_t; 272 typedef cudaEvent_t cupmEvent_t; 273 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 274 #define cupmGetDevice(x) cudaGetDevice(x) 275 #define cupmSetDevice(x) cudaSetDevice(x) 276 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 277 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 278 #define cupmStreamCreate(x) cudaStreamCreate(x) 279 #define cupmGetLastError() cudaGetLastError() 280 #define cupmDeviceMapHost cudaDeviceMapHost 281 #define cupmSuccess cudaSuccess 282 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 283 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 284 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 285 #define CHKERRCUPM(x) CHKERRCUDA(x) 286 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 287 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 288 #define PetscCUPMInitialize PetscCUDAInitialize 289 #define PetscCUPMInitialized PetscCUDAInitialized 290 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 291 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 292 #define PetscCUPMSynchronize PetscCUDASynchronize 293 #define PetscNotUseCUPM PetscNotUseCUDA 294 #define cupmOptionsStr "CUDA options" 295 #define cupmSetDeviceStr "-cuda_device" 296 #define cupmViewStr "-cuda_view" 297 #define cupmSynchronizeStr "-cuda_synchronize" 298 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 299 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 300 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 301 #define PetscDefaultCupmStream PetscDefaultCudaStream 302 #define cupmEventCreate(x) cudaEventCreate(x) 303 #include "cupminit.inc" 304 #endif 305 306 #if defined(PETSC_HAVE_HIP) 307 typedef hipError_t cupmError_t; 308 typedef hipDeviceProp_t cupmDeviceProp; 309 typedef hipStream_t cupmStream_t; 310 typedef hipEvent_t cupmEvent_t; 311 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 312 #define cupmGetDevice(x) hipGetDevice(x) 313 #define cupmSetDevice(x) hipSetDevice(x) 314 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 315 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 316 #define cupmStreamCreate(x) hipStreamCreate(x) 317 #define cupmEventCreate(x) hipEventCreate(x); 318 #define cupmGetLastError() hipGetLastError() 319 #define cupmDeviceMapHost hipDeviceMapHost 320 #define cupmSuccess hipSuccess 321 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 322 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 323 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 324 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 325 #define PetscCUPMBLASInitializeHandle() 0 326 #define PetscCUPMSOLVERDnInitializeHandle() 0 327 #define PetscCUPMInitialize PetscHIPInitialize 328 #define PetscCUPMInitialized PetscHIPInitialized 329 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 330 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 331 #define PetscCUPMSynchronize PetscHIPSynchronize 332 #define PetscNotUseCUPM PetscNotUseHIP 333 #define cupmOptionsStr "HIP options" 334 #define cupmSetDeviceStr "-hip_device" 335 #define cupmViewStr "-hip_view" 336 #define cupmSynchronizeStr "-hip_synchronize" 337 #define PetscCUPMInitializeStr "PetscHIPInitialize" 338 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 339 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 340 #define PetscDefaultCupmStream PetscDefaultHipStream 341 #include "cupminit.inc" 342 #endif 343 344 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 345 { 346 char string[64]; 347 MPI_Comm comm = PETSC_COMM_WORLD; 348 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 349 PetscErrorCode ierr; 350 PetscReal si; 351 PetscInt intensity; 352 int i; 353 PetscMPIInt rank; 354 char version[256]; 355 #if defined(PETSC_USE_LOG) 356 char mname[PETSC_MAX_PATH_LEN]; 357 PetscViewerFormat format; 358 PetscBool flg4 = PETSC_FALSE; 359 #endif 360 361 PetscFunctionBegin; 362 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 363 364 #if !defined(PETSC_HAVE_THREADSAFETY) 365 if (!(PETSC_RUNNING_ON_VALGRIND)) { 366 /* 367 Setup the memory management; support for tracing malloc() usage 368 */ 369 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 370 371 if (PetscDefined(USE_DEBUG)) { 372 mdebug = PETSC_TRUE; 373 initializenan = PETSC_TRUE; 374 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 375 } else { 376 /* don't warn about unused option */ 377 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 378 flg1 = PETSC_FALSE; 379 } 380 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 381 if (flg1 || flg2) { 382 mdebug = PETSC_TRUE; 383 eachcall = PETSC_TRUE; 384 initializenan = PETSC_TRUE; 385 } else if (flg3 && !flg2) { 386 mdebug = PETSC_FALSE; 387 eachcall = PETSC_FALSE; 388 initializenan = PETSC_FALSE; 389 } 390 391 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 392 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 393 394 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 395 if (mlog) { 396 mdebug = PETSC_TRUE; 397 } 398 /* the next line is deprecated */ 399 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 400 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 401 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 402 if (mdebug) { 403 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 404 } 405 if (mlog) { 406 PetscReal logthreshold = 0; 407 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 408 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 409 } 410 #if defined(PETSC_USE_LOG) 411 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 412 #endif 413 } 414 415 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 416 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 417 flg1 = PETSC_FALSE; 418 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 419 /* ignore this option if malloc is already set */ 420 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 421 422 flg1 = PETSC_FALSE; 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 424 if (!flg1) { 425 flg1 = PETSC_FALSE; 426 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 427 } 428 if (flg1) { 429 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 430 } 431 #endif 432 433 #if defined(PETSC_USE_LOG) 434 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 435 #endif 436 437 /* 438 Set the display variable for graphics 439 */ 440 ierr = PetscSetDisplay();CHKERRQ(ierr); 441 442 /* 443 Print main application help message 444 */ 445 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 446 if (help && hasHelp) { 447 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 448 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 449 } 450 451 /* 452 Print the PETSc version information 453 */ 454 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 455 if (flg1 || hasHelp) { 456 /* 457 Print "higher-level" package version message 458 */ 459 if (PetscExternalVersionFunction) { 460 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 461 } 462 463 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 464 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 465 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 471 } 472 473 /* 474 Print "higher-level" package help message 475 */ 476 if (hasHelp) { 477 PetscBool hasHelpIntro; 478 479 if (PetscExternalHelpFunction) { 480 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 481 } 482 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 483 if (hasHelpIntro) { 484 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 485 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 486 ierr = MPI_Finalize();CHKERRMPI(ierr); 487 exit(0); 488 } 489 } 490 491 /* 492 Setup the error handling 493 */ 494 flg1 = PETSC_FALSE; 495 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 496 if (flg1) { 497 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 498 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 499 } 500 flg1 = PETSC_FALSE; 501 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 502 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 503 flg1 = PETSC_FALSE; 504 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 505 if (flg1) { 506 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 507 } 508 flg1 = PETSC_FALSE; 509 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 510 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 511 512 /* 513 Setup debugger information 514 */ 515 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 516 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 517 if (flg1) { 518 MPI_Errhandler err_handler; 519 520 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 521 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 522 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 523 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 524 } 525 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 526 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 527 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 528 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 529 if (flg1 || flg2) { 530 PetscMPIInt size; 531 PetscInt lsize,*ranks; 532 MPI_Errhandler err_handler; 533 /* 534 we have to make sure that all processors have opened 535 connections to all other processors, otherwise once the 536 debugger has stated it is likely to receive a SIGUSR1 537 and kill the program. 538 */ 539 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 540 if (size > 2) { 541 PetscMPIInt dummy = 0; 542 MPI_Status status; 543 for (i=0; i<size; i++) { 544 if (rank != i) { 545 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 546 } 547 } 548 for (i=0; i<size; i++) { 549 if (rank != i) { 550 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 551 } 552 } 553 } 554 /* check if this processor node should be in debugger */ 555 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 556 lsize = size; 557 /* Deprecated in 3.14 */ 558 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 559 if (flag) { 560 const char * const quietopt="-options_suppress_deprecated_warnings"; 561 char msg[4096]; 562 PetscBool quiet = PETSC_FALSE; 563 564 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 565 if (!quiet) { 566 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 567 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 568 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 569 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 570 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 571 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 572 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 573 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 576 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 577 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 578 } 579 } else { 580 lsize = size; 581 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 582 } 583 if (flag) { 584 for (i=0; i<lsize; i++) { 585 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 586 } 587 } 588 if (!flag) { 589 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 590 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 591 if (flg1) { 592 ierr = PetscAttachDebugger();CHKERRQ(ierr); 593 } else { 594 ierr = PetscStopForDebugger();CHKERRQ(ierr); 595 } 596 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 597 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 598 } else { 599 ierr = PetscWaitOnError();CHKERRQ(ierr); 600 } 601 ierr = PetscFree(ranks);CHKERRQ(ierr); 602 } 603 604 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 605 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 606 607 /* 608 Setup profiling and logging 609 */ 610 #if defined(PETSC_USE_INFO) 611 { 612 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 613 } 614 #endif 615 ierr = PetscDetermineInitialFPTrap(); 616 flg1 = PETSC_FALSE; 617 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 618 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 619 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 620 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 621 #if defined(PETSC_USE_LOG) 622 mname[0] = 0; 623 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 624 if (flg1) { 625 if (mname[0]) { 626 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 627 } else { 628 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 629 } 630 } 631 632 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 633 634 #if defined(PETSC_HAVE_MPE) 635 flg1 = PETSC_FALSE; 636 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 637 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 638 #endif 639 flg1 = PETSC_FALSE; 640 flg3 = PETSC_FALSE; 641 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 642 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 643 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 644 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 645 646 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 647 if (flg1) { 648 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 649 FILE *file; 650 if (mname[0]) { 651 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 652 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 653 file = fopen(fname,"w"); 654 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 655 } else file = PETSC_STDOUT; 656 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 657 } 658 659 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 660 if (flg4) { 661 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 662 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 663 } else { 664 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 665 } 666 } 667 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 668 PetscReal threshold = PetscRealConstant(0.01); 669 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 670 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 671 } 672 #endif 673 674 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 675 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 676 /* 677 If collecting logging information, by default, wait for device to complete its operations 678 before returning to the CPU in order to get accurate timings of each event 679 */ 680 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 681 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 682 683 #if defined(PETSC_HAVE_CUDA) 684 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 685 #endif 686 687 #if defined(PETSC_HAVE_HIP) 688 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 689 #endif 690 691 /* 692 Print basic help message 693 */ 694 if (hasHelp) { 695 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 730 #if defined(PETSC_USE_LOG) 731 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 735 #if defined(PETSC_HAVE_MPE) 736 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 737 #endif 738 #endif 739 #if defined(PETSC_USE_INFO) 740 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 741 #endif 742 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 746 } 747 748 #if defined(PETSC_HAVE_POPEN) 749 { 750 char machine[128]; 751 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 752 if (flg1) { 753 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 754 } 755 } 756 #endif 757 758 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 759 if (flg1) { 760 ierr = PetscSleep(si);CHKERRQ(ierr); 761 } 762 763 #if defined(PETSC_HAVE_VIENNACL) 764 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 765 if (!flg3) { 766 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 767 } 768 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 769 PetscViennaCLSynchronize = flg3; 770 ierr = PetscViennaCLInit();CHKERRQ(ierr); 771 #endif 772 773 /* 774 Creates the logging data structures; this is enabled even if logging is not turned on 775 This is the last thing we do before returning to the user code to prevent having the 776 logging numbers contaminated by any startup time associated with MPI and the GPUs 777 */ 778 #if defined(PETSC_USE_LOG) 779 ierr = PetscLogInitialize();CHKERRQ(ierr); 780 #endif 781 782 PetscFunctionReturn(0); 783 } 784