1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 #if defined(PETSC_HAVE_NVSHMEM) 62 PetscBool PetscBeganNvshmem = PETSC_FALSE; 63 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 64 #endif 65 66 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 67 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 68 69 #if defined(PETSC_HAVE_COMPLEX) 70 #if defined(PETSC_COMPLEX_INSTANTIATE) 71 template <> class std::complex<double>; /* instantiate complex template class */ 72 #endif 73 74 /*MC 75 PETSC_i - the imaginary number i 76 77 Synopsis: 78 #include <petscsys.h> 79 PetscComplex PETSC_i; 80 81 Level: beginner 82 83 Note: 84 Complex numbers are automatically available if PETSc located a working complex implementation 85 86 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 87 M*/ 88 PetscComplex PETSC_i; 89 MPI_Datatype MPIU___COMPLEX128 = 0; 90 #endif /* PETSC_HAVE_COMPLEX */ 91 #if defined(PETSC_USE_REAL___FLOAT128) 92 MPI_Datatype MPIU___FLOAT128 = 0; 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 #if defined(PETSC_USE_64BIT_INDICES) 98 MPI_Datatype MPIU_2INT = 0; 99 #endif 100 MPI_Datatype MPIU_BOOL; 101 MPI_Datatype MPIU_ENUM; 102 MPI_Datatype MPIU_FORTRANADDR; 103 MPI_Datatype MPIU_SIZE_T; 104 105 /* 106 Function that is called to display all error messages 107 */ 108 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 109 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 110 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 111 /* 112 This is needed to turn on/off GPU synchronization 113 */ 114 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 115 116 /* ------------------------------------------------------------------------------*/ 117 /* 118 Optional file where all PETSc output from various prints is saved 119 */ 120 PETSC_INTERN FILE *petsc_history; 121 FILE *petsc_history = NULL; 122 123 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 124 { 125 PetscErrorCode ierr; 126 PetscMPIInt rank,size; 127 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 128 char version[256]; 129 130 PetscFunctionBegin; 131 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 132 if (!rank) { 133 char arch[10]; 134 int err; 135 136 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 137 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 138 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 139 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 140 if (filename) { 141 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 142 } else { 143 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 146 } 147 148 *fd = fopen(fname,"a"); 149 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 150 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 153 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 157 err = fflush(*fd); 158 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 159 } 160 PetscFunctionReturn(0); 161 } 162 163 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 164 { 165 PetscErrorCode ierr; 166 PetscMPIInt rank; 167 char date[64]; 168 int err; 169 170 PetscFunctionBegin; 171 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 172 if (!rank) { 173 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 err = fflush(*fd); 178 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 179 err = fclose(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 181 } 182 PetscFunctionReturn(0); 183 } 184 185 /* ------------------------------------------------------------------------------*/ 186 187 /* 188 This is ugly and probably belongs somewhere else, but I want to 189 be able to put a true MPI abort error handler with command line args. 190 191 This is so MPI errors in the debugger will leave all the stack 192 frames. The default MP_Abort() cleans up and exits thus providing no useful information 193 in the debugger hence we call abort() instead of MPI_Abort(). 194 */ 195 196 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 197 { 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 abort(); 201 } 202 203 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 204 { 205 PetscErrorCode ierr; 206 207 PetscFunctionBegin; 208 (*PetscErrorPrintf)("MPI error %d\n",*flag); 209 ierr = PetscAttachDebugger(); 210 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 211 } 212 213 /*@C 214 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 215 wishes a clean exit somewhere deep in the program. 216 217 Collective on PETSC_COMM_WORLD 218 219 Options Database Keys are the same as for PetscFinalize() 220 221 Level: advanced 222 223 Note: 224 See PetscInitialize() for more general runtime options. 225 226 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 227 @*/ 228 PetscErrorCode PetscEnd(void) 229 { 230 PetscFunctionBegin; 231 PetscFinalize(); 232 exit(0); 233 return 0; 234 } 235 236 PetscBool PetscOptionsPublish = PETSC_FALSE; 237 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 238 PETSC_INTERN PetscBool petscsetmallocvisited; 239 static char emacsmachinename[256]; 240 241 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 242 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 243 244 /*@C 245 PetscSetHelpVersionFunctions - Sets functions that print help and version information 246 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 247 This routine enables a "higher-level" package that uses PETSc to print its messages first. 248 249 Input Parameter: 250 + help - the help function (may be NULL) 251 - version - the version function (may be NULL) 252 253 Level: developer 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #if defined(PETSC_USE_LOG) 265 PETSC_INTERN PetscBool PetscObjectsLog; 266 #endif 267 268 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 269 Use the following macros to define CUDA/HIP initialization related vars/routines. 270 */ 271 #if defined(PETSC_HAVE_CUDA) 272 typedef cudaError_t cupmError_t; 273 typedef struct cudaDeviceProp cupmDeviceProp; 274 typedef cudaStream_t cupmStream_t; 275 typedef cudaEvent_t cupmEvent_t; 276 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 277 #define cupmGetDevice(x) cudaGetDevice(x) 278 #define cupmSetDevice(x) cudaSetDevice(x) 279 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 280 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 281 #define cupmStreamCreate(x) cudaStreamCreate(x) 282 #define cupmGetLastError() cudaGetLastError() 283 #define cupmDeviceMapHost cudaDeviceMapHost 284 #define cupmSuccess cudaSuccess 285 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 286 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 287 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 288 #define CHKERRCUPM(x) CHKERRCUDA(x) 289 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 290 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 291 #define PetscCUPMInitialize PetscCUDAInitialize 292 #define PetscCUPMInitialized PetscCUDAInitialized 293 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 294 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 295 #define PetscCUPMSynchronize PetscCUDASynchronize 296 #define PetscNotUseCUPM PetscNotUseCUDA 297 #define cupmOptionsStr "CUDA options" 298 #define cupmSetDeviceStr "-cuda_device" 299 #define cupmViewStr "-cuda_view" 300 #define cupmSynchronizeStr "-cuda_synchronize" 301 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 302 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 303 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 304 #define PetscDefaultCupmStream PetscDefaultCudaStream 305 #define cupmEventCreate(x) cudaEventCreate(x) 306 #include "cupminit.inc" 307 #endif 308 309 #if defined(PETSC_HAVE_HIP) 310 typedef hipError_t cupmError_t; 311 typedef hipDeviceProp_t cupmDeviceProp; 312 typedef hipStream_t cupmStream_t; 313 typedef hipEvent_t cupmEvent_t; 314 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 315 #define cupmGetDevice(x) hipGetDevice(x) 316 #define cupmSetDevice(x) hipSetDevice(x) 317 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 318 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 319 #define cupmStreamCreate(x) hipStreamCreate(x) 320 #define cupmEventCreate(x) hipEventCreate(x); 321 #define cupmGetLastError() hipGetLastError() 322 #define cupmDeviceMapHost hipDeviceMapHost 323 #define cupmSuccess hipSuccess 324 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 325 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 326 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 327 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 328 #define PetscCUPMBLASInitializeHandle() 0 329 #define PetscCUPMSOLVERDnInitializeHandle() 0 330 #define PetscCUPMInitialize PetscHIPInitialize 331 #define PetscCUPMInitialized PetscHIPInitialized 332 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 333 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 334 #define PetscCUPMSynchronize PetscHIPSynchronize 335 #define PetscNotUseCUPM PetscNotUseHIP 336 #define cupmOptionsStr "HIP options" 337 #define cupmSetDeviceStr "-hip_device" 338 #define cupmViewStr "-hip_view" 339 #define cupmSynchronizeStr "-hip_synchronize" 340 #define PetscCUPMInitializeStr "PetscHIPInitialize" 341 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 342 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 343 #define PetscDefaultCupmStream PetscDefaultHipStream 344 #include "cupminit.inc" 345 #endif 346 347 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 348 { 349 char string[64]; 350 MPI_Comm comm = PETSC_COMM_WORLD; 351 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 352 PetscErrorCode ierr; 353 PetscReal si; 354 PetscInt intensity; 355 int i; 356 PetscMPIInt rank; 357 char version[256]; 358 #if defined(PETSC_USE_LOG) 359 char mname[PETSC_MAX_PATH_LEN]; 360 PetscViewerFormat format; 361 PetscBool flg4 = PETSC_FALSE; 362 #endif 363 364 PetscFunctionBegin; 365 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 366 367 #if !defined(PETSC_HAVE_THREADSAFETY) 368 if (!(PETSC_RUNNING_ON_VALGRIND)) { 369 /* 370 Setup the memory management; support for tracing malloc() usage 371 */ 372 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 373 374 if (PetscDefined(USE_DEBUG)) { 375 mdebug = PETSC_TRUE; 376 initializenan = PETSC_TRUE; 377 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 378 } else { 379 /* don't warn about unused option */ 380 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 381 flg1 = PETSC_FALSE; 382 } 383 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 384 if (flg1 || flg2) { 385 mdebug = PETSC_TRUE; 386 eachcall = PETSC_TRUE; 387 initializenan = PETSC_TRUE; 388 } else if (flg3 && !flg2) { 389 mdebug = PETSC_FALSE; 390 eachcall = PETSC_FALSE; 391 initializenan = PETSC_FALSE; 392 } 393 394 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 395 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 396 397 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 398 if (mlog) { 399 mdebug = PETSC_TRUE; 400 } 401 /* the next line is deprecated */ 402 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 403 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 404 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 405 if (mdebug) { 406 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 407 } 408 if (mlog) { 409 PetscReal logthreshold = 0; 410 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 411 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 412 } 413 #if defined(PETSC_USE_LOG) 414 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 415 #endif 416 } 417 418 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 419 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 420 flg1 = PETSC_FALSE; 421 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 422 /* ignore this option if malloc is already set */ 423 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 424 425 flg1 = PETSC_FALSE; 426 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 427 if (!flg1) { 428 flg1 = PETSC_FALSE; 429 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 430 } 431 if (flg1) { 432 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 433 } 434 #endif 435 436 #if defined(PETSC_USE_LOG) 437 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 438 #endif 439 440 /* 441 Set the display variable for graphics 442 */ 443 ierr = PetscSetDisplay();CHKERRQ(ierr); 444 445 /* 446 Print main application help message 447 */ 448 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 449 if (help && hasHelp) { 450 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 451 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 452 } 453 454 /* 455 Print the PETSc version information 456 */ 457 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 458 if (flg1 || hasHelp) { 459 /* 460 Print "higher-level" package version message 461 */ 462 if (PetscExternalVersionFunction) { 463 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 464 } 465 466 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 474 } 475 476 /* 477 Print "higher-level" package help message 478 */ 479 if (hasHelp) { 480 PetscBool hasHelpIntro; 481 482 if (PetscExternalHelpFunction) { 483 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 484 } 485 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 486 if (hasHelpIntro) { 487 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 488 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 489 ierr = MPI_Finalize();CHKERRMPI(ierr); 490 exit(0); 491 } 492 } 493 494 /* 495 Setup the error handling 496 */ 497 flg1 = PETSC_FALSE; 498 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 499 if (flg1) { 500 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 501 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 502 } 503 flg1 = PETSC_FALSE; 504 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 505 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 506 flg1 = PETSC_FALSE; 507 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 508 if (flg1) { 509 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 510 } 511 flg1 = PETSC_FALSE; 512 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 513 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 514 515 /* 516 Setup debugger information 517 */ 518 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 519 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 520 if (flg1) { 521 MPI_Errhandler err_handler; 522 523 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 524 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 525 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 526 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 527 } 528 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 529 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 530 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 531 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 532 if (flg1 || flg2) { 533 PetscMPIInt size; 534 PetscInt lsize,*ranks; 535 MPI_Errhandler err_handler; 536 /* 537 we have to make sure that all processors have opened 538 connections to all other processors, otherwise once the 539 debugger has stated it is likely to receive a SIGUSR1 540 and kill the program. 541 */ 542 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 543 if (size > 2) { 544 PetscMPIInt dummy = 0; 545 MPI_Status status; 546 for (i=0; i<size; i++) { 547 if (rank != i) { 548 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 549 } 550 } 551 for (i=0; i<size; i++) { 552 if (rank != i) { 553 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 554 } 555 } 556 } 557 /* check if this processor node should be in debugger */ 558 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 559 lsize = size; 560 /* Deprecated in 3.14 */ 561 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 562 if (flag) { 563 const char * const quietopt="-options_suppress_deprecated_warnings"; 564 char msg[4096]; 565 PetscBool quiet = PETSC_FALSE; 566 567 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 568 if (!quiet) { 569 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 570 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 571 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 572 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 573 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 576 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 577 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 579 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 580 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 581 } 582 } else { 583 lsize = size; 584 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 585 } 586 if (flag) { 587 for (i=0; i<lsize; i++) { 588 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 589 } 590 } 591 if (!flag) { 592 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 593 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 594 if (flg1) { 595 ierr = PetscAttachDebugger();CHKERRQ(ierr); 596 } else { 597 ierr = PetscStopForDebugger();CHKERRQ(ierr); 598 } 599 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 600 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 601 } else { 602 ierr = PetscWaitOnError();CHKERRQ(ierr); 603 } 604 ierr = PetscFree(ranks);CHKERRQ(ierr); 605 } 606 607 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 608 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 609 610 /* 611 Setup profiling and logging 612 */ 613 #if defined(PETSC_USE_INFO) 614 { 615 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 616 } 617 #endif 618 ierr = PetscDetermineInitialFPTrap(); 619 flg1 = PETSC_FALSE; 620 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 621 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 622 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 623 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 624 #if defined(PETSC_USE_LOG) 625 mname[0] = 0; 626 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 627 if (flg1) { 628 if (mname[0]) { 629 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 630 } else { 631 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 632 } 633 } 634 635 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 636 637 #if defined(PETSC_HAVE_MPE) 638 flg1 = PETSC_FALSE; 639 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 640 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 641 #endif 642 flg1 = PETSC_FALSE; 643 flg3 = PETSC_FALSE; 644 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 645 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 646 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 647 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 648 649 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 650 if (flg1) { 651 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 652 FILE *file; 653 if (mname[0]) { 654 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 655 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 656 file = fopen(fname,"w"); 657 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 658 } else file = PETSC_STDOUT; 659 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 660 } 661 662 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 663 if (flg4) { 664 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 665 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 666 } else { 667 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 668 } 669 } 670 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 671 PetscReal threshold = PetscRealConstant(0.01); 672 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 673 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 674 } 675 #endif 676 677 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 678 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 679 /* 680 If collecting logging information, by default, wait for device to complete its operations 681 before returning to the CPU in order to get accurate timings of each event 682 */ 683 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 684 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 685 686 #if defined(PETSC_HAVE_CUDA) 687 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 688 #endif 689 690 #if defined(PETSC_HAVE_HIP) 691 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 692 #endif 693 694 /* 695 Print basic help message 696 */ 697 if (hasHelp) { 698 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 733 #if defined(PETSC_USE_LOG) 734 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 738 #if defined(PETSC_HAVE_MPE) 739 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 740 #endif 741 #endif 742 #if defined(PETSC_USE_INFO) 743 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 744 #endif 745 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 746 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 747 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 749 } 750 751 #if defined(PETSC_HAVE_POPEN) 752 { 753 char machine[128]; 754 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 755 if (flg1) { 756 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 757 } 758 } 759 #endif 760 761 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 762 if (flg1) { 763 ierr = PetscSleep(si);CHKERRQ(ierr); 764 } 765 766 #if defined(PETSC_HAVE_VIENNACL) 767 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 768 if (!flg3) { 769 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 770 } 771 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 772 PetscViennaCLSynchronize = flg3; 773 ierr = PetscViennaCLInit();CHKERRQ(ierr); 774 #endif 775 776 /* 777 Creates the logging data structures; this is enabled even if logging is not turned on 778 This is the last thing we do before returning to the user code to prevent having the 779 logging numbers contaminated by any startup time associated with MPI and the GPUs 780 */ 781 #if defined(PETSC_USE_LOG) 782 ierr = PetscLogInitialize();CHKERRQ(ierr); 783 #endif 784 785 PetscFunctionReturn(0); 786 } 787