1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 #if defined(PETSC_HAVE_NVSHMEM) 62 PetscBool PetscBeganNvshmem = PETSC_FALSE; 63 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 64 #endif 65 66 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 67 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 68 69 #if defined(PETSC_HAVE_COMPLEX) 70 #if defined(PETSC_COMPLEX_INSTANTIATE) 71 template <> class std::complex<double>; /* instantiate complex template class */ 72 #endif 73 74 /*MC 75 PETSC_i - the imaginary number i 76 77 Synopsis: 78 #include <petscsys.h> 79 PetscComplex PETSC_i; 80 81 Level: beginner 82 83 Note: 84 Complex numbers are automatically available if PETSc located a working complex implementation 85 86 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 87 M*/ 88 PetscComplex PETSC_i; 89 MPI_Datatype MPIU___COMPLEX128 = 0; 90 #endif /* PETSC_HAVE_COMPLEX */ 91 #if defined(PETSC_USE_REAL___FLOAT128) 92 MPI_Datatype MPIU___FLOAT128 = 0; 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 #if defined(PETSC_USE_64BIT_INDICES) 98 MPI_Datatype MPIU_2INT = 0; 99 #endif 100 MPI_Datatype MPIU_BOOL; 101 MPI_Datatype MPIU_ENUM; 102 MPI_Datatype MPIU_FORTRANADDR; 103 MPI_Datatype MPIU_SIZE_T; 104 105 /* 106 Function that is called to display all error messages 107 */ 108 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 109 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 110 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 111 /* 112 This is needed to turn on/off GPU synchronization 113 */ 114 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 115 116 /* ------------------------------------------------------------------------------*/ 117 /* 118 Optional file where all PETSc output from various prints is saved 119 */ 120 PETSC_INTERN FILE *petsc_history; 121 FILE *petsc_history = NULL; 122 123 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 124 { 125 PetscErrorCode ierr; 126 PetscMPIInt rank,size; 127 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 128 char version[256]; 129 130 PetscFunctionBegin; 131 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 132 if (!rank) { 133 char arch[10]; 134 int err; 135 136 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 137 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 138 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 139 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 140 if (filename) { 141 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 142 } else { 143 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 146 } 147 148 *fd = fopen(fname,"a"); 149 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 150 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 153 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 157 err = fflush(*fd); 158 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 159 } 160 PetscFunctionReturn(0); 161 } 162 163 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 164 { 165 PetscErrorCode ierr; 166 PetscMPIInt rank; 167 char date[64]; 168 int err; 169 170 PetscFunctionBegin; 171 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 172 if (!rank) { 173 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 err = fflush(*fd); 178 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 179 err = fclose(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 181 } 182 PetscFunctionReturn(0); 183 } 184 185 /* ------------------------------------------------------------------------------*/ 186 187 /* 188 This is ugly and probably belongs somewhere else, but I want to 189 be able to put a true MPI abort error handler with command line args. 190 191 This is so MPI errors in the debugger will leave all the stack 192 frames. The default MP_Abort() cleans up and exits thus providing no useful information 193 in the debugger hence we call abort() instead of MPI_Abort(). 194 */ 195 196 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 197 { 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 abort(); 201 } 202 203 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 204 { 205 PetscErrorCode ierr; 206 207 PetscFunctionBegin; 208 (*PetscErrorPrintf)("MPI error %d\n",*flag); 209 ierr = PetscAttachDebugger(); 210 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 211 } 212 213 /*@C 214 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 215 wishes a clean exit somewhere deep in the program. 216 217 Collective on PETSC_COMM_WORLD 218 219 Options Database Keys are the same as for PetscFinalize() 220 221 Level: advanced 222 223 Note: 224 See PetscInitialize() for more general runtime options. 225 226 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 227 @*/ 228 PetscErrorCode PetscEnd(void) 229 { 230 PetscFunctionBegin; 231 PetscFinalize(); 232 exit(0); 233 return 0; 234 } 235 236 PetscBool PetscOptionsPublish = PETSC_FALSE; 237 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 238 PETSC_INTERN PetscBool petscsetmallocvisited; 239 static char emacsmachinename[256]; 240 241 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 242 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 243 244 /*@C 245 PetscSetHelpVersionFunctions - Sets functions that print help and version information 246 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 247 This routine enables a "higher-level" package that uses PETSc to print its messages first. 248 249 Input Parameter: 250 + help - the help function (may be NULL) 251 - version - the version function (may be NULL) 252 253 Level: developer 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #if defined(PETSC_USE_LOG) 265 PETSC_INTERN PetscBool PetscObjectsLog; 266 #endif 267 268 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 269 Use the following macros to define CUDA/HIP initialization related vars/routines. 270 */ 271 #if defined(PETSC_HAVE_CUDA) 272 typedef cudaError_t cupmError_t; 273 typedef struct cudaDeviceProp cupmDeviceProp; 274 typedef cudaStream_t cupmStream_t; 275 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 276 #define cupmGetDevice(x) cudaGetDevice(x) 277 #define cupmSetDevice(x) cudaSetDevice(x) 278 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 279 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 280 #define cupmStreamCreate(x) cudaStreamCreate(x) 281 #define cupmGetLastError() cudaGetLastError() 282 #define cupmDeviceMapHost cudaDeviceMapHost 283 #define cupmSuccess cudaSuccess 284 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 285 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 286 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 287 #define CHKERRCUPM(x) CHKERRCUDA(x) 288 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 289 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 290 #define PetscCUPMInitialize PetscCUDAInitialize 291 #define PetscCUPMInitialized PetscCUDAInitialized 292 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 293 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 294 #define PetscCUPMSynchronize PetscCUDASynchronize 295 #define PetscNotUseCUPM PetscNotUseCUDA 296 #define cupmOptionsStr "CUDA options" 297 #define cupmSetDeviceStr "-cuda_device" 298 #define cupmViewStr "-cuda_view" 299 #define cupmSynchronizeStr "-cuda_synchronize" 300 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 301 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 302 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 303 #define PetscDefaultCupmStream PetscDefaultCudaStream 304 #include "cupminit.inc" 305 #endif 306 307 #if defined(PETSC_HAVE_HIP) 308 typedef hipError_t cupmError_t; 309 typedef hipDeviceProp_t cupmDeviceProp; 310 typedef hipStream_t cupmStream_t; 311 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 312 #define cupmGetDevice(x) hipGetDevice(x) 313 #define cupmSetDevice(x) hipSetDevice(x) 314 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 315 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 316 #define cupmStreamCreate(x) hipStreamCreate(x) 317 #define cupmGetLastError() hipGetLastError() 318 #define cupmDeviceMapHost hipDeviceMapHost 319 #define cupmSuccess hipSuccess 320 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 321 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 322 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 323 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 324 #define PetscCUPMBLASInitializeHandle() 0 325 #define PetscCUPMSOLVERDnInitializeHandle() 0 326 #define PetscCUPMInitialize PetscHIPInitialize 327 #define PetscCUPMInitialized PetscHIPInitialized 328 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 329 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 330 #define PetscCUPMSynchronize PetscHIPSynchronize 331 #define PetscNotUseCUPM PetscNotUseHIP 332 #define cupmOptionsStr "HIP options" 333 #define cupmSetDeviceStr "-hip_device" 334 #define cupmViewStr "-hip_view" 335 #define cupmSynchronizeStr "-hip_synchronize" 336 #define PetscCUPMInitializeStr "PetscHIPInitialize" 337 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 338 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 339 #define PetscDefaultCupmStream PetscDefaultHipStream 340 #include "cupminit.inc" 341 #endif 342 343 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 344 { 345 char string[64]; 346 MPI_Comm comm = PETSC_COMM_WORLD; 347 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 348 PetscErrorCode ierr; 349 PetscReal si; 350 PetscInt intensity; 351 int i; 352 PetscMPIInt rank; 353 char version[256]; 354 #if defined(PETSC_USE_LOG) 355 char mname[PETSC_MAX_PATH_LEN]; 356 PetscViewerFormat format; 357 PetscBool flg4 = PETSC_FALSE; 358 #endif 359 360 PetscFunctionBegin; 361 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 362 363 #if !defined(PETSC_HAVE_THREADSAFETY) 364 if (!(PETSC_RUNNING_ON_VALGRIND)) { 365 /* 366 Setup the memory management; support for tracing malloc() usage 367 */ 368 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 369 370 if (PetscDefined(USE_DEBUG)) { 371 mdebug = PETSC_TRUE; 372 initializenan = PETSC_TRUE; 373 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 374 } else { 375 /* don't warn about unused option */ 376 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 377 flg1 = PETSC_FALSE; 378 } 379 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 380 if (flg1 || flg2) { 381 mdebug = PETSC_TRUE; 382 eachcall = PETSC_TRUE; 383 initializenan = PETSC_TRUE; 384 } else if (flg3 && !flg2) { 385 mdebug = PETSC_FALSE; 386 eachcall = PETSC_FALSE; 387 initializenan = PETSC_FALSE; 388 } 389 390 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 391 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 392 393 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 394 if (mlog) { 395 mdebug = PETSC_TRUE; 396 } 397 /* the next line is deprecated */ 398 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 399 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 400 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 401 if (mdebug) { 402 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 403 } 404 if (mlog) { 405 PetscReal logthreshold = 0; 406 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 407 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 408 } 409 #if defined(PETSC_USE_LOG) 410 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 411 #endif 412 } 413 414 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 415 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 416 flg1 = PETSC_FALSE; 417 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 418 /* ignore this option if malloc is already set */ 419 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 420 421 flg1 = PETSC_FALSE; 422 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 423 if (!flg1) { 424 flg1 = PETSC_FALSE; 425 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 426 } 427 if (flg1) { 428 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 429 } 430 #endif 431 432 #if defined(PETSC_USE_LOG) 433 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 434 #endif 435 436 /* 437 Set the display variable for graphics 438 */ 439 ierr = PetscSetDisplay();CHKERRQ(ierr); 440 441 /* 442 Print main application help message 443 */ 444 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 445 if (help && hasHelp) { 446 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 447 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 448 } 449 450 /* 451 Print the PETSc version information 452 */ 453 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 454 if (flg1 || hasHelp) { 455 /* 456 Print "higher-level" package version message 457 */ 458 if (PetscExternalVersionFunction) { 459 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 460 } 461 462 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 463 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 464 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 465 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 470 } 471 472 /* 473 Print "higher-level" package help message 474 */ 475 if (hasHelp) { 476 PetscBool hasHelpIntro; 477 478 if (PetscExternalHelpFunction) { 479 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 480 } 481 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 482 if (hasHelpIntro) { 483 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 484 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 485 ierr = MPI_Finalize();CHKERRMPI(ierr); 486 exit(0); 487 } 488 } 489 490 /* 491 Setup the error handling 492 */ 493 flg1 = PETSC_FALSE; 494 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 495 if (flg1) { 496 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 497 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 498 } 499 flg1 = PETSC_FALSE; 500 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 501 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 502 flg1 = PETSC_FALSE; 503 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 504 if (flg1) { 505 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 506 } 507 flg1 = PETSC_FALSE; 508 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 509 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 510 511 /* 512 Setup debugger information 513 */ 514 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 515 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 516 if (flg1) { 517 MPI_Errhandler err_handler; 518 519 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 520 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 521 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 522 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 523 } 524 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 525 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 526 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 527 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 528 if (flg1 || flg2) { 529 PetscMPIInt size; 530 PetscInt lsize,*ranks; 531 MPI_Errhandler err_handler; 532 /* 533 we have to make sure that all processors have opened 534 connections to all other processors, otherwise once the 535 debugger has stated it is likely to receive a SIGUSR1 536 and kill the program. 537 */ 538 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 539 if (size > 2) { 540 PetscMPIInt dummy = 0; 541 MPI_Status status; 542 for (i=0; i<size; i++) { 543 if (rank != i) { 544 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 545 } 546 } 547 for (i=0; i<size; i++) { 548 if (rank != i) { 549 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 550 } 551 } 552 } 553 /* check if this processor node should be in debugger */ 554 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 555 lsize = size; 556 /* Deprecated in 3.14 */ 557 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 558 if (flag) { 559 const char * const quietopt="-options_suppress_deprecated_warnings"; 560 char msg[4096]; 561 PetscBool quiet = PETSC_FALSE; 562 563 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 564 if (!quiet) { 565 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 566 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 567 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 568 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 569 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 570 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 571 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 572 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 573 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 575 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 576 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 577 } 578 } else { 579 lsize = size; 580 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 581 } 582 if (flag) { 583 for (i=0; i<lsize; i++) { 584 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 585 } 586 } 587 if (!flag) { 588 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 589 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 590 if (flg1) { 591 ierr = PetscAttachDebugger();CHKERRQ(ierr); 592 } else { 593 ierr = PetscStopForDebugger();CHKERRQ(ierr); 594 } 595 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 596 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 597 } else { 598 ierr = PetscWaitOnError();CHKERRQ(ierr); 599 } 600 ierr = PetscFree(ranks);CHKERRQ(ierr); 601 } 602 603 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 604 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 605 606 /* 607 Setup profiling and logging 608 */ 609 #if defined(PETSC_USE_INFO) 610 { 611 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 612 } 613 #endif 614 ierr = PetscDetermineInitialFPTrap(); 615 flg1 = PETSC_FALSE; 616 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 617 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 618 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 619 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 620 #if defined(PETSC_USE_LOG) 621 mname[0] = 0; 622 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 623 if (flg1) { 624 if (mname[0]) { 625 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 626 } else { 627 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 628 } 629 } 630 631 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 632 633 #if defined(PETSC_HAVE_MPE) 634 flg1 = PETSC_FALSE; 635 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 636 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 637 #endif 638 flg1 = PETSC_FALSE; 639 flg3 = PETSC_FALSE; 640 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 641 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 642 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 643 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 644 645 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 646 if (flg1) { 647 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 648 FILE *file; 649 if (mname[0]) { 650 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 651 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 652 file = fopen(fname,"w"); 653 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 654 } else file = PETSC_STDOUT; 655 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 656 } 657 658 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 659 if (flg4) { 660 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 661 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 662 } else { 663 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 664 } 665 } 666 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 667 PetscReal threshold = PetscRealConstant(0.01); 668 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 669 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 670 } 671 #endif 672 673 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 674 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 675 /* 676 If collecting logging information, by default, wait for device to complete its operations 677 before returning to the CPU in order to get accurate timings of each event 678 */ 679 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 680 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 681 682 #if defined(PETSC_HAVE_CUDA) 683 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 684 #endif 685 686 #if defined(PETSC_HAVE_HIP) 687 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 688 #endif 689 690 /* 691 Print basic help message 692 */ 693 if (hasHelp) { 694 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 729 #if defined(PETSC_USE_LOG) 730 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 734 #if defined(PETSC_HAVE_MPE) 735 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 736 #endif 737 #endif 738 #if defined(PETSC_USE_INFO) 739 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 740 #endif 741 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 745 } 746 747 #if defined(PETSC_HAVE_POPEN) 748 { 749 char machine[128]; 750 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 751 if (flg1) { 752 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 753 } 754 } 755 #endif 756 757 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 758 if (flg1) { 759 ierr = PetscSleep(si);CHKERRQ(ierr); 760 } 761 762 #if defined(PETSC_HAVE_VIENNACL) 763 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 764 if (!flg3) { 765 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 766 } 767 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 768 PetscViennaCLSynchronize = flg3; 769 ierr = PetscViennaCLInit();CHKERRQ(ierr); 770 #endif 771 772 /* 773 Creates the logging data structures; this is enabled even if logging is not turned on 774 This is the last thing we do before returning to the user code to prevent having the 775 logging numbers contaminated by any startup time associated with MPI and the GPUs 776 */ 777 #if defined(PETSC_USE_LOG) 778 ierr = PetscLogInitialize();CHKERRQ(ierr); 779 #endif 780 781 PetscFunctionReturn(0); 782 } 783