1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 63 #if defined(PETSC_HAVE_COMPLEX) 64 #if defined(PETSC_COMPLEX_INSTANTIATE) 65 template <> class std::complex<double>; /* instantiate complex template class */ 66 #endif 67 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 68 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 69 MPI_Datatype MPIU_C_COMPLEX; 70 #endif 71 72 /*MC 73 PETSC_i - the imaginary number i 74 75 Synopsis: 76 #include <petscsys.h> 77 PetscComplex PETSC_i; 78 79 Level: beginner 80 81 Note: 82 Complex numbers are automatically available if PETSc located a working complex implementation 83 84 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 85 M*/ 86 PetscComplex PETSC_i; 87 #endif 88 #if defined(PETSC_USE_REAL___FLOAT128) 89 MPI_Datatype MPIU___FLOAT128 = 0; 90 #if defined(PETSC_HAVE_COMPLEX) 91 MPI_Datatype MPIU___COMPLEX128 = 0; 92 #endif 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 #if defined(PETSC_USE_64BIT_INDICES) 98 MPI_Datatype MPIU_2INT = 0; 99 #endif 100 MPI_Datatype MPIU_BOOL; 101 MPI_Datatype MPIU_ENUM; 102 MPI_Datatype MPIU_FORTRANADDR; 103 MPI_Datatype MPIU_SIZE_T; 104 105 /* 106 Function that is called to display all error messages 107 */ 108 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 109 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 110 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 111 /* 112 This is needed to turn on/off GPU synchronization 113 */ 114 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 115 116 /* ------------------------------------------------------------------------------*/ 117 /* 118 Optional file where all PETSc output from various prints is saved 119 */ 120 PETSC_INTERN FILE *petsc_history; 121 FILE *petsc_history = NULL; 122 123 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 124 { 125 PetscErrorCode ierr; 126 PetscMPIInt rank,size; 127 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 128 char version[256]; 129 130 PetscFunctionBegin; 131 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 132 if (!rank) { 133 char arch[10]; 134 int err; 135 136 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 137 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 138 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 139 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 140 if (filename) { 141 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 142 } else { 143 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 146 } 147 148 *fd = fopen(fname,"a"); 149 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 150 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 153 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 157 err = fflush(*fd); 158 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 159 } 160 PetscFunctionReturn(0); 161 } 162 163 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 164 { 165 PetscErrorCode ierr; 166 PetscMPIInt rank; 167 char date[64]; 168 int err; 169 170 PetscFunctionBegin; 171 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 172 if (!rank) { 173 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 err = fflush(*fd); 178 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 179 err = fclose(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 181 } 182 PetscFunctionReturn(0); 183 } 184 185 /* ------------------------------------------------------------------------------*/ 186 187 /* 188 This is ugly and probably belongs somewhere else, but I want to 189 be able to put a true MPI abort error handler with command line args. 190 191 This is so MPI errors in the debugger will leave all the stack 192 frames. The default MP_Abort() cleans up and exits thus providing no useful information 193 in the debugger hence we call abort() instead of MPI_Abort(). 194 */ 195 196 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 197 { 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 abort(); 201 } 202 203 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 204 { 205 PetscErrorCode ierr; 206 207 PetscFunctionBegin; 208 (*PetscErrorPrintf)("MPI error %d\n",*flag); 209 ierr = PetscAttachDebugger(); 210 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 211 } 212 213 /*@C 214 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 215 wishes a clean exit somewhere deep in the program. 216 217 Collective on PETSC_COMM_WORLD 218 219 Options Database Keys are the same as for PetscFinalize() 220 221 Level: advanced 222 223 Note: 224 See PetscInitialize() for more general runtime options. 225 226 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 227 @*/ 228 PetscErrorCode PetscEnd(void) 229 { 230 PetscFunctionBegin; 231 PetscFinalize(); 232 exit(0); 233 return 0; 234 } 235 236 PetscBool PetscOptionsPublish = PETSC_FALSE; 237 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 238 PETSC_INTERN PetscBool petscsetmallocvisited; 239 static char emacsmachinename[256]; 240 241 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 242 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 243 244 /*@C 245 PetscSetHelpVersionFunctions - Sets functions that print help and version information 246 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 247 This routine enables a "higher-level" package that uses PETSc to print its messages first. 248 249 Input Parameter: 250 + help - the help function (may be NULL) 251 - version - the version function (may be NULL) 252 253 Level: developer 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #if defined(PETSC_USE_LOG) 265 PETSC_INTERN PetscBool PetscObjectsLog; 266 #endif 267 268 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 269 { 270 if (PetscUnlikely(*err)) { 271 PetscMPIInt len; 272 char errstring[MPI_MAX_ERROR_STRING]; 273 274 MPI_Error_string(*err,errstring,&len); 275 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 276 } 277 return; 278 } 279 280 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 281 Use the following macros to define CUDA/HIP initialization related vars/routines. 282 */ 283 #if defined(PETSC_HAVE_CUDA) 284 typedef cudaError_t cupmError_t; 285 typedef struct cudaDeviceProp cupmDeviceProp; 286 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 287 #define cupmGetDevice(x) cudaGetDevice(x) 288 #define cupmSetDevice(x) cudaSetDevice(x) 289 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 290 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 291 #define cupmGetLastError() cudaGetLastError() 292 #define cupmDeviceMapHost cudaDeviceMapHost 293 #define cupmSuccess cudaSuccess 294 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 295 #define CHKERRCUPM(x) CHKERRCUDA(x) 296 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 297 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 298 #define PetscCUPMInitialize PetscCUDAInitialize 299 #define PetscCUPMInitialized PetscCUDAInitialized 300 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 301 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 302 #define PetscCUPMSynchronize PetscCUDASynchronize 303 #define PetscNotUseCUPM PetscNotUseCUDA 304 #define cupmOptionsStr "CUDA options" 305 #define cupmSetDeviceStr "-cuda_device" 306 #define cupmViewStr "-cuda_view" 307 #define cupmSynchronizeStr "-cuda_synchronize" 308 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 309 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 310 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 311 #include "cupminit.inc" 312 #endif 313 314 #if defined(PETSC_HAVE_HIP) 315 typedef hipError_t cupmError_t; 316 typedef hipDeviceProp_t cupmDeviceProp; 317 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 318 #define cupmGetDevice(x) hipGetDevice(x) 319 #define cupmSetDevice(x) hipSetDevice(x) 320 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 321 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 322 #define cupmGetLastError() hipGetLastError() 323 #define cupmDeviceMapHost hipDeviceMapHost 324 #define cupmSuccess hipSuccess 325 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 326 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 327 #define PetscCUPMBLASInitializeHandle() 0 328 #define PetscCUPMSOLVERDnInitializeHandle() 0 329 #define PetscCUPMInitialize PetscHIPInitialize 330 #define PetscCUPMInitialized PetscHIPInitialized 331 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 332 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 333 #define PetscCUPMSynchronize PetscHIPSynchronize 334 #define PetscNotUseCUPM PetscNotUseHIP 335 #define cupmOptionsStr "HIP options" 336 #define cupmSetDeviceStr "-hip_device" 337 #define cupmViewStr "-hip_view" 338 #define cupmSynchronizeStr "-hip_synchronize" 339 #define PetscCUPMInitializeStr "PetscHIPInitialize" 340 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 341 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 342 #include "cupminit.inc" 343 #endif 344 345 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 346 { 347 char string[64]; 348 MPI_Comm comm = PETSC_COMM_WORLD; 349 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 350 PetscErrorCode ierr; 351 PetscReal si; 352 PetscInt intensity; 353 int i; 354 PetscMPIInt rank; 355 char version[256]; 356 #if defined(PETSC_USE_LOG) 357 char mname[PETSC_MAX_PATH_LEN]; 358 PetscViewerFormat format; 359 PetscBool flg4 = PETSC_FALSE; 360 #endif 361 362 PetscFunctionBegin; 363 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 364 365 #if !defined(PETSC_HAVE_THREADSAFETY) 366 if (!(PETSC_RUNNING_ON_VALGRIND)) { 367 /* 368 Setup the memory management; support for tracing malloc() usage 369 */ 370 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 371 372 if (PetscDefined(USE_DEBUG)) { 373 mdebug = PETSC_TRUE; 374 initializenan = PETSC_TRUE; 375 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 376 } else { 377 /* don't warn about unused option */ 378 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 379 flg1 = PETSC_FALSE; 380 } 381 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 382 if (flg1 || flg2) { 383 mdebug = PETSC_TRUE; 384 eachcall = PETSC_TRUE; 385 initializenan = PETSC_TRUE; 386 } else if (flg3 && !flg2) { 387 mdebug = PETSC_FALSE; 388 eachcall = PETSC_FALSE; 389 initializenan = PETSC_FALSE; 390 } 391 392 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 393 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 394 395 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 396 if (mlog) { 397 mdebug = PETSC_TRUE; 398 } 399 /* the next line is deprecated */ 400 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 401 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 402 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 403 if (mdebug) { 404 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 405 } 406 if (mlog) { 407 PetscReal logthreshold = 0; 408 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 409 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 410 } 411 #if defined(PETSC_USE_LOG) 412 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 413 #endif 414 } 415 416 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 417 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 418 flg1 = PETSC_FALSE; 419 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 420 /* ignore this option if malloc is already set */ 421 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 422 423 flg1 = PETSC_FALSE; 424 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 425 if (!flg1) { 426 flg1 = PETSC_FALSE; 427 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 428 } 429 if (flg1) { 430 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 431 } 432 #endif 433 434 #if defined(PETSC_USE_LOG) 435 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 436 #endif 437 438 /* 439 Set the display variable for graphics 440 */ 441 ierr = PetscSetDisplay();CHKERRQ(ierr); 442 443 /* 444 Print main application help message 445 */ 446 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 447 if (help && hasHelp) { 448 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 449 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 450 } 451 452 /* 453 Print the PETSc version information 454 */ 455 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 456 if (flg1 || hasHelp) { 457 /* 458 Print "higher-level" package version message 459 */ 460 if (PetscExternalVersionFunction) { 461 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 462 } 463 464 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 465 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 472 } 473 474 /* 475 Print "higher-level" package help message 476 */ 477 if (hasHelp) { 478 PetscBool hasHelpIntro; 479 480 if (PetscExternalHelpFunction) { 481 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 482 } 483 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 484 if (hasHelpIntro) { 485 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 486 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 487 ierr = MPI_Finalize();CHKERRQ(ierr); 488 exit(0); 489 } 490 } 491 492 /* 493 Setup the error handling 494 */ 495 flg1 = PETSC_FALSE; 496 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 497 if (flg1) { 498 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 499 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 500 } 501 flg1 = PETSC_FALSE; 502 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 503 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 504 flg1 = PETSC_FALSE; 505 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 506 if (flg1) { 507 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 508 } 509 /* experimental */ 510 flg1 = PETSC_FALSE; 511 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 512 if (flg1) { 513 MPI_Errhandler eh; 514 515 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 516 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 517 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 518 } 519 flg1 = PETSC_FALSE; 520 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 521 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 522 523 /* 524 Setup debugger information 525 */ 526 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 527 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 528 if (flg1) { 529 MPI_Errhandler err_handler; 530 531 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 532 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 533 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 534 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 535 } 536 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 537 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 538 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 539 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 540 if (flg1 || flg2) { 541 PetscMPIInt size; 542 PetscInt lsize,*ranks; 543 MPI_Errhandler err_handler; 544 /* 545 we have to make sure that all processors have opened 546 connections to all other processors, otherwise once the 547 debugger has stated it is likely to receive a SIGUSR1 548 and kill the program. 549 */ 550 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 551 if (size > 2) { 552 PetscMPIInt dummy = 0; 553 MPI_Status status; 554 for (i=0; i<size; i++) { 555 if (rank != i) { 556 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 557 } 558 } 559 for (i=0; i<size; i++) { 560 if (rank != i) { 561 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 562 } 563 } 564 } 565 /* check if this processor node should be in debugger */ 566 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 567 lsize = size; 568 /* Deprecated in 3.14 */ 569 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 570 if (flag) { 571 const char * const quietopt="-options_suppress_deprecated_warnings"; 572 char msg[4096]; 573 PetscBool quiet = PETSC_FALSE; 574 575 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 576 if (!quiet) { 577 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 579 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 580 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 581 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 582 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 583 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 584 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 585 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 586 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 587 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 588 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 589 } 590 } else { 591 lsize = size; 592 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 593 } 594 if (flag) { 595 for (i=0; i<lsize; i++) { 596 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 597 } 598 } 599 if (!flag) { 600 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 601 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 602 if (flg1) { 603 ierr = PetscAttachDebugger();CHKERRQ(ierr); 604 } else { 605 ierr = PetscStopForDebugger();CHKERRQ(ierr); 606 } 607 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 608 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 609 } else { 610 ierr = PetscWaitOnError();CHKERRQ(ierr); 611 } 612 ierr = PetscFree(ranks);CHKERRQ(ierr); 613 } 614 615 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 616 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 617 618 /* 619 Setup profiling and logging 620 */ 621 #if defined(PETSC_USE_INFO) 622 { 623 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 624 } 625 #endif 626 ierr = PetscDetermineInitialFPTrap(); 627 flg1 = PETSC_FALSE; 628 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 629 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 630 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 631 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 632 #if defined(PETSC_USE_LOG) 633 mname[0] = 0; 634 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 635 if (flg1) { 636 if (mname[0]) { 637 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 638 } else { 639 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 640 } 641 } 642 643 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 644 645 #if defined(PETSC_HAVE_MPE) 646 flg1 = PETSC_FALSE; 647 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 648 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 649 #endif 650 flg1 = PETSC_FALSE; 651 flg3 = PETSC_FALSE; 652 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 653 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 654 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 655 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 656 657 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 658 if (flg1) { 659 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 660 FILE *file; 661 if (mname[0]) { 662 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 663 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 664 file = fopen(fname,"w"); 665 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 666 } else file = PETSC_STDOUT; 667 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 668 } 669 670 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 671 if (flg4) { 672 if (format == PETSC_VIEWER_ASCII_XML) { 673 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 674 } else { 675 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 676 } 677 } 678 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 679 PetscReal threshold = PetscRealConstant(0.01); 680 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 681 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 682 } 683 #endif 684 685 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 686 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 687 /* 688 If collecting logging information, by default, wait for device to complete its operations 689 before returning to the CPU in order to get accurate timings of each event 690 */ 691 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 692 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 693 694 #if defined(PETSC_HAVE_CUDA) 695 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 696 #endif 697 698 #if defined(PETSC_HAVE_HIP) 699 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 700 #endif 701 702 /* 703 Print basic help message 704 */ 705 if (hasHelp) { 706 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 741 #if defined(PETSC_USE_LOG) 742 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 746 #if defined(PETSC_HAVE_MPE) 747 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 748 #endif 749 #endif 750 #if defined(PETSC_USE_INFO) 751 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 752 #endif 753 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 754 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 755 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 756 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 757 } 758 759 #if defined(PETSC_HAVE_POPEN) 760 { 761 char machine[128]; 762 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 763 if (flg1) { 764 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 765 } 766 } 767 #endif 768 769 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 770 if (flg1) { 771 ierr = PetscSleep(si);CHKERRQ(ierr); 772 } 773 774 #if defined(PETSC_HAVE_VIENNACL) 775 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 776 if (!flg3) { 777 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 778 } 779 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 780 PetscViennaCLSynchronize = flg3; 781 ierr = PetscViennaCLInit();CHKERRQ(ierr); 782 #endif 783 784 /* 785 Creates the logging data structures; this is enabled even if logging is not turned on 786 This is the last thing we do before returning to the user code to prevent having the 787 logging numbers contaminated by any startup time associated with MPI and the GPUs 788 */ 789 #if defined(PETSC_USE_LOG) 790 ierr = PetscLogInitialize();CHKERRQ(ierr); 791 #endif 792 793 PetscFunctionReturn(0); 794 } 795