1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 63 64 #if defined(PETSC_HAVE_COMPLEX) 65 #if defined(PETSC_COMPLEX_INSTANTIATE) 66 template <> class std::complex<double>; /* instantiate complex template class */ 67 #endif 68 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 69 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 70 MPI_Datatype MPIU_C_COMPLEX; 71 #endif 72 73 /*MC 74 PETSC_i - the imaginary number i 75 76 Synopsis: 77 #include <petscsys.h> 78 PetscComplex PETSC_i; 79 80 Level: beginner 81 82 Note: 83 Complex numbers are automatically available if PETSc located a working complex implementation 84 85 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 86 M*/ 87 PetscComplex PETSC_i; 88 #endif 89 #if defined(PETSC_USE_REAL___FLOAT128) 90 MPI_Datatype MPIU___FLOAT128 = 0; 91 #if defined(PETSC_HAVE_COMPLEX) 92 MPI_Datatype MPIU___COMPLEX128 = 0; 93 #endif 94 #elif defined(PETSC_USE_REAL___FP16) 95 MPI_Datatype MPIU___FP16 = 0; 96 #endif 97 MPI_Datatype MPIU_2SCALAR = 0; 98 #if defined(PETSC_USE_64BIT_INDICES) 99 MPI_Datatype MPIU_2INT = 0; 100 #endif 101 MPI_Datatype MPIU_BOOL; 102 MPI_Datatype MPIU_ENUM; 103 MPI_Datatype MPIU_FORTRANADDR; 104 MPI_Datatype MPIU_SIZE_T; 105 106 /* 107 Function that is called to display all error messages 108 */ 109 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 110 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 111 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 112 /* 113 This is needed to turn on/off GPU synchronization 114 */ 115 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 116 117 /* ------------------------------------------------------------------------------*/ 118 /* 119 Optional file where all PETSc output from various prints is saved 120 */ 121 PETSC_INTERN FILE *petsc_history; 122 FILE *petsc_history = NULL; 123 124 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 125 { 126 PetscErrorCode ierr; 127 PetscMPIInt rank,size; 128 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 129 char version[256]; 130 131 PetscFunctionBegin; 132 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 133 if (!rank) { 134 char arch[10]; 135 int err; 136 137 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 138 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 139 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 140 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 141 if (filename) { 142 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 143 } else { 144 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 146 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 147 } 148 149 *fd = fopen(fname,"a"); 150 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 151 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 153 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 154 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 156 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 157 158 err = fflush(*fd); 159 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 160 } 161 PetscFunctionReturn(0); 162 } 163 164 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 165 { 166 PetscErrorCode ierr; 167 PetscMPIInt rank; 168 char date[64]; 169 int err; 170 171 PetscFunctionBegin; 172 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 173 if (!rank) { 174 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 177 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 178 err = fflush(*fd); 179 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 180 err = fclose(*fd); 181 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 182 } 183 PetscFunctionReturn(0); 184 } 185 186 /* ------------------------------------------------------------------------------*/ 187 188 /* 189 This is ugly and probably belongs somewhere else, but I want to 190 be able to put a true MPI abort error handler with command line args. 191 192 This is so MPI errors in the debugger will leave all the stack 193 frames. The default MP_Abort() cleans up and exits thus providing no useful information 194 in the debugger hence we call abort() instead of MPI_Abort(). 195 */ 196 197 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 198 { 199 PetscFunctionBegin; 200 (*PetscErrorPrintf)("MPI error %d\n",*flag); 201 abort(); 202 } 203 204 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 205 { 206 PetscErrorCode ierr; 207 208 PetscFunctionBegin; 209 (*PetscErrorPrintf)("MPI error %d\n",*flag); 210 ierr = PetscAttachDebugger(); 211 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 212 } 213 214 /*@C 215 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 216 wishes a clean exit somewhere deep in the program. 217 218 Collective on PETSC_COMM_WORLD 219 220 Options Database Keys are the same as for PetscFinalize() 221 222 Level: advanced 223 224 Note: 225 See PetscInitialize() for more general runtime options. 226 227 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 228 @*/ 229 PetscErrorCode PetscEnd(void) 230 { 231 PetscFunctionBegin; 232 PetscFinalize(); 233 exit(0); 234 return 0; 235 } 236 237 PetscBool PetscOptionsPublish = PETSC_FALSE; 238 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 239 PETSC_INTERN PetscBool petscsetmallocvisited; 240 static char emacsmachinename[256]; 241 242 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 243 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 244 245 /*@C 246 PetscSetHelpVersionFunctions - Sets functions that print help and version information 247 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 248 This routine enables a "higher-level" package that uses PETSc to print its messages first. 249 250 Input Parameter: 251 + help - the help function (may be NULL) 252 - version - the version function (may be NULL) 253 254 Level: developer 255 256 @*/ 257 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 258 { 259 PetscFunctionBegin; 260 PetscExternalHelpFunction = help; 261 PetscExternalVersionFunction = version; 262 PetscFunctionReturn(0); 263 } 264 265 #if defined(PETSC_USE_LOG) 266 PETSC_INTERN PetscBool PetscObjectsLog; 267 #endif 268 269 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 270 { 271 if (PetscUnlikely(*err)) { 272 PETSC_UNUSED PetscMPIInt len; 273 char errstring[MPI_MAX_ERROR_STRING]; 274 275 MPI_Error_string(*err,errstring,&len); 276 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 277 } 278 return; 279 } 280 281 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 282 Use the following macros to define CUDA/HIP initialization related vars/routines. 283 */ 284 #if defined(PETSC_HAVE_CUDA) 285 typedef cudaError_t cupmError_t; 286 typedef struct cudaDeviceProp cupmDeviceProp; 287 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 288 #define cupmGetDevice(x) cudaGetDevice(x) 289 #define cupmSetDevice(x) cudaSetDevice(x) 290 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 291 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 292 #define cupmGetLastError() cudaGetLastError() 293 #define cupmDeviceMapHost cudaDeviceMapHost 294 #define cupmSuccess cudaSuccess 295 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 296 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 297 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 298 #define CHKERRCUPM(x) CHKERRCUDA(x) 299 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 300 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 301 #define PetscCUPMInitialize PetscCUDAInitialize 302 #define PetscCUPMInitialized PetscCUDAInitialized 303 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 304 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 305 #define PetscCUPMSynchronize PetscCUDASynchronize 306 #define PetscNotUseCUPM PetscNotUseCUDA 307 #define cupmOptionsStr "CUDA options" 308 #define cupmSetDeviceStr "-cuda_device" 309 #define cupmViewStr "-cuda_view" 310 #define cupmSynchronizeStr "-cuda_synchronize" 311 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 312 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 313 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 314 #include "cupminit.inc" 315 #endif 316 317 #if defined(PETSC_HAVE_HIP) 318 typedef hipError_t cupmError_t; 319 typedef hipDeviceProp_t cupmDeviceProp; 320 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 321 #define cupmGetDevice(x) hipGetDevice(x) 322 #define cupmSetDevice(x) hipSetDevice(x) 323 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 324 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 325 #define cupmGetLastError() hipGetLastError() 326 #define cupmDeviceMapHost hipDeviceMapHost 327 #define cupmSuccess hipSuccess 328 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 329 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 330 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 331 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 332 #define PetscCUPMBLASInitializeHandle() 0 333 #define PetscCUPMSOLVERDnInitializeHandle() 0 334 #define PetscCUPMInitialize PetscHIPInitialize 335 #define PetscCUPMInitialized PetscHIPInitialized 336 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 337 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 338 #define PetscCUPMSynchronize PetscHIPSynchronize 339 #define PetscNotUseCUPM PetscNotUseHIP 340 #define cupmOptionsStr "HIP options" 341 #define cupmSetDeviceStr "-hip_device" 342 #define cupmViewStr "-hip_view" 343 #define cupmSynchronizeStr "-hip_synchronize" 344 #define PetscCUPMInitializeStr "PetscHIPInitialize" 345 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 346 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 347 #include "cupminit.inc" 348 #endif 349 350 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 351 { 352 char string[64]; 353 MPI_Comm comm = PETSC_COMM_WORLD; 354 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 355 PetscErrorCode ierr; 356 PetscReal si; 357 PetscInt intensity; 358 int i; 359 PetscMPIInt rank; 360 char version[256]; 361 #if defined(PETSC_USE_LOG) 362 char mname[PETSC_MAX_PATH_LEN]; 363 PetscViewerFormat format; 364 PetscBool flg4 = PETSC_FALSE; 365 #endif 366 367 PetscFunctionBegin; 368 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 369 370 #if !defined(PETSC_HAVE_THREADSAFETY) 371 if (!(PETSC_RUNNING_ON_VALGRIND)) { 372 /* 373 Setup the memory management; support for tracing malloc() usage 374 */ 375 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 376 377 if (PetscDefined(USE_DEBUG)) { 378 mdebug = PETSC_TRUE; 379 initializenan = PETSC_TRUE; 380 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 381 } else { 382 /* don't warn about unused option */ 383 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 384 flg1 = PETSC_FALSE; 385 } 386 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 387 if (flg1 || flg2) { 388 mdebug = PETSC_TRUE; 389 eachcall = PETSC_TRUE; 390 initializenan = PETSC_TRUE; 391 } else if (flg3 && !flg2) { 392 mdebug = PETSC_FALSE; 393 eachcall = PETSC_FALSE; 394 initializenan = PETSC_FALSE; 395 } 396 397 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 398 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 399 400 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 401 if (mlog) { 402 mdebug = PETSC_TRUE; 403 } 404 /* the next line is deprecated */ 405 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 406 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 407 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 408 if (mdebug) { 409 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 410 } 411 if (mlog) { 412 PetscReal logthreshold = 0; 413 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 414 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 415 } 416 #if defined(PETSC_USE_LOG) 417 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 418 #endif 419 } 420 421 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 422 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 423 flg1 = PETSC_FALSE; 424 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 425 /* ignore this option if malloc is already set */ 426 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 427 428 flg1 = PETSC_FALSE; 429 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 430 if (!flg1) { 431 flg1 = PETSC_FALSE; 432 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 433 } 434 if (flg1) { 435 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 436 } 437 #endif 438 439 #if defined(PETSC_USE_LOG) 440 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 441 #endif 442 443 /* 444 Set the display variable for graphics 445 */ 446 ierr = PetscSetDisplay();CHKERRQ(ierr); 447 448 /* 449 Print main application help message 450 */ 451 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 452 if (help && hasHelp) { 453 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 454 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 455 } 456 457 /* 458 Print the PETSc version information 459 */ 460 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 461 if (flg1 || hasHelp) { 462 /* 463 Print "higher-level" package version message 464 */ 465 if (PetscExternalVersionFunction) { 466 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 467 } 468 469 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 477 } 478 479 /* 480 Print "higher-level" package help message 481 */ 482 if (hasHelp) { 483 PetscBool hasHelpIntro; 484 485 if (PetscExternalHelpFunction) { 486 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 487 } 488 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 489 if (hasHelpIntro) { 490 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 491 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 492 ierr = MPI_Finalize();CHKERRQ(ierr); 493 exit(0); 494 } 495 } 496 497 /* 498 Setup the error handling 499 */ 500 flg1 = PETSC_FALSE; 501 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 502 if (flg1) { 503 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 504 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 505 } 506 flg1 = PETSC_FALSE; 507 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 508 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 509 flg1 = PETSC_FALSE; 510 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 511 if (flg1) { 512 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 513 } 514 /* experimental */ 515 flg1 = PETSC_FALSE; 516 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 517 if (flg1) { 518 MPI_Errhandler eh; 519 520 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 521 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 522 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 523 } 524 flg1 = PETSC_FALSE; 525 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 526 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 527 528 /* 529 Setup debugger information 530 */ 531 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 532 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 533 if (flg1) { 534 MPI_Errhandler err_handler; 535 536 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 537 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 538 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 539 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 540 } 541 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 542 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 543 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 544 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 545 if (flg1 || flg2) { 546 PetscMPIInt size; 547 PetscInt lsize,*ranks; 548 MPI_Errhandler err_handler; 549 /* 550 we have to make sure that all processors have opened 551 connections to all other processors, otherwise once the 552 debugger has stated it is likely to receive a SIGUSR1 553 and kill the program. 554 */ 555 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 556 if (size > 2) { 557 PetscMPIInt dummy = 0; 558 MPI_Status status; 559 for (i=0; i<size; i++) { 560 if (rank != i) { 561 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 562 } 563 } 564 for (i=0; i<size; i++) { 565 if (rank != i) { 566 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 567 } 568 } 569 } 570 /* check if this processor node should be in debugger */ 571 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 572 lsize = size; 573 /* Deprecated in 3.14 */ 574 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 575 if (flag) { 576 const char * const quietopt="-options_suppress_deprecated_warnings"; 577 char msg[4096]; 578 PetscBool quiet = PETSC_FALSE; 579 580 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 581 if (!quiet) { 582 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 583 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 584 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 585 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 586 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 587 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 588 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 589 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 590 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 591 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 592 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 593 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 594 } 595 } else { 596 lsize = size; 597 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 598 } 599 if (flag) { 600 for (i=0; i<lsize; i++) { 601 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 602 } 603 } 604 if (!flag) { 605 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 606 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 607 if (flg1) { 608 ierr = PetscAttachDebugger();CHKERRQ(ierr); 609 } else { 610 ierr = PetscStopForDebugger();CHKERRQ(ierr); 611 } 612 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 613 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 614 } else { 615 ierr = PetscWaitOnError();CHKERRQ(ierr); 616 } 617 ierr = PetscFree(ranks);CHKERRQ(ierr); 618 } 619 620 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 621 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 622 623 /* 624 Setup profiling and logging 625 */ 626 #if defined(PETSC_USE_INFO) 627 { 628 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 629 } 630 #endif 631 ierr = PetscDetermineInitialFPTrap(); 632 flg1 = PETSC_FALSE; 633 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 634 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 635 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 636 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 637 #if defined(PETSC_USE_LOG) 638 mname[0] = 0; 639 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 640 if (flg1) { 641 if (mname[0]) { 642 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 643 } else { 644 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 645 } 646 } 647 648 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 649 650 #if defined(PETSC_HAVE_MPE) 651 flg1 = PETSC_FALSE; 652 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 653 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 654 #endif 655 flg1 = PETSC_FALSE; 656 flg3 = PETSC_FALSE; 657 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 658 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 659 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 660 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 661 662 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 663 if (flg1) { 664 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 665 FILE *file; 666 if (mname[0]) { 667 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 668 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 669 file = fopen(fname,"w"); 670 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 671 } else file = PETSC_STDOUT; 672 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 673 } 674 675 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 676 if (flg4) { 677 if (format == PETSC_VIEWER_ASCII_XML) { 678 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 679 } else { 680 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 681 } 682 } 683 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 684 PetscReal threshold = PetscRealConstant(0.01); 685 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 686 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 687 } 688 #endif 689 690 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 691 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 692 /* 693 If collecting logging information, by default, wait for device to complete its operations 694 before returning to the CPU in order to get accurate timings of each event 695 */ 696 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 697 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 698 699 #if defined(PETSC_HAVE_CUDA) 700 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 701 #endif 702 703 #if defined(PETSC_HAVE_HIP) 704 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 705 #endif 706 707 /* 708 Print basic help message 709 */ 710 if (hasHelp) { 711 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 746 #if defined(PETSC_USE_LOG) 747 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 749 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 750 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 751 #if defined(PETSC_HAVE_MPE) 752 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 753 #endif 754 #endif 755 #if defined(PETSC_USE_INFO) 756 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 757 #endif 758 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 759 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 760 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 761 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 762 } 763 764 #if defined(PETSC_HAVE_POPEN) 765 { 766 char machine[128]; 767 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 768 if (flg1) { 769 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 770 } 771 } 772 #endif 773 774 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 775 if (flg1) { 776 ierr = PetscSleep(si);CHKERRQ(ierr); 777 } 778 779 #if defined(PETSC_HAVE_VIENNACL) 780 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 781 if (!flg3) { 782 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 783 } 784 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 785 PetscViennaCLSynchronize = flg3; 786 ierr = PetscViennaCLInit();CHKERRQ(ierr); 787 #endif 788 789 /* 790 Creates the logging data structures; this is enabled even if logging is not turned on 791 This is the last thing we do before returning to the user code to prevent having the 792 logging numbers contaminated by any startup time associated with MPI and the GPUs 793 */ 794 #if defined(PETSC_USE_LOG) 795 ierr = PetscLogInitialize();CHKERRQ(ierr); 796 #endif 797 798 PetscFunctionReturn(0); 799 } 800