1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 63 #if defined(PETSC_HAVE_COMPLEX) 64 #if defined(PETSC_COMPLEX_INSTANTIATE) 65 template <> class std::complex<double>; /* instantiate complex template class */ 66 #endif 67 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 68 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 69 MPI_Datatype MPIU_C_COMPLEX; 70 #endif 71 72 /*MC 73 PETSC_i - the imaginary number i 74 75 Synopsis: 76 #include <petscsys.h> 77 PetscComplex PETSC_i; 78 79 Level: beginner 80 81 Note: 82 Complex numbers are automatically available if PETSc located a working complex implementation 83 84 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 85 M*/ 86 PetscComplex PETSC_i; 87 #endif 88 #if defined(PETSC_USE_REAL___FLOAT128) 89 MPI_Datatype MPIU___FLOAT128 = 0; 90 #if defined(PETSC_HAVE_COMPLEX) 91 MPI_Datatype MPIU___COMPLEX128 = 0; 92 #endif 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 #if defined(PETSC_USE_64BIT_INDICES) 98 MPI_Datatype MPIU_2INT = 0; 99 #endif 100 MPI_Datatype MPIU_BOOL; 101 MPI_Datatype MPIU_ENUM; 102 MPI_Datatype MPIU_FORTRANADDR; 103 MPI_Datatype MPIU_SIZE_T; 104 105 /* 106 Function that is called to display all error messages 107 */ 108 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 109 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 110 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 111 /* 112 This is needed to turn on/off GPU synchronization 113 */ 114 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 115 116 /* ------------------------------------------------------------------------------*/ 117 /* 118 Optional file where all PETSc output from various prints is saved 119 */ 120 PETSC_INTERN FILE *petsc_history; 121 FILE *petsc_history = NULL; 122 123 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 124 { 125 PetscErrorCode ierr; 126 PetscMPIInt rank,size; 127 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 128 char version[256]; 129 130 PetscFunctionBegin; 131 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 132 if (!rank) { 133 char arch[10]; 134 int err; 135 136 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 137 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 138 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 139 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 140 if (filename) { 141 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 142 } else { 143 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 146 } 147 148 *fd = fopen(fname,"a"); 149 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 150 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 153 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 157 err = fflush(*fd); 158 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 159 } 160 PetscFunctionReturn(0); 161 } 162 163 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 164 { 165 PetscErrorCode ierr; 166 PetscMPIInt rank; 167 char date[64]; 168 int err; 169 170 PetscFunctionBegin; 171 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 172 if (!rank) { 173 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 err = fflush(*fd); 178 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 179 err = fclose(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 181 } 182 PetscFunctionReturn(0); 183 } 184 185 /* ------------------------------------------------------------------------------*/ 186 187 /* 188 This is ugly and probably belongs somewhere else, but I want to 189 be able to put a true MPI abort error handler with command line args. 190 191 This is so MPI errors in the debugger will leave all the stack 192 frames. The default MP_Abort() cleans up and exits thus providing no useful information 193 in the debugger hence we call abort() instead of MPI_Abort(). 194 */ 195 196 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 197 { 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 abort(); 201 } 202 203 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 204 { 205 PetscErrorCode ierr; 206 207 PetscFunctionBegin; 208 (*PetscErrorPrintf)("MPI error %d\n",*flag); 209 ierr = PetscAttachDebugger(); 210 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 211 } 212 213 /*@C 214 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 215 wishes a clean exit somewhere deep in the program. 216 217 Collective on PETSC_COMM_WORLD 218 219 Options Database Keys are the same as for PetscFinalize() 220 221 Level: advanced 222 223 Note: 224 See PetscInitialize() for more general runtime options. 225 226 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 227 @*/ 228 PetscErrorCode PetscEnd(void) 229 { 230 PetscFunctionBegin; 231 PetscFinalize(); 232 exit(0); 233 return 0; 234 } 235 236 PetscBool PetscOptionsPublish = PETSC_FALSE; 237 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 238 PETSC_INTERN PetscBool petscsetmallocvisited; 239 static char emacsmachinename[256]; 240 241 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 242 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 243 244 /*@C 245 PetscSetHelpVersionFunctions - Sets functions that print help and version information 246 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 247 This routine enables a "higher-level" package that uses PETSc to print its messages first. 248 249 Input Parameter: 250 + help - the help function (may be NULL) 251 - version - the version function (may be NULL) 252 253 Level: developer 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #if defined(PETSC_USE_LOG) 265 PETSC_INTERN PetscBool PetscObjectsLog; 266 #endif 267 268 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 269 { 270 if (PetscUnlikely(*err)) { 271 PetscMPIInt len; 272 char errstring[MPI_MAX_ERROR_STRING]; 273 274 MPI_Error_string(*err,errstring,&len); 275 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 276 } 277 return; 278 } 279 280 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 281 Use the following macros to define CUDA/HIP initialization related vars/routines. 282 */ 283 #if defined(PETSC_HAVE_CUDA) 284 typedef cudaError_t cupmError_t; 285 typedef struct cudaDeviceProp cupmDeviceProp; 286 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 287 #define cupmGetDevice(x) cudaGetDevice(x) 288 #define cupmSetDevice(x) cudaSetDevice(x) 289 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 290 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 291 #define cupmGetLastError() cudaGetLastError() 292 #define cupmDeviceMapHost cudaDeviceMapHost 293 #define cupmSuccess cudaSuccess 294 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 295 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 296 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 297 #define CHKERRCUPM(x) CHKERRCUDA(x) 298 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 299 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 300 #define PetscCUPMInitialize PetscCUDAInitialize 301 #define PetscCUPMInitialized PetscCUDAInitialized 302 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 303 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 304 #define PetscCUPMSynchronize PetscCUDASynchronize 305 #define PetscNotUseCUPM PetscNotUseCUDA 306 #define cupmOptionsStr "CUDA options" 307 #define cupmSetDeviceStr "-cuda_device" 308 #define cupmViewStr "-cuda_view" 309 #define cupmSynchronizeStr "-cuda_synchronize" 310 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 311 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 312 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 313 #include "cupminit.inc" 314 #endif 315 316 #if defined(PETSC_HAVE_HIP) 317 typedef hipError_t cupmError_t; 318 typedef hipDeviceProp_t cupmDeviceProp; 319 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 320 #define cupmGetDevice(x) hipGetDevice(x) 321 #define cupmSetDevice(x) hipSetDevice(x) 322 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 323 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 324 #define cupmGetLastError() hipGetLastError() 325 #define cupmDeviceMapHost hipDeviceMapHost 326 #define cupmSuccess hipSuccess 327 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 328 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 329 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 330 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 331 #define PetscCUPMBLASInitializeHandle() 0 332 #define PetscCUPMSOLVERDnInitializeHandle() 0 333 #define PetscCUPMInitialize PetscHIPInitialize 334 #define PetscCUPMInitialized PetscHIPInitialized 335 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 336 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 337 #define PetscCUPMSynchronize PetscHIPSynchronize 338 #define PetscNotUseCUPM PetscNotUseHIP 339 #define cupmOptionsStr "HIP options" 340 #define cupmSetDeviceStr "-hip_device" 341 #define cupmViewStr "-hip_view" 342 #define cupmSynchronizeStr "-hip_synchronize" 343 #define PetscCUPMInitializeStr "PetscHIPInitialize" 344 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 345 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 346 #include "cupminit.inc" 347 #endif 348 349 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 350 { 351 char string[64]; 352 MPI_Comm comm = PETSC_COMM_WORLD; 353 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 354 PetscErrorCode ierr; 355 PetscReal si; 356 PetscInt intensity; 357 int i; 358 PetscMPIInt rank; 359 char version[256]; 360 #if defined(PETSC_USE_LOG) 361 char mname[PETSC_MAX_PATH_LEN]; 362 PetscViewerFormat format; 363 PetscBool flg4 = PETSC_FALSE; 364 #endif 365 366 PetscFunctionBegin; 367 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 368 369 #if !defined(PETSC_HAVE_THREADSAFETY) 370 if (!(PETSC_RUNNING_ON_VALGRIND)) { 371 /* 372 Setup the memory management; support for tracing malloc() usage 373 */ 374 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 375 376 if (PetscDefined(USE_DEBUG)) { 377 mdebug = PETSC_TRUE; 378 initializenan = PETSC_TRUE; 379 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 380 } else { 381 /* don't warn about unused option */ 382 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 383 flg1 = PETSC_FALSE; 384 } 385 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 386 if (flg1 || flg2) { 387 mdebug = PETSC_TRUE; 388 eachcall = PETSC_TRUE; 389 initializenan = PETSC_TRUE; 390 } else if (flg3 && !flg2) { 391 mdebug = PETSC_FALSE; 392 eachcall = PETSC_FALSE; 393 initializenan = PETSC_FALSE; 394 } 395 396 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 397 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 398 399 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 400 if (mlog) { 401 mdebug = PETSC_TRUE; 402 } 403 /* the next line is deprecated */ 404 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 405 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 406 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 407 if (mdebug) { 408 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 409 } 410 if (mlog) { 411 PetscReal logthreshold = 0; 412 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 413 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 414 } 415 #if defined(PETSC_USE_LOG) 416 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 417 #endif 418 } 419 420 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 421 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 422 flg1 = PETSC_FALSE; 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 424 /* ignore this option if malloc is already set */ 425 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 426 427 flg1 = PETSC_FALSE; 428 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 429 if (!flg1) { 430 flg1 = PETSC_FALSE; 431 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 432 } 433 if (flg1) { 434 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 435 } 436 #endif 437 438 #if defined(PETSC_USE_LOG) 439 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 440 #endif 441 442 /* 443 Set the display variable for graphics 444 */ 445 ierr = PetscSetDisplay();CHKERRQ(ierr); 446 447 /* 448 Print main application help message 449 */ 450 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 451 if (help && hasHelp) { 452 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 453 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 454 } 455 456 /* 457 Print the PETSc version information 458 */ 459 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 460 if (flg1 || hasHelp) { 461 /* 462 Print "higher-level" package version message 463 */ 464 if (PetscExternalVersionFunction) { 465 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 466 } 467 468 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 476 } 477 478 /* 479 Print "higher-level" package help message 480 */ 481 if (hasHelp) { 482 PetscBool hasHelpIntro; 483 484 if (PetscExternalHelpFunction) { 485 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 486 } 487 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 488 if (hasHelpIntro) { 489 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 490 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 491 ierr = MPI_Finalize();CHKERRQ(ierr); 492 exit(0); 493 } 494 } 495 496 /* 497 Setup the error handling 498 */ 499 flg1 = PETSC_FALSE; 500 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 501 if (flg1) { 502 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 503 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 504 } 505 flg1 = PETSC_FALSE; 506 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 507 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 508 flg1 = PETSC_FALSE; 509 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 510 if (flg1) { 511 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 512 } 513 /* experimental */ 514 flg1 = PETSC_FALSE; 515 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 516 if (flg1) { 517 MPI_Errhandler eh; 518 519 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 520 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 521 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 522 } 523 flg1 = PETSC_FALSE; 524 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 525 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 526 527 /* 528 Setup debugger information 529 */ 530 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 531 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 532 if (flg1) { 533 MPI_Errhandler err_handler; 534 535 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 536 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 537 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 538 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 539 } 540 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 541 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 542 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 543 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 544 if (flg1 || flg2) { 545 PetscMPIInt size; 546 PetscInt lsize,*ranks; 547 MPI_Errhandler err_handler; 548 /* 549 we have to make sure that all processors have opened 550 connections to all other processors, otherwise once the 551 debugger has stated it is likely to receive a SIGUSR1 552 and kill the program. 553 */ 554 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 555 if (size > 2) { 556 PetscMPIInt dummy = 0; 557 MPI_Status status; 558 for (i=0; i<size; i++) { 559 if (rank != i) { 560 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 561 } 562 } 563 for (i=0; i<size; i++) { 564 if (rank != i) { 565 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 566 } 567 } 568 } 569 /* check if this processor node should be in debugger */ 570 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 571 lsize = size; 572 /* Deprecated in 3.14 */ 573 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 574 if (flag) { 575 const char * const quietopt="-options_suppress_deprecated_warnings"; 576 char msg[4096]; 577 PetscBool quiet = PETSC_FALSE; 578 579 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 580 if (!quiet) { 581 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 582 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 583 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 584 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 585 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 586 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 587 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 588 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 589 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 590 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 591 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 592 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 593 } 594 } else { 595 lsize = size; 596 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 597 } 598 if (flag) { 599 for (i=0; i<lsize; i++) { 600 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 601 } 602 } 603 if (!flag) { 604 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 605 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 606 if (flg1) { 607 ierr = PetscAttachDebugger();CHKERRQ(ierr); 608 } else { 609 ierr = PetscStopForDebugger();CHKERRQ(ierr); 610 } 611 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 612 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 613 } else { 614 ierr = PetscWaitOnError();CHKERRQ(ierr); 615 } 616 ierr = PetscFree(ranks);CHKERRQ(ierr); 617 } 618 619 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 620 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 621 622 /* 623 Setup profiling and logging 624 */ 625 #if defined(PETSC_USE_INFO) 626 { 627 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 628 } 629 #endif 630 ierr = PetscDetermineInitialFPTrap(); 631 flg1 = PETSC_FALSE; 632 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 633 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 634 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 635 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 636 #if defined(PETSC_USE_LOG) 637 mname[0] = 0; 638 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 639 if (flg1) { 640 if (mname[0]) { 641 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 642 } else { 643 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 644 } 645 } 646 647 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 648 649 #if defined(PETSC_HAVE_MPE) 650 flg1 = PETSC_FALSE; 651 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 652 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 653 #endif 654 flg1 = PETSC_FALSE; 655 flg3 = PETSC_FALSE; 656 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 657 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 658 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 659 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 660 661 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 662 if (flg1) { 663 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 664 FILE *file; 665 if (mname[0]) { 666 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 667 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 668 file = fopen(fname,"w"); 669 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 670 } else file = PETSC_STDOUT; 671 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 672 } 673 674 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 675 if (flg4) { 676 if (format == PETSC_VIEWER_ASCII_XML) { 677 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 678 } else { 679 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 680 } 681 } 682 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 683 PetscReal threshold = PetscRealConstant(0.01); 684 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 685 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 686 } 687 #endif 688 689 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 690 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 691 /* 692 If collecting logging information, by default, wait for device to complete its operations 693 before returning to the CPU in order to get accurate timings of each event 694 */ 695 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 696 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 697 698 #if defined(PETSC_HAVE_CUDA) 699 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 700 #endif 701 702 #if defined(PETSC_HAVE_HIP) 703 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 704 #endif 705 706 /* 707 Print basic help message 708 */ 709 if (hasHelp) { 710 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 745 #if defined(PETSC_USE_LOG) 746 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 747 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 749 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 750 #if defined(PETSC_HAVE_MPE) 751 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 752 #endif 753 #endif 754 #if defined(PETSC_USE_INFO) 755 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 756 #endif 757 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 758 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 759 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 760 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 761 } 762 763 #if defined(PETSC_HAVE_POPEN) 764 { 765 char machine[128]; 766 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 767 if (flg1) { 768 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 769 } 770 } 771 #endif 772 773 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 774 if (flg1) { 775 ierr = PetscSleep(si);CHKERRQ(ierr); 776 } 777 778 #if defined(PETSC_HAVE_VIENNACL) 779 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 780 if (!flg3) { 781 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 782 } 783 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 784 PetscViennaCLSynchronize = flg3; 785 ierr = PetscViennaCLInit();CHKERRQ(ierr); 786 #endif 787 788 /* 789 Creates the logging data structures; this is enabled even if logging is not turned on 790 This is the last thing we do before returning to the user code to prevent having the 791 logging numbers contaminated by any startup time associated with MPI and the GPUs 792 */ 793 #if defined(PETSC_USE_LOG) 794 ierr = PetscLogInitialize();CHKERRQ(ierr); 795 #endif 796 797 PetscFunctionReturn(0); 798 } 799