1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 63 #if defined(PETSC_HAVE_COMPLEX) 64 #if defined(PETSC_COMPLEX_INSTANTIATE) 65 template <> class std::complex<double>; /* instantiate complex template class */ 66 #endif 67 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 68 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 69 MPI_Datatype MPIU_C_COMPLEX; 70 #endif 71 72 /*MC 73 PETSC_i - the imaginary number i 74 75 Synopsis: 76 #include <petscsys.h> 77 PetscComplex PETSC_i; 78 79 Level: beginner 80 81 Note: 82 Complex numbers are automatically available if PETSc located a working complex implementation 83 84 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 85 M*/ 86 PetscComplex PETSC_i; 87 #endif 88 #if defined(PETSC_USE_REAL___FLOAT128) 89 MPI_Datatype MPIU___FLOAT128 = 0; 90 #if defined(PETSC_HAVE_COMPLEX) 91 MPI_Datatype MPIU___COMPLEX128 = 0; 92 #endif 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 #if defined(PETSC_USE_64BIT_INDICES) 98 MPI_Datatype MPIU_2INT = 0; 99 #endif 100 MPI_Datatype MPIU_BOOL; 101 MPI_Datatype MPIU_ENUM; 102 MPI_Datatype MPIU_FORTRANADDR; 103 MPI_Datatype MPIU_SIZE_T; 104 105 /* 106 Function that is called to display all error messages 107 */ 108 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 109 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 110 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 111 /* 112 This is needed to turn on/off GPU synchronization 113 */ 114 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 115 116 /* ------------------------------------------------------------------------------*/ 117 /* 118 Optional file where all PETSc output from various prints is saved 119 */ 120 PETSC_INTERN FILE *petsc_history; 121 FILE *petsc_history = NULL; 122 123 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 124 { 125 PetscErrorCode ierr; 126 PetscMPIInt rank,size; 127 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 128 char version[256]; 129 130 PetscFunctionBegin; 131 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 132 if (!rank) { 133 char arch[10]; 134 int err; 135 136 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 137 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 138 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 139 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 140 if (filename) { 141 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 142 } else { 143 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 146 } 147 148 *fd = fopen(fname,"a"); 149 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 150 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 153 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 157 err = fflush(*fd); 158 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 159 } 160 PetscFunctionReturn(0); 161 } 162 163 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 164 { 165 PetscErrorCode ierr; 166 PetscMPIInt rank; 167 char date[64]; 168 int err; 169 170 PetscFunctionBegin; 171 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 172 if (!rank) { 173 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 err = fflush(*fd); 178 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 179 err = fclose(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 181 } 182 PetscFunctionReturn(0); 183 } 184 185 /* ------------------------------------------------------------------------------*/ 186 187 /* 188 This is ugly and probably belongs somewhere else, but I want to 189 be able to put a true MPI abort error handler with command line args. 190 191 This is so MPI errors in the debugger will leave all the stack 192 frames. The default MP_Abort() cleans up and exits thus providing no useful information 193 in the debugger hence we call abort() instead of MPI_Abort(). 194 */ 195 196 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 197 { 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 abort(); 201 } 202 203 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 204 { 205 PetscErrorCode ierr; 206 207 PetscFunctionBegin; 208 (*PetscErrorPrintf)("MPI error %d\n",*flag); 209 ierr = PetscAttachDebugger(); 210 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 211 } 212 213 /*@C 214 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 215 wishes a clean exit somewhere deep in the program. 216 217 Collective on PETSC_COMM_WORLD 218 219 Options Database Keys are the same as for PetscFinalize() 220 221 Level: advanced 222 223 Note: 224 See PetscInitialize() for more general runtime options. 225 226 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 227 @*/ 228 PetscErrorCode PetscEnd(void) 229 { 230 PetscFunctionBegin; 231 PetscFinalize(); 232 exit(0); 233 return 0; 234 } 235 236 PetscBool PetscOptionsPublish = PETSC_FALSE; 237 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 238 PETSC_INTERN PetscBool petscsetmallocvisited; 239 static char emacsmachinename[256]; 240 241 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 242 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 243 244 /*@C 245 PetscSetHelpVersionFunctions - Sets functions that print help and version information 246 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 247 This routine enables a "higher-level" package that uses PETSc to print its messages first. 248 249 Input Parameter: 250 + help - the help function (may be NULL) 251 - version - the version function (may be NULL) 252 253 Level: developer 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #if defined(PETSC_USE_LOG) 265 PETSC_INTERN PetscBool PetscObjectsLog; 266 #endif 267 268 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 269 { 270 if (PetscUnlikely(*err)) { 271 PetscMPIInt len; 272 char errstring[MPI_MAX_ERROR_STRING]; 273 274 MPI_Error_string(*err,errstring,&len); 275 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 276 } 277 return; 278 } 279 280 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 281 Use the following macros to define CUDA/HIP initialization related vars/routines. 282 */ 283 #if defined(PETSC_HAVE_CUDA) 284 typedef cudaError_t cupmError_t; 285 typedef struct cudaDeviceProp cupmDeviceProp; 286 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 287 #define cupmGetDevice(x) cudaGetDevice(x) 288 #define cupmSetDevice(x) cudaSetDevice(x) 289 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 290 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 291 #define cupmGetLastError() cudaGetLastError() 292 #define cupmDeviceMapHost cudaDeviceMapHost 293 #define cupmSuccess cudaSuccess 294 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 295 #define CHKERRCUPM(x) CHKERRCUDA(x) 296 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 297 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 298 #define PetscCUPMInitialize PetscCUDAInitialize 299 #define PetscCUPMInitialized PetscCUDAInitialized 300 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 301 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 302 #define PetscCUPMSynchronize PetscCUDASynchronize 303 #define PetscNotUseCUPM PetscNotUseCUDA 304 #define cupmOptionsStr "CUDA options" 305 #define cupmSetDeviceStr "-cuda_set_device" 306 #define cupmViewStr "-cuda_view" 307 #define cupmSynchronizeStr "-cuda_synchronize" 308 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 309 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 310 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 311 #include "cupminit.inc" 312 #endif 313 314 #if defined(PETSC_HAVE_HIP) 315 typedef hipError_t cupmError_t; 316 typedef hipDeviceProp_t cupmDeviceProp; 317 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 318 #define cupmGetDevice(x) hipGetDevice(x) 319 #define cupmSetDevice(x) hipSetDevice(x) 320 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 321 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 322 #define cupmGetLastError() hipGetLastError() 323 #define cupmDeviceMapHost hipDeviceMapHost 324 #define cupmSuccess hipSuccess 325 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 326 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 327 #define PetscCUPMBLASInitializeHandle() 0 328 #define PetscCUPMSOLVERDnInitializeHandle() 0 329 #define PetscCUPMInitialize PetscHIPInitialize 330 #define PetscCUPMInitialized PetscHIPInitialized 331 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 332 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 333 #define PetscCUPMSynchronize PetscHIPSynchronize 334 #define PetscNotUseCUPM PetscNotUseHIP 335 #define cupmOptionsStr "HIP options" 336 #define cupmSetDeviceStr "-hip_set_device" 337 #define cupmViewStr "-hip_view" 338 #define cupmSynchronizeStr "-hip_synchronize" 339 #define PetscCUPMInitializeStr "PetscHIPInitialize" 340 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 341 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 342 #include "cupminit.inc" 343 #endif 344 345 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 346 { 347 char string[64]; 348 MPI_Comm comm = PETSC_COMM_WORLD; 349 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 350 PetscErrorCode ierr; 351 PetscReal si; 352 PetscInt intensity; 353 int i; 354 PetscMPIInt rank; 355 char version[256]; 356 #if defined(PETSC_USE_LOG) 357 char mname[PETSC_MAX_PATH_LEN]; 358 PetscViewerFormat format; 359 PetscBool flg4 = PETSC_FALSE; 360 #endif 361 362 PetscFunctionBegin; 363 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 364 365 #if !defined(PETSC_HAVE_THREADSAFETY) 366 if (!(PETSC_RUNNING_ON_VALGRIND)) { 367 /* 368 Setup the memory management; support for tracing malloc() usage 369 */ 370 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 371 372 if (PetscDefined(USE_DEBUG)) { 373 mdebug = PETSC_TRUE; 374 initializenan = PETSC_TRUE; 375 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 376 } else { 377 /* don't warn about unused option */ 378 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 379 flg1 = PETSC_FALSE; 380 } 381 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 382 if (flg1 || flg2) { 383 mdebug = PETSC_TRUE; 384 eachcall = PETSC_TRUE; 385 initializenan = PETSC_TRUE; 386 } else if (flg3 && !flg2) { 387 mdebug = PETSC_FALSE; 388 eachcall = PETSC_FALSE; 389 initializenan = PETSC_FALSE; 390 } 391 392 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 393 if (mlog) { 394 mdebug = PETSC_TRUE; 395 } 396 /* the next line is deprecated */ 397 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 398 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 399 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 400 if (mdebug) { 401 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 402 } 403 if (mlog) { 404 PetscReal logthreshold = 0; 405 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 406 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 407 } 408 #if defined(PETSC_USE_LOG) 409 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 410 #endif 411 } 412 413 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 414 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 415 flg1 = PETSC_FALSE; 416 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 417 /* ignore this option if malloc is already set */ 418 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 419 420 flg1 = PETSC_FALSE; 421 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 422 if (!flg1) { 423 flg1 = PETSC_FALSE; 424 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 425 } 426 if (flg1) { 427 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 428 } 429 #endif 430 431 #if defined(PETSC_USE_LOG) 432 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 433 #endif 434 435 /* 436 Set the display variable for graphics 437 */ 438 ierr = PetscSetDisplay();CHKERRQ(ierr); 439 440 /* 441 Print main application help message 442 */ 443 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 444 if (help && hasHelp) { 445 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 446 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 447 } 448 449 /* 450 Print the PETSc version information 451 */ 452 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 453 if (flg1 || hasHelp) { 454 /* 455 Print "higher-level" package version message 456 */ 457 if (PetscExternalVersionFunction) { 458 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 459 } 460 461 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 462 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 463 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 464 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 465 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 469 } 470 471 /* 472 Print "higher-level" package help message 473 */ 474 if (hasHelp) { 475 PetscBool hasHelpIntro; 476 477 if (PetscExternalHelpFunction) { 478 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 479 } 480 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 481 if (hasHelpIntro) { 482 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 483 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 484 ierr = MPI_Finalize();CHKERRQ(ierr); 485 exit(0); 486 } 487 } 488 489 /* 490 Setup the error handling 491 */ 492 flg1 = PETSC_FALSE; 493 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 494 if (flg1) { 495 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 496 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 497 } 498 flg1 = PETSC_FALSE; 499 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 500 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 501 flg1 = PETSC_FALSE; 502 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 503 if (flg1) { 504 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 505 } 506 /* experimental */ 507 flg1 = PETSC_FALSE; 508 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 509 if (flg1) { 510 MPI_Errhandler eh; 511 512 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 513 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 514 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 515 } 516 flg1 = PETSC_FALSE; 517 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 518 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 519 520 /* 521 Setup debugger information 522 */ 523 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 524 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 525 if (flg1) { 526 MPI_Errhandler err_handler; 527 528 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 529 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 530 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 531 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 532 } 533 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 534 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 535 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 536 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 537 if (flg1 || flg2) { 538 PetscMPIInt size; 539 PetscInt lsize,*nodes; 540 MPI_Errhandler err_handler; 541 /* 542 we have to make sure that all processors have opened 543 connections to all other processors, otherwise once the 544 debugger has stated it is likely to receive a SIGUSR1 545 and kill the program. 546 */ 547 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 548 if (size > 2) { 549 PetscMPIInt dummy = 0; 550 MPI_Status status; 551 for (i=0; i<size; i++) { 552 if (rank != i) { 553 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 554 } 555 } 556 for (i=0; i<size; i++) { 557 if (rank != i) { 558 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 559 } 560 } 561 } 562 /* check if this processor node should be in debugger */ 563 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 564 lsize = size; 565 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 566 if (flag) { 567 for (i=0; i<lsize; i++) { 568 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 569 } 570 } 571 if (!flag) { 572 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 573 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 574 if (flg1) { 575 ierr = PetscAttachDebugger();CHKERRQ(ierr); 576 } else { 577 ierr = PetscStopForDebugger();CHKERRQ(ierr); 578 } 579 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 580 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 581 } 582 ierr = PetscFree(nodes);CHKERRQ(ierr); 583 } 584 585 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 586 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 587 588 /* 589 Setup profiling and logging 590 */ 591 #if defined(PETSC_USE_INFO) 592 { 593 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 594 } 595 #endif 596 ierr = PetscDetermineInitialFPTrap(); 597 flg1 = PETSC_FALSE; 598 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 599 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 600 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 601 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 602 #if defined(PETSC_USE_LOG) 603 mname[0] = 0; 604 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 605 if (flg1) { 606 if (mname[0]) { 607 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 608 } else { 609 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 610 } 611 } 612 613 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 614 615 #if defined(PETSC_HAVE_MPE) 616 flg1 = PETSC_FALSE; 617 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 618 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 619 #endif 620 flg1 = PETSC_FALSE; 621 flg3 = PETSC_FALSE; 622 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 623 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 624 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 625 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 626 627 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 628 if (flg1) { 629 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 630 FILE *file; 631 if (mname[0]) { 632 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 633 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 634 file = fopen(fname,"w"); 635 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 636 } else file = PETSC_STDOUT; 637 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 638 } 639 640 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 641 if (flg4) { 642 if (format == PETSC_VIEWER_ASCII_XML) { 643 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 644 } else { 645 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 646 } 647 } 648 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 649 PetscReal threshold = PetscRealConstant(0.01); 650 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 651 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 652 } 653 #endif 654 655 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 656 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 657 /* 658 If collecting logging information, by default, wait for device to complete its operations 659 before returning to the CPU in order to get accurate timings of each event 660 */ 661 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 662 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 663 664 #if defined(PETSC_HAVE_CUDA) 665 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 666 #endif 667 668 #if defined(PETSC_HAVE_HIP) 669 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 670 #endif 671 672 /* 673 Print basic help message 674 */ 675 if (hasHelp) { 676 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 677 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 678 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 679 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 680 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 681 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 682 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 683 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 684 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 685 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 686 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 687 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 688 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 689 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 690 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 693 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 694 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 711 #if defined(PETSC_USE_LOG) 712 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 716 #if defined(PETSC_HAVE_MPE) 717 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 718 #endif 719 #endif 720 #if defined(PETSC_USE_INFO) 721 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 722 #endif 723 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 727 } 728 729 #if defined(PETSC_HAVE_POPEN) 730 { 731 char machine[128]; 732 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 733 if (flg1) { 734 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 735 } 736 } 737 #endif 738 739 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 740 if (flg1) { 741 ierr = PetscSleep(si);CHKERRQ(ierr); 742 } 743 744 #if defined(PETSC_HAVE_VIENNACL) 745 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 746 if (!flg3) { 747 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 748 } 749 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 750 PetscViennaCLSynchronize = flg3; 751 ierr = PetscViennaCLInit();CHKERRQ(ierr); 752 #endif 753 754 /* 755 Creates the logging data structures; this is enabled even if logging is not turned on 756 This is the last thing we do before returning to the user code to prevent having the 757 logging numbers contaminated by any startup time associated with MPI and the GPUs 758 */ 759 #if defined(PETSC_USE_LOG) 760 ierr = PetscLogInitialize();CHKERRQ(ierr); 761 #endif 762 763 PetscFunctionReturn(0); 764 } 765