1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 63 64 #if defined(PETSC_HAVE_COMPLEX) 65 #if defined(PETSC_COMPLEX_INSTANTIATE) 66 template <> class std::complex<double>; /* instantiate complex template class */ 67 #endif 68 69 /*MC 70 PETSC_i - the imaginary number i 71 72 Synopsis: 73 #include <petscsys.h> 74 PetscComplex PETSC_i; 75 76 Level: beginner 77 78 Note: 79 Complex numbers are automatically available if PETSc located a working complex implementation 80 81 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 82 M*/ 83 PetscComplex PETSC_i; 84 MPI_Datatype MPIU___COMPLEX128 = 0; 85 #endif /* PETSC_HAVE_COMPLEX */ 86 #if defined(PETSC_USE_REAL___FLOAT128) 87 MPI_Datatype MPIU___FLOAT128 = 0; 88 #elif defined(PETSC_USE_REAL___FP16) 89 MPI_Datatype MPIU___FP16 = 0; 90 #endif 91 MPI_Datatype MPIU_2SCALAR = 0; 92 #if defined(PETSC_USE_64BIT_INDICES) 93 MPI_Datatype MPIU_2INT = 0; 94 #endif 95 MPI_Datatype MPIU_BOOL; 96 MPI_Datatype MPIU_ENUM; 97 MPI_Datatype MPIU_FORTRANADDR; 98 MPI_Datatype MPIU_SIZE_T; 99 100 /* 101 Function that is called to display all error messages 102 */ 103 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 104 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 105 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 106 /* 107 This is needed to turn on/off GPU synchronization 108 */ 109 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 110 111 /* ------------------------------------------------------------------------------*/ 112 /* 113 Optional file where all PETSc output from various prints is saved 114 */ 115 PETSC_INTERN FILE *petsc_history; 116 FILE *petsc_history = NULL; 117 118 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 119 { 120 PetscErrorCode ierr; 121 PetscMPIInt rank,size; 122 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 123 char version[256]; 124 125 PetscFunctionBegin; 126 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 127 if (!rank) { 128 char arch[10]; 129 int err; 130 131 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 132 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 133 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 134 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 135 if (filename) { 136 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 137 } else { 138 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 139 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 140 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 141 } 142 143 *fd = fopen(fname,"a"); 144 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 145 146 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 147 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 148 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 149 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 150 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 151 152 err = fflush(*fd); 153 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 154 } 155 PetscFunctionReturn(0); 156 } 157 158 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 159 { 160 PetscErrorCode ierr; 161 PetscMPIInt rank; 162 char date[64]; 163 int err; 164 165 PetscFunctionBegin; 166 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 167 if (!rank) { 168 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 169 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 170 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 171 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 172 err = fflush(*fd); 173 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 174 err = fclose(*fd); 175 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 176 } 177 PetscFunctionReturn(0); 178 } 179 180 /* ------------------------------------------------------------------------------*/ 181 182 /* 183 This is ugly and probably belongs somewhere else, but I want to 184 be able to put a true MPI abort error handler with command line args. 185 186 This is so MPI errors in the debugger will leave all the stack 187 frames. The default MP_Abort() cleans up and exits thus providing no useful information 188 in the debugger hence we call abort() instead of MPI_Abort(). 189 */ 190 191 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 192 { 193 PetscFunctionBegin; 194 (*PetscErrorPrintf)("MPI error %d\n",*flag); 195 abort(); 196 } 197 198 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 199 { 200 PetscErrorCode ierr; 201 202 PetscFunctionBegin; 203 (*PetscErrorPrintf)("MPI error %d\n",*flag); 204 ierr = PetscAttachDebugger(); 205 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 206 } 207 208 /*@C 209 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 210 wishes a clean exit somewhere deep in the program. 211 212 Collective on PETSC_COMM_WORLD 213 214 Options Database Keys are the same as for PetscFinalize() 215 216 Level: advanced 217 218 Note: 219 See PetscInitialize() for more general runtime options. 220 221 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 222 @*/ 223 PetscErrorCode PetscEnd(void) 224 { 225 PetscFunctionBegin; 226 PetscFinalize(); 227 exit(0); 228 return 0; 229 } 230 231 PetscBool PetscOptionsPublish = PETSC_FALSE; 232 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 233 PETSC_INTERN PetscBool petscsetmallocvisited; 234 static char emacsmachinename[256]; 235 236 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 237 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 238 239 /*@C 240 PetscSetHelpVersionFunctions - Sets functions that print help and version information 241 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 242 This routine enables a "higher-level" package that uses PETSc to print its messages first. 243 244 Input Parameter: 245 + help - the help function (may be NULL) 246 - version - the version function (may be NULL) 247 248 Level: developer 249 250 @*/ 251 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 252 { 253 PetscFunctionBegin; 254 PetscExternalHelpFunction = help; 255 PetscExternalVersionFunction = version; 256 PetscFunctionReturn(0); 257 } 258 259 #if defined(PETSC_USE_LOG) 260 PETSC_INTERN PetscBool PetscObjectsLog; 261 #endif 262 263 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 264 Use the following macros to define CUDA/HIP initialization related vars/routines. 265 */ 266 #if defined(PETSC_HAVE_CUDA) 267 typedef cudaError_t cupmError_t; 268 typedef struct cudaDeviceProp cupmDeviceProp; 269 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 270 #define cupmGetDevice(x) cudaGetDevice(x) 271 #define cupmSetDevice(x) cudaSetDevice(x) 272 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 273 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 274 #define cupmGetLastError() cudaGetLastError() 275 #define cupmDeviceMapHost cudaDeviceMapHost 276 #define cupmSuccess cudaSuccess 277 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 278 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 279 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 280 #define CHKERRCUPM(x) CHKERRCUDA(x) 281 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 282 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 283 #define PetscCUPMInitialize PetscCUDAInitialize 284 #define PetscCUPMInitialized PetscCUDAInitialized 285 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 286 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 287 #define PetscCUPMSynchronize PetscCUDASynchronize 288 #define PetscNotUseCUPM PetscNotUseCUDA 289 #define cupmOptionsStr "CUDA options" 290 #define cupmSetDeviceStr "-cuda_device" 291 #define cupmViewStr "-cuda_view" 292 #define cupmSynchronizeStr "-cuda_synchronize" 293 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 294 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 295 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 296 #include "cupminit.inc" 297 #endif 298 299 #if defined(PETSC_HAVE_HIP) 300 typedef hipError_t cupmError_t; 301 typedef hipDeviceProp_t cupmDeviceProp; 302 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 303 #define cupmGetDevice(x) hipGetDevice(x) 304 #define cupmSetDevice(x) hipSetDevice(x) 305 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 306 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 307 #define cupmGetLastError() hipGetLastError() 308 #define cupmDeviceMapHost hipDeviceMapHost 309 #define cupmSuccess hipSuccess 310 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 311 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 312 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 313 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 314 #define PetscCUPMBLASInitializeHandle() 0 315 #define PetscCUPMSOLVERDnInitializeHandle() 0 316 #define PetscCUPMInitialize PetscHIPInitialize 317 #define PetscCUPMInitialized PetscHIPInitialized 318 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 319 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 320 #define PetscCUPMSynchronize PetscHIPSynchronize 321 #define PetscNotUseCUPM PetscNotUseHIP 322 #define cupmOptionsStr "HIP options" 323 #define cupmSetDeviceStr "-hip_device" 324 #define cupmViewStr "-hip_view" 325 #define cupmSynchronizeStr "-hip_synchronize" 326 #define PetscCUPMInitializeStr "PetscHIPInitialize" 327 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 328 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 329 #include "cupminit.inc" 330 #endif 331 332 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 333 { 334 char string[64]; 335 MPI_Comm comm = PETSC_COMM_WORLD; 336 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 337 PetscErrorCode ierr; 338 PetscReal si; 339 PetscInt intensity; 340 int i; 341 PetscMPIInt rank; 342 char version[256]; 343 #if defined(PETSC_USE_LOG) 344 char mname[PETSC_MAX_PATH_LEN]; 345 PetscViewerFormat format; 346 PetscBool flg4 = PETSC_FALSE; 347 #endif 348 349 PetscFunctionBegin; 350 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 351 352 #if !defined(PETSC_HAVE_THREADSAFETY) 353 if (!(PETSC_RUNNING_ON_VALGRIND)) { 354 /* 355 Setup the memory management; support for tracing malloc() usage 356 */ 357 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 358 359 if (PetscDefined(USE_DEBUG)) { 360 mdebug = PETSC_TRUE; 361 initializenan = PETSC_TRUE; 362 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 363 } else { 364 /* don't warn about unused option */ 365 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 366 flg1 = PETSC_FALSE; 367 } 368 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 369 if (flg1 || flg2) { 370 mdebug = PETSC_TRUE; 371 eachcall = PETSC_TRUE; 372 initializenan = PETSC_TRUE; 373 } else if (flg3 && !flg2) { 374 mdebug = PETSC_FALSE; 375 eachcall = PETSC_FALSE; 376 initializenan = PETSC_FALSE; 377 } 378 379 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 380 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 381 382 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 383 if (mlog) { 384 mdebug = PETSC_TRUE; 385 } 386 /* the next line is deprecated */ 387 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 388 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 389 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 390 if (mdebug) { 391 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 392 } 393 if (mlog) { 394 PetscReal logthreshold = 0; 395 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 396 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 397 } 398 #if defined(PETSC_USE_LOG) 399 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 400 #endif 401 } 402 403 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 404 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 405 flg1 = PETSC_FALSE; 406 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 407 /* ignore this option if malloc is already set */ 408 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 409 410 flg1 = PETSC_FALSE; 411 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 412 if (!flg1) { 413 flg1 = PETSC_FALSE; 414 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 415 } 416 if (flg1) { 417 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 418 } 419 #endif 420 421 #if defined(PETSC_USE_LOG) 422 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 423 #endif 424 425 /* 426 Set the display variable for graphics 427 */ 428 ierr = PetscSetDisplay();CHKERRQ(ierr); 429 430 /* 431 Print main application help message 432 */ 433 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 434 if (help && hasHelp) { 435 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 436 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 437 } 438 439 /* 440 Print the PETSc version information 441 */ 442 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 443 if (flg1 || hasHelp) { 444 /* 445 Print "higher-level" package version message 446 */ 447 if (PetscExternalVersionFunction) { 448 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 449 } 450 451 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 452 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 453 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 454 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 455 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 456 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 457 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 458 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 459 } 460 461 /* 462 Print "higher-level" package help message 463 */ 464 if (hasHelp) { 465 PetscBool hasHelpIntro; 466 467 if (PetscExternalHelpFunction) { 468 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 469 } 470 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 471 if (hasHelpIntro) { 472 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 473 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 474 ierr = MPI_Finalize();CHKERRMPI(ierr); 475 exit(0); 476 } 477 } 478 479 /* 480 Setup the error handling 481 */ 482 flg1 = PETSC_FALSE; 483 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 484 if (flg1) { 485 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 486 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 487 } 488 flg1 = PETSC_FALSE; 489 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 490 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 491 flg1 = PETSC_FALSE; 492 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 493 if (flg1) { 494 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 495 } 496 flg1 = PETSC_FALSE; 497 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 498 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 499 500 /* 501 Setup debugger information 502 */ 503 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 504 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 505 if (flg1) { 506 MPI_Errhandler err_handler; 507 508 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 509 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 510 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 511 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 512 } 513 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 514 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 515 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 516 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 517 if (flg1 || flg2) { 518 PetscMPIInt size; 519 PetscInt lsize,*ranks; 520 MPI_Errhandler err_handler; 521 /* 522 we have to make sure that all processors have opened 523 connections to all other processors, otherwise once the 524 debugger has stated it is likely to receive a SIGUSR1 525 and kill the program. 526 */ 527 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 528 if (size > 2) { 529 PetscMPIInt dummy = 0; 530 MPI_Status status; 531 for (i=0; i<size; i++) { 532 if (rank != i) { 533 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 534 } 535 } 536 for (i=0; i<size; i++) { 537 if (rank != i) { 538 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 539 } 540 } 541 } 542 /* check if this processor node should be in debugger */ 543 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 544 lsize = size; 545 /* Deprecated in 3.14 */ 546 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 547 if (flag) { 548 const char * const quietopt="-options_suppress_deprecated_warnings"; 549 char msg[4096]; 550 PetscBool quiet = PETSC_FALSE; 551 552 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 553 if (!quiet) { 554 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 555 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 556 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 557 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 558 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 559 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 560 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 561 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 562 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 563 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 564 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 565 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 566 } 567 } else { 568 lsize = size; 569 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 570 } 571 if (flag) { 572 for (i=0; i<lsize; i++) { 573 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 574 } 575 } 576 if (!flag) { 577 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 578 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 579 if (flg1) { 580 ierr = PetscAttachDebugger();CHKERRQ(ierr); 581 } else { 582 ierr = PetscStopForDebugger();CHKERRQ(ierr); 583 } 584 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 585 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 586 } else { 587 ierr = PetscWaitOnError();CHKERRQ(ierr); 588 } 589 ierr = PetscFree(ranks);CHKERRQ(ierr); 590 } 591 592 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 593 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 594 595 /* 596 Setup profiling and logging 597 */ 598 #if defined(PETSC_USE_INFO) 599 { 600 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 601 } 602 #endif 603 ierr = PetscDetermineInitialFPTrap(); 604 flg1 = PETSC_FALSE; 605 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 606 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 607 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 608 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 609 #if defined(PETSC_USE_LOG) 610 mname[0] = 0; 611 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 612 if (flg1) { 613 if (mname[0]) { 614 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 615 } else { 616 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 617 } 618 } 619 620 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 621 622 #if defined(PETSC_HAVE_MPE) 623 flg1 = PETSC_FALSE; 624 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 625 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 626 #endif 627 flg1 = PETSC_FALSE; 628 flg3 = PETSC_FALSE; 629 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 630 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 631 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 632 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 633 634 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 635 if (flg1) { 636 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 637 FILE *file; 638 if (mname[0]) { 639 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 640 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 641 file = fopen(fname,"w"); 642 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 643 } else file = PETSC_STDOUT; 644 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 645 } 646 647 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 648 if (flg4) { 649 if (format == PETSC_VIEWER_ASCII_XML) { 650 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 651 } else { 652 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 653 } 654 } 655 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 656 PetscReal threshold = PetscRealConstant(0.01); 657 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 658 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 659 } 660 #endif 661 662 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 663 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 664 /* 665 If collecting logging information, by default, wait for device to complete its operations 666 before returning to the CPU in order to get accurate timings of each event 667 */ 668 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 669 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 670 671 #if defined(PETSC_HAVE_CUDA) 672 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 673 #endif 674 675 #if defined(PETSC_HAVE_HIP) 676 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 677 #endif 678 679 /* 680 Print basic help message 681 */ 682 if (hasHelp) { 683 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 684 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 685 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 686 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 687 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 688 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 689 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 690 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 693 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 694 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 718 #if defined(PETSC_USE_LOG) 719 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 723 #if defined(PETSC_HAVE_MPE) 724 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 725 #endif 726 #endif 727 #if defined(PETSC_USE_INFO) 728 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 729 #endif 730 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 734 } 735 736 #if defined(PETSC_HAVE_POPEN) 737 { 738 char machine[128]; 739 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 740 if (flg1) { 741 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 742 } 743 } 744 #endif 745 746 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 747 if (flg1) { 748 ierr = PetscSleep(si);CHKERRQ(ierr); 749 } 750 751 #if defined(PETSC_HAVE_VIENNACL) 752 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 753 if (!flg3) { 754 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 755 } 756 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 757 PetscViennaCLSynchronize = flg3; 758 ierr = PetscViennaCLInit();CHKERRQ(ierr); 759 #endif 760 761 /* 762 Creates the logging data structures; this is enabled even if logging is not turned on 763 This is the last thing we do before returning to the user code to prevent having the 764 logging numbers contaminated by any startup time associated with MPI and the GPUs 765 */ 766 #if defined(PETSC_USE_LOG) 767 ierr = PetscLogInitialize();CHKERRQ(ierr); 768 #endif 769 770 PetscFunctionReturn(0); 771 } 772