1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscviewer.h> 12 #if defined(PETSC_USE_LOG) 13 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 14 #endif 15 16 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 17 #include <sys/sysinfo.h> 18 #endif 19 #if defined(PETSC_HAVE_UNISTD_H) 20 #include <unistd.h> 21 #endif 22 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <cuda_runtime_api.h> 26 #include <cuda_device_runtime_api.h> 27 #include <petsccublas.h> 28 #endif 29 30 #if defined(PETSC_HAVE_HIP) 31 #include <hip/hip_runtime.h> 32 #endif 33 34 #if defined(PETSC_HAVE_DEVICE) 35 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 36 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 37 #endif 38 #endif 39 40 #if defined(PETSC_HAVE_VIENNACL) 41 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 42 #endif 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 #if defined(PETSC_HAVE_NVSHMEM) 62 PetscBool PetscBeganNvshmem = PETSC_FALSE; 63 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 64 #endif 65 66 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 67 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 68 69 #if defined(PETSC_HAVE_COMPLEX) 70 #if defined(PETSC_COMPLEX_INSTANTIATE) 71 template <> class std::complex<double>; /* instantiate complex template class */ 72 #endif 73 74 /*MC 75 PETSC_i - the imaginary number i 76 77 Synopsis: 78 #include <petscsys.h> 79 PetscComplex PETSC_i; 80 81 Level: beginner 82 83 Note: 84 Complex numbers are automatically available if PETSc located a working complex implementation 85 86 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 87 M*/ 88 PetscComplex PETSC_i; 89 MPI_Datatype MPIU___COMPLEX128 = 0; 90 #endif /* PETSC_HAVE_COMPLEX */ 91 #if defined(PETSC_USE_REAL___FLOAT128) 92 MPI_Datatype MPIU___FLOAT128 = 0; 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 MPI_Datatype MPIU_REAL_INT = 0; 98 MPI_Datatype MPIU_SCALAR_INT = 0; 99 #if defined(PETSC_USE_64BIT_INDICES) 100 MPI_Datatype MPIU_2INT = 0; 101 #endif 102 MPI_Datatype MPI_4INT = 0; 103 MPI_Datatype MPIU_4INT = 0; 104 MPI_Datatype MPIU_BOOL; 105 MPI_Datatype MPIU_ENUM; 106 MPI_Datatype MPIU_FORTRANADDR; 107 MPI_Datatype MPIU_SIZE_T; 108 109 /* 110 Function that is called to display all error messages 111 */ 112 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 113 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 114 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 115 /* 116 This is needed to turn on/off GPU synchronization 117 */ 118 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 119 120 /* ------------------------------------------------------------------------------*/ 121 /* 122 Optional file where all PETSc output from various prints is saved 123 */ 124 PETSC_INTERN FILE *petsc_history; 125 FILE *petsc_history = NULL; 126 127 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 128 { 129 PetscErrorCode ierr; 130 PetscMPIInt rank,size; 131 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 132 char version[256]; 133 134 PetscFunctionBegin; 135 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 136 if (!rank) { 137 char arch[10]; 138 int err; 139 140 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 141 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 142 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 143 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 144 if (filename) { 145 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 146 } else { 147 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 148 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 149 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 150 } 151 152 *fd = fopen(fname,"a"); 153 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 154 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 156 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 157 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 158 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 159 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 160 161 err = fflush(*fd); 162 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 163 } 164 PetscFunctionReturn(0); 165 } 166 167 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 168 { 169 PetscErrorCode ierr; 170 PetscMPIInt rank; 171 char date[64]; 172 int err; 173 174 PetscFunctionBegin; 175 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 176 if (!rank) { 177 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 178 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 179 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 180 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 181 err = fflush(*fd); 182 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 183 err = fclose(*fd); 184 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 185 } 186 PetscFunctionReturn(0); 187 } 188 189 /* ------------------------------------------------------------------------------*/ 190 191 /* 192 This is ugly and probably belongs somewhere else, but I want to 193 be able to put a true MPI abort error handler with command line args. 194 195 This is so MPI errors in the debugger will leave all the stack 196 frames. The default MP_Abort() cleans up and exits thus providing no useful information 197 in the debugger hence we call abort() instead of MPI_Abort(). 198 */ 199 200 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 201 { 202 PetscFunctionBegin; 203 (*PetscErrorPrintf)("MPI error %d\n",*flag); 204 abort(); 205 } 206 207 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 208 { 209 PetscErrorCode ierr; 210 211 PetscFunctionBegin; 212 (*PetscErrorPrintf)("MPI error %d\n",*flag); 213 ierr = PetscAttachDebugger(); 214 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 215 } 216 217 /*@C 218 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 219 wishes a clean exit somewhere deep in the program. 220 221 Collective on PETSC_COMM_WORLD 222 223 Options Database Keys are the same as for PetscFinalize() 224 225 Level: advanced 226 227 Note: 228 See PetscInitialize() for more general runtime options. 229 230 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 231 @*/ 232 PetscErrorCode PetscEnd(void) 233 { 234 PetscFunctionBegin; 235 PetscFinalize(); 236 exit(0); 237 return 0; 238 } 239 240 PetscBool PetscOptionsPublish = PETSC_FALSE; 241 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 242 PETSC_INTERN PetscBool petscsetmallocvisited; 243 static char emacsmachinename[256]; 244 245 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 246 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 247 248 /*@C 249 PetscSetHelpVersionFunctions - Sets functions that print help and version information 250 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 251 This routine enables a "higher-level" package that uses PETSc to print its messages first. 252 253 Input Parameter: 254 + help - the help function (may be NULL) 255 - version - the version function (may be NULL) 256 257 Level: developer 258 259 @*/ 260 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 261 { 262 PetscFunctionBegin; 263 PetscExternalHelpFunction = help; 264 PetscExternalVersionFunction = version; 265 PetscFunctionReturn(0); 266 } 267 268 #if defined(PETSC_USE_LOG) 269 PETSC_INTERN PetscBool PetscObjectsLog; 270 #endif 271 272 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 273 Use the following macros to define CUDA/HIP initialization related vars/routines. 274 */ 275 #if defined(PETSC_HAVE_CUDA) 276 typedef cudaError_t cupmError_t; 277 typedef struct cudaDeviceProp cupmDeviceProp; 278 typedef cudaStream_t cupmStream_t; 279 typedef cudaEvent_t cupmEvent_t; 280 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 281 #define cupmGetDevice(x) cudaGetDevice(x) 282 #define cupmSetDevice(x) cudaSetDevice(x) 283 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 284 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 285 #define cupmStreamCreate(x) cudaStreamCreate(x) 286 #define cupmGetLastError() cudaGetLastError() 287 #define cupmDeviceMapHost cudaDeviceMapHost 288 #define cupmSuccess cudaSuccess 289 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 290 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 291 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 292 #define CHKERRCUPM(x) CHKERRCUDA(x) 293 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 294 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 295 #define PetscCUPMInitialize PetscCUDAInitialize 296 #define PetscCUPMInitialized PetscCUDAInitialized 297 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 298 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 299 #define PetscCUPMSynchronize PetscCUDASynchronize 300 #define PetscNotUseCUPM PetscNotUseCUDA 301 #define cupmOptionsStr "CUDA options" 302 #define cupmSetDeviceStr "-cuda_device" 303 #define cupmViewStr "-cuda_view" 304 #define cupmSynchronizeStr "-cuda_synchronize" 305 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 306 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 307 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 308 #define PetscDefaultCupmStream PetscDefaultCudaStream 309 #define cupmEventCreate(x) cudaEventCreate(x) 310 #include "cupminit.inc" 311 #endif 312 313 #if defined(PETSC_HAVE_HIP) 314 typedef hipError_t cupmError_t; 315 typedef hipDeviceProp_t cupmDeviceProp; 316 typedef hipStream_t cupmStream_t; 317 typedef hipEvent_t cupmEvent_t; 318 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 319 #define cupmGetDevice(x) hipGetDevice(x) 320 #define cupmSetDevice(x) hipSetDevice(x) 321 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 322 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 323 #define cupmStreamCreate(x) hipStreamCreate(x) 324 #define cupmEventCreate(x) hipEventCreate(x); 325 #define cupmGetLastError() hipGetLastError() 326 #define cupmDeviceMapHost hipDeviceMapHost 327 #define cupmSuccess hipSuccess 328 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 329 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 330 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 331 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 332 #define PetscCUPMBLASInitializeHandle() 0 333 #define PetscCUPMSOLVERDnInitializeHandle() 0 334 #define PetscCUPMInitialize PetscHIPInitialize 335 #define PetscCUPMInitialized PetscHIPInitialized 336 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 337 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 338 #define PetscCUPMSynchronize PetscHIPSynchronize 339 #define PetscNotUseCUPM PetscNotUseHIP 340 #define cupmOptionsStr "HIP options" 341 #define cupmSetDeviceStr "-hip_device" 342 #define cupmViewStr "-hip_view" 343 #define cupmSynchronizeStr "-hip_synchronize" 344 #define PetscCUPMInitializeStr "PetscHIPInitialize" 345 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 346 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 347 #define PetscDefaultCupmStream PetscDefaultHipStream 348 #include "cupminit.inc" 349 #endif 350 351 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 352 { 353 char string[64]; 354 MPI_Comm comm = PETSC_COMM_WORLD; 355 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 356 PetscErrorCode ierr; 357 PetscReal si; 358 PetscInt intensity; 359 int i; 360 PetscMPIInt rank; 361 char version[256]; 362 #if defined(PETSC_USE_LOG) 363 char mname[PETSC_MAX_PATH_LEN]; 364 PetscViewerFormat format; 365 PetscBool flg4 = PETSC_FALSE; 366 #endif 367 368 PetscFunctionBegin; 369 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 370 371 #if !defined(PETSC_HAVE_THREADSAFETY) 372 if (!(PETSC_RUNNING_ON_VALGRIND)) { 373 /* 374 Setup the memory management; support for tracing malloc() usage 375 */ 376 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 377 378 if (PetscDefined(USE_DEBUG)) { 379 mdebug = PETSC_TRUE; 380 initializenan = PETSC_TRUE; 381 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 382 } else { 383 /* don't warn about unused option */ 384 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 385 flg1 = PETSC_FALSE; 386 } 387 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 388 if (flg1 || flg2) { 389 mdebug = PETSC_TRUE; 390 eachcall = PETSC_TRUE; 391 initializenan = PETSC_TRUE; 392 } else if (flg3 && !flg2) { 393 mdebug = PETSC_FALSE; 394 eachcall = PETSC_FALSE; 395 initializenan = PETSC_FALSE; 396 } 397 398 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 399 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 400 401 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 402 if (mlog) { 403 mdebug = PETSC_TRUE; 404 } 405 /* the next line is deprecated */ 406 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 407 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 408 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 409 if (mdebug) { 410 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 411 } 412 if (mlog) { 413 PetscReal logthreshold = 0; 414 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 415 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 416 } 417 #if defined(PETSC_USE_LOG) 418 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 419 #endif 420 } 421 422 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 423 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 424 flg1 = PETSC_FALSE; 425 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 426 /* ignore this option if malloc is already set */ 427 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 428 429 flg1 = PETSC_FALSE; 430 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 431 if (!flg1) { 432 flg1 = PETSC_FALSE; 433 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 434 } 435 if (flg1) { 436 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 437 } 438 #endif 439 440 #if defined(PETSC_USE_LOG) 441 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 442 #endif 443 444 /* 445 Set the display variable for graphics 446 */ 447 ierr = PetscSetDisplay();CHKERRQ(ierr); 448 449 /* 450 Print main application help message 451 */ 452 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 453 if (help && hasHelp) { 454 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 455 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 456 } 457 458 /* 459 Print the PETSc version information 460 */ 461 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 462 if (flg1 || hasHelp) { 463 /* 464 Print "higher-level" package version message 465 */ 466 if (PetscExternalVersionFunction) { 467 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 468 } 469 470 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 477 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 478 } 479 480 /* 481 Print "higher-level" package help message 482 */ 483 if (hasHelp) { 484 PetscBool hasHelpIntro; 485 486 if (PetscExternalHelpFunction) { 487 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 488 } 489 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 490 if (hasHelpIntro) { 491 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 492 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 493 ierr = MPI_Finalize();CHKERRMPI(ierr); 494 exit(0); 495 } 496 } 497 498 /* 499 Setup the error handling 500 */ 501 flg1 = PETSC_FALSE; 502 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 503 if (flg1) { 504 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 505 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 506 } 507 flg1 = PETSC_FALSE; 508 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 509 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 510 flg1 = PETSC_FALSE; 511 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 512 if (flg1) { 513 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 514 } 515 flg1 = PETSC_FALSE; 516 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 517 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 518 519 /* 520 Setup debugger information 521 */ 522 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 523 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 524 if (flg1) { 525 MPI_Errhandler err_handler; 526 527 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 528 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 529 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 530 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 531 } 532 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 533 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 534 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 535 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 536 if (flg1 || flg2) { 537 PetscMPIInt size; 538 PetscInt lsize,*ranks; 539 MPI_Errhandler err_handler; 540 /* 541 we have to make sure that all processors have opened 542 connections to all other processors, otherwise once the 543 debugger has stated it is likely to receive a SIGUSR1 544 and kill the program. 545 */ 546 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 547 if (size > 2) { 548 PetscMPIInt dummy = 0; 549 MPI_Status status; 550 for (i=0; i<size; i++) { 551 if (rank != i) { 552 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 553 } 554 } 555 for (i=0; i<size; i++) { 556 if (rank != i) { 557 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 558 } 559 } 560 } 561 /* check if this processor node should be in debugger */ 562 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 563 lsize = size; 564 /* Deprecated in 3.14 */ 565 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 566 if (flag) { 567 const char * const quietopt="-options_suppress_deprecated_warnings"; 568 char msg[4096]; 569 PetscBool quiet = PETSC_FALSE; 570 571 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 572 if (!quiet) { 573 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 576 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 577 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 579 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 580 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 581 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 582 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 583 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 584 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 585 } 586 } else { 587 lsize = size; 588 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 589 } 590 if (flag) { 591 for (i=0; i<lsize; i++) { 592 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 593 } 594 } 595 if (!flag) { 596 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 597 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 598 if (flg1) { 599 ierr = PetscAttachDebugger();CHKERRQ(ierr); 600 } else { 601 ierr = PetscStopForDebugger();CHKERRQ(ierr); 602 } 603 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 604 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 605 } else { 606 ierr = PetscWaitOnError();CHKERRQ(ierr); 607 } 608 ierr = PetscFree(ranks);CHKERRQ(ierr); 609 } 610 611 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 612 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 613 614 /* 615 Setup profiling and logging 616 */ 617 #if defined(PETSC_USE_INFO) 618 { 619 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 620 } 621 #endif 622 ierr = PetscDetermineInitialFPTrap(); 623 flg1 = PETSC_FALSE; 624 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 625 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 626 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 627 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 628 #if defined(PETSC_USE_LOG) 629 mname[0] = 0; 630 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 631 if (flg1) { 632 if (mname[0]) { 633 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 634 } else { 635 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 636 } 637 } 638 639 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 640 641 #if defined(PETSC_HAVE_MPE) 642 flg1 = PETSC_FALSE; 643 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 644 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 645 #endif 646 flg1 = PETSC_FALSE; 647 flg3 = PETSC_FALSE; 648 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 649 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 650 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 651 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 652 653 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 654 if (flg1) { 655 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 656 FILE *file; 657 if (mname[0]) { 658 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 659 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 660 file = fopen(fname,"w"); 661 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 662 } else file = PETSC_STDOUT; 663 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 664 } 665 666 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 667 if (flg4) { 668 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 669 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 670 } else { 671 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 672 } 673 } 674 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 675 PetscReal threshold = PetscRealConstant(0.01); 676 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 677 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 678 } 679 #endif 680 681 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 682 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 683 /* 684 If collecting logging information, by default, wait for device to complete its operations 685 before returning to the CPU in order to get accurate timings of each event 686 */ 687 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 688 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 689 690 #if defined(PETSC_HAVE_CUDA) 691 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 692 #endif 693 694 #if defined(PETSC_HAVE_HIP) 695 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 696 #endif 697 698 /* 699 Print basic help message 700 */ 701 if (hasHelp) { 702 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 737 #if defined(PETSC_USE_LOG) 738 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 742 #if defined(PETSC_HAVE_MPE) 743 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 744 #endif 745 #endif 746 #if defined(PETSC_USE_INFO) 747 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 748 #endif 749 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 750 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 751 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 752 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 753 } 754 755 #if defined(PETSC_HAVE_POPEN) 756 { 757 char machine[128]; 758 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 759 if (flg1) { 760 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 761 } 762 } 763 #endif 764 765 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 766 if (flg1) { 767 ierr = PetscSleep(si);CHKERRQ(ierr); 768 } 769 770 #if defined(PETSC_HAVE_VIENNACL) 771 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 772 if (!flg3) { 773 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 774 } 775 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 776 PetscViennaCLSynchronize = flg3; 777 ierr = PetscViennaCLInit();CHKERRQ(ierr); 778 #endif 779 780 /* 781 Creates the logging data structures; this is enabled even if logging is not turned on 782 This is the last thing we do before returning to the user code to prevent having the 783 logging numbers contaminated by any startup time associated with MPI and the GPUs 784 */ 785 #if defined(PETSC_USE_LOG) 786 ierr = PetscLogInitialize();CHKERRQ(ierr); 787 #endif 788 789 PetscFunctionReturn(0); 790 } 791