1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <cuda_runtime_api.h> 27 #include <cuda_device_runtime_api.h> 28 #include <petsccublas.h> 29 #endif 30 31 #if defined(PETSC_HAVE_HIP) 32 #include <hip/hip_runtime.h> 33 #endif 34 35 #if defined(PETSC_HAVE_DEVICE) 36 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 37 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 38 #endif 39 #endif 40 41 #if defined(PETSC_HAVE_VIENNACL) 42 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 43 #endif 44 45 /* ------------------------Nasty global variables -------------------------------*/ 46 /* 47 Indicates if PETSc started up MPI, or it was 48 already started before PETSc was initialized. 49 */ 50 PetscBool PetscBeganMPI = PETSC_FALSE; 51 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 52 PetscBool PetscInitializeCalled = PETSC_FALSE; 53 PetscBool PetscFinalizeCalled = PETSC_FALSE; 54 55 PetscMPIInt PetscGlobalRank = -1; 56 PetscMPIInt PetscGlobalSize = -1; 57 58 #if defined(PETSC_HAVE_KOKKOS) 59 PetscBool PetscBeganKokkos = PETSC_FALSE; 60 #endif 61 62 #if defined(PETSC_HAVE_NVSHMEM) 63 PetscBool PetscBeganNvshmem = PETSC_FALSE; 64 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 65 #endif 66 67 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 68 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 69 70 #if defined(PETSC_HAVE_COMPLEX) 71 #if defined(PETSC_COMPLEX_INSTANTIATE) 72 template <> class std::complex<double>; /* instantiate complex template class */ 73 #endif 74 75 /*MC 76 PETSC_i - the imaginary number i 77 78 Synopsis: 79 #include <petscsys.h> 80 PetscComplex PETSC_i; 81 82 Level: beginner 83 84 Note: 85 Complex numbers are automatically available if PETSc located a working complex implementation 86 87 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 88 M*/ 89 PetscComplex PETSC_i; 90 MPI_Datatype MPIU___COMPLEX128 = 0; 91 #endif /* PETSC_HAVE_COMPLEX */ 92 #if defined(PETSC_USE_REAL___FLOAT128) 93 MPI_Datatype MPIU___FLOAT128 = 0; 94 #elif defined(PETSC_USE_REAL___FP16) 95 MPI_Datatype MPIU___FP16 = 0; 96 #endif 97 MPI_Datatype MPIU_2SCALAR = 0; 98 MPI_Datatype MPIU_REAL_INT = 0; 99 MPI_Datatype MPIU_SCALAR_INT = 0; 100 #if defined(PETSC_USE_64BIT_INDICES) 101 MPI_Datatype MPIU_2INT = 0; 102 #endif 103 MPI_Datatype MPIU_BOOL; 104 MPI_Datatype MPIU_ENUM; 105 MPI_Datatype MPIU_FORTRANADDR; 106 MPI_Datatype MPIU_SIZE_T; 107 108 /* 109 Function that is called to display all error messages 110 */ 111 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 112 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 113 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 114 /* 115 This is needed to turn on/off GPU synchronization 116 */ 117 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 118 119 /* ------------------------------------------------------------------------------*/ 120 /* 121 Optional file where all PETSc output from various prints is saved 122 */ 123 PETSC_INTERN FILE *petsc_history; 124 FILE *petsc_history = NULL; 125 126 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 127 { 128 PetscErrorCode ierr; 129 PetscMPIInt rank,size; 130 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 131 char version[256]; 132 133 PetscFunctionBegin; 134 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 135 if (!rank) { 136 char arch[10]; 137 int err; 138 139 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 140 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 141 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 142 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 143 if (filename) { 144 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 145 } else { 146 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 147 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 148 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 149 } 150 151 *fd = fopen(fname,"a"); 152 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 153 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 156 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 157 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 158 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 159 160 err = fflush(*fd); 161 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 162 } 163 PetscFunctionReturn(0); 164 } 165 166 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 167 { 168 PetscErrorCode ierr; 169 PetscMPIInt rank; 170 char date[64]; 171 int err; 172 173 PetscFunctionBegin; 174 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 175 if (!rank) { 176 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 177 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 178 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 179 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 180 err = fflush(*fd); 181 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 182 err = fclose(*fd); 183 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 184 } 185 PetscFunctionReturn(0); 186 } 187 188 /* ------------------------------------------------------------------------------*/ 189 190 /* 191 This is ugly and probably belongs somewhere else, but I want to 192 be able to put a true MPI abort error handler with command line args. 193 194 This is so MPI errors in the debugger will leave all the stack 195 frames. The default MP_Abort() cleans up and exits thus providing no useful information 196 in the debugger hence we call abort() instead of MPI_Abort(). 197 */ 198 199 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 200 { 201 PetscFunctionBegin; 202 (*PetscErrorPrintf)("MPI error %d\n",*flag); 203 abort(); 204 } 205 206 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 207 { 208 PetscErrorCode ierr; 209 210 PetscFunctionBegin; 211 (*PetscErrorPrintf)("MPI error %d\n",*flag); 212 ierr = PetscAttachDebugger(); 213 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 214 } 215 216 /*@C 217 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 218 wishes a clean exit somewhere deep in the program. 219 220 Collective on PETSC_COMM_WORLD 221 222 Options Database Keys are the same as for PetscFinalize() 223 224 Level: advanced 225 226 Note: 227 See PetscInitialize() for more general runtime options. 228 229 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 230 @*/ 231 PetscErrorCode PetscEnd(void) 232 { 233 PetscFunctionBegin; 234 PetscFinalize(); 235 exit(0); 236 return 0; 237 } 238 239 PetscBool PetscOptionsPublish = PETSC_FALSE; 240 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 241 PETSC_INTERN PetscBool petscsetmallocvisited; 242 static char emacsmachinename[256]; 243 244 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 245 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 246 247 /*@C 248 PetscSetHelpVersionFunctions - Sets functions that print help and version information 249 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 250 This routine enables a "higher-level" package that uses PETSc to print its messages first. 251 252 Input Parameter: 253 + help - the help function (may be NULL) 254 - version - the version function (may be NULL) 255 256 Level: developer 257 258 @*/ 259 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 260 { 261 PetscFunctionBegin; 262 PetscExternalHelpFunction = help; 263 PetscExternalVersionFunction = version; 264 PetscFunctionReturn(0); 265 } 266 267 #if defined(PETSC_USE_LOG) 268 PETSC_INTERN PetscBool PetscObjectsLog; 269 #endif 270 271 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 272 Use the following macros to define CUDA/HIP initialization related vars/routines. 273 */ 274 #if defined(PETSC_HAVE_CUDA) 275 typedef cudaError_t cupmError_t; 276 typedef struct cudaDeviceProp cupmDeviceProp; 277 typedef cudaStream_t cupmStream_t; 278 typedef cudaEvent_t cupmEvent_t; 279 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 280 #define cupmGetDevice(x) cudaGetDevice(x) 281 #define cupmSetDevice(x) cudaSetDevice(x) 282 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 283 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 284 #define cupmStreamCreate(x) cudaStreamCreate(x) 285 #define cupmGetLastError() cudaGetLastError() 286 #define cupmDeviceMapHost cudaDeviceMapHost 287 #define cupmSuccess cudaSuccess 288 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 289 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 290 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 291 #define CHKERRCUPM(x) CHKERRCUDA(x) 292 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 293 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 294 #define PetscCUPMInitialize PetscCUDAInitialize 295 #define PetscCUPMInitialized PetscCUDAInitialized 296 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 297 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 298 #define PetscCUPMSynchronize PetscCUDASynchronize 299 #define PetscNotUseCUPM PetscNotUseCUDA 300 #define cupmOptionsStr "CUDA options" 301 #define cupmSetDeviceStr "-cuda_device" 302 #define cupmViewStr "-cuda_view" 303 #define cupmSynchronizeStr "-cuda_synchronize" 304 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 305 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 306 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 307 #define PetscDefaultCupmStream PetscDefaultCudaStream 308 #define cupmEventCreate(x) cudaEventCreate(x) 309 #include "cupminit.inc" 310 #endif 311 312 #if defined(PETSC_HAVE_HIP) 313 typedef hipError_t cupmError_t; 314 typedef hipDeviceProp_t cupmDeviceProp; 315 typedef hipStream_t cupmStream_t; 316 typedef hipEvent_t cupmEvent_t; 317 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 318 #define cupmGetDevice(x) hipGetDevice(x) 319 #define cupmSetDevice(x) hipSetDevice(x) 320 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 321 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 322 #define cupmStreamCreate(x) hipStreamCreate(x) 323 #define cupmEventCreate(x) hipEventCreate(x); 324 #define cupmGetLastError() hipGetLastError() 325 #define cupmDeviceMapHost hipDeviceMapHost 326 #define cupmSuccess hipSuccess 327 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 328 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 329 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 330 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 331 #define PetscCUPMBLASInitializeHandle() 0 332 #define PetscCUPMSOLVERDnInitializeHandle() 0 333 #define PetscCUPMInitialize PetscHIPInitialize 334 #define PetscCUPMInitialized PetscHIPInitialized 335 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 336 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 337 #define PetscCUPMSynchronize PetscHIPSynchronize 338 #define PetscNotUseCUPM PetscNotUseHIP 339 #define cupmOptionsStr "HIP options" 340 #define cupmSetDeviceStr "-hip_device" 341 #define cupmViewStr "-hip_view" 342 #define cupmSynchronizeStr "-hip_synchronize" 343 #define PetscCUPMInitializeStr "PetscHIPInitialize" 344 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 345 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 346 #define PetscDefaultCupmStream PetscDefaultHipStream 347 #include "cupminit.inc" 348 #endif 349 350 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 351 { 352 char string[64]; 353 MPI_Comm comm = PETSC_COMM_WORLD; 354 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 355 PetscErrorCode ierr; 356 PetscReal si; 357 PetscInt intensity; 358 int i; 359 PetscMPIInt rank; 360 char version[256]; 361 #if defined(PETSC_USE_LOG) 362 char mname[PETSC_MAX_PATH_LEN]; 363 PetscViewerFormat format; 364 PetscBool flg4 = PETSC_FALSE; 365 #endif 366 367 PetscFunctionBegin; 368 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 369 370 #if !defined(PETSC_HAVE_THREADSAFETY) 371 if (!(PETSC_RUNNING_ON_VALGRIND)) { 372 /* 373 Setup the memory management; support for tracing malloc() usage 374 */ 375 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 376 377 if (PetscDefined(USE_DEBUG)) { 378 mdebug = PETSC_TRUE; 379 initializenan = PETSC_TRUE; 380 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 381 } else { 382 /* don't warn about unused option */ 383 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 384 flg1 = PETSC_FALSE; 385 } 386 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 387 if (flg1 || flg2) { 388 mdebug = PETSC_TRUE; 389 eachcall = PETSC_TRUE; 390 initializenan = PETSC_TRUE; 391 } else if (flg3 && !flg2) { 392 mdebug = PETSC_FALSE; 393 eachcall = PETSC_FALSE; 394 initializenan = PETSC_FALSE; 395 } 396 397 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 398 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 399 400 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 401 if (mlog) { 402 mdebug = PETSC_TRUE; 403 } 404 /* the next line is deprecated */ 405 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 406 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 407 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 408 if (mdebug) { 409 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 410 } 411 if (mlog) { 412 PetscReal logthreshold = 0; 413 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 414 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 415 } 416 #if defined(PETSC_USE_LOG) 417 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 418 #endif 419 } 420 421 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 422 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 423 flg1 = PETSC_FALSE; 424 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 425 /* ignore this option if malloc is already set */ 426 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 427 428 flg1 = PETSC_FALSE; 429 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 430 if (!flg1) { 431 flg1 = PETSC_FALSE; 432 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 433 } 434 if (flg1) { 435 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 436 } 437 #endif 438 439 #if defined(PETSC_USE_LOG) 440 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 441 #endif 442 443 /* 444 Set the display variable for graphics 445 */ 446 ierr = PetscSetDisplay();CHKERRQ(ierr); 447 448 /* 449 Print main application help message 450 */ 451 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 452 if (help && hasHelp) { 453 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 454 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 455 } 456 457 /* 458 Print the PETSc version information 459 */ 460 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 461 if (flg1 || hasHelp) { 462 /* 463 Print "higher-level" package version message 464 */ 465 if (PetscExternalVersionFunction) { 466 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 467 } 468 469 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 477 } 478 479 /* 480 Print "higher-level" package help message 481 */ 482 if (hasHelp) { 483 PetscBool hasHelpIntro; 484 485 if (PetscExternalHelpFunction) { 486 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 487 } 488 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 489 if (hasHelpIntro) { 490 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 491 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 492 ierr = MPI_Finalize();CHKERRMPI(ierr); 493 exit(0); 494 } 495 } 496 497 /* 498 Setup the error handling 499 */ 500 flg1 = PETSC_FALSE; 501 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 502 if (flg1) { 503 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 504 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 505 } 506 flg1 = PETSC_FALSE; 507 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 508 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 509 flg1 = PETSC_FALSE; 510 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 511 if (flg1) { 512 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 513 } 514 flg1 = PETSC_FALSE; 515 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 516 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 517 518 /* 519 Setup debugger information 520 */ 521 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 522 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 523 if (flg1) { 524 MPI_Errhandler err_handler; 525 526 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 527 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 528 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 529 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 530 } 531 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 532 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 533 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 534 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 535 if (flg1 || flg2) { 536 PetscMPIInt size; 537 PetscInt lsize,*ranks; 538 MPI_Errhandler err_handler; 539 /* 540 we have to make sure that all processors have opened 541 connections to all other processors, otherwise once the 542 debugger has stated it is likely to receive a SIGUSR1 543 and kill the program. 544 */ 545 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 546 if (size > 2) { 547 PetscMPIInt dummy = 0; 548 MPI_Status status; 549 for (i=0; i<size; i++) { 550 if (rank != i) { 551 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 552 } 553 } 554 for (i=0; i<size; i++) { 555 if (rank != i) { 556 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 557 } 558 } 559 } 560 /* check if this processor node should be in debugger */ 561 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 562 lsize = size; 563 /* Deprecated in 3.14 */ 564 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 565 if (flag) { 566 const char * const quietopt="-options_suppress_deprecated_warnings"; 567 char msg[4096]; 568 PetscBool quiet = PETSC_FALSE; 569 570 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 571 if (!quiet) { 572 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 573 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 576 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 577 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 579 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 580 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 581 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 582 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 583 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 584 } 585 } else { 586 lsize = size; 587 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 588 } 589 if (flag) { 590 for (i=0; i<lsize; i++) { 591 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 592 } 593 } 594 if (!flag) { 595 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 596 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 597 if (flg1) { 598 ierr = PetscAttachDebugger();CHKERRQ(ierr); 599 } else { 600 ierr = PetscStopForDebugger();CHKERRQ(ierr); 601 } 602 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 603 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 604 } else { 605 ierr = PetscWaitOnError();CHKERRQ(ierr); 606 } 607 ierr = PetscFree(ranks);CHKERRQ(ierr); 608 } 609 610 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 611 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 612 613 /* 614 Setup profiling and logging 615 */ 616 #if defined(PETSC_USE_INFO) 617 { 618 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 619 } 620 #endif 621 ierr = PetscDetermineInitialFPTrap(); 622 flg1 = PETSC_FALSE; 623 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 624 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 625 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 626 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 627 #if defined(PETSC_USE_LOG) 628 mname[0] = 0; 629 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 630 if (flg1) { 631 if (mname[0]) { 632 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 633 } else { 634 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 635 } 636 } 637 638 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 639 640 #if defined(PETSC_HAVE_MPE) 641 flg1 = PETSC_FALSE; 642 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 643 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 644 #endif 645 flg1 = PETSC_FALSE; 646 flg3 = PETSC_FALSE; 647 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 648 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 649 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 650 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 651 652 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 653 if (flg1) { 654 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 655 FILE *file; 656 if (mname[0]) { 657 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 658 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 659 file = fopen(fname,"w"); 660 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 661 } else file = PETSC_STDOUT; 662 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 663 } 664 665 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 666 if (flg4) { 667 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 668 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 669 } else { 670 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 671 } 672 } 673 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 674 PetscReal threshold = PetscRealConstant(0.01); 675 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 676 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 677 } 678 #endif 679 680 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 681 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 682 /* 683 If collecting logging information, by default, wait for device to complete its operations 684 before returning to the CPU in order to get accurate timings of each event 685 */ 686 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 687 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 688 689 #if defined(PETSC_HAVE_CUDA) 690 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 691 #endif 692 693 #if defined(PETSC_HAVE_HIP) 694 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 695 #endif 696 697 /* 698 Print basic help message 699 */ 700 if (hasHelp) { 701 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 736 #if defined(PETSC_USE_LOG) 737 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 741 #if defined(PETSC_HAVE_MPE) 742 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 743 #endif 744 #endif 745 #if defined(PETSC_USE_INFO) 746 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 747 #endif 748 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 749 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 750 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 751 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 752 } 753 754 #if defined(PETSC_HAVE_POPEN) 755 { 756 char machine[128]; 757 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 758 if (flg1) { 759 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 760 } 761 } 762 #endif 763 764 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 765 if (flg1) { 766 ierr = PetscSleep(si);CHKERRQ(ierr); 767 } 768 769 #if defined(PETSC_HAVE_VIENNACL) 770 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 771 if (!flg3) { 772 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 773 } 774 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 775 PetscViennaCLSynchronize = flg3; 776 ierr = PetscViennaCLInit();CHKERRQ(ierr); 777 #endif 778 779 /* 780 Creates the logging data structures; this is enabled even if logging is not turned on 781 This is the last thing we do before returning to the user code to prevent having the 782 logging numbers contaminated by any startup time associated with MPI and the GPUs 783 */ 784 #if defined(PETSC_USE_LOG) 785 ierr = PetscLogInitialize();CHKERRQ(ierr); 786 #endif 787 788 PetscFunctionReturn(0); 789 } 790