1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscviewer.h> 12 #if defined(PETSC_USE_LOG) 13 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 14 #endif 15 16 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 17 #include <sys/sysinfo.h> 18 #endif 19 #if defined(PETSC_HAVE_UNISTD_H) 20 #include <unistd.h> 21 #endif 22 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <cuda_runtime_api.h> 26 #include <cuda_device_runtime_api.h> 27 #include <petsccublas.h> 28 #endif 29 30 #if defined(PETSC_HAVE_HIP) 31 #include <hip/hip_runtime.h> 32 #endif 33 34 #if defined(PETSC_HAVE_DEVICE) 35 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 36 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 37 #endif 38 #endif 39 40 #if defined(PETSC_HAVE_VIENNACL) 41 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 42 #endif 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 #if defined(PETSC_HAVE_NVSHMEM) 62 PetscBool PetscBeganNvshmem = PETSC_FALSE; 63 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 64 #endif 65 66 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 67 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 68 69 #if defined(PETSC_HAVE_COMPLEX) 70 #if defined(PETSC_COMPLEX_INSTANTIATE) 71 template <> class std::complex<double>; /* instantiate complex template class */ 72 #endif 73 74 /*MC 75 PETSC_i - the imaginary number i 76 77 Synopsis: 78 #include <petscsys.h> 79 PetscComplex PETSC_i; 80 81 Level: beginner 82 83 Note: 84 Complex numbers are automatically available if PETSc located a working complex implementation 85 86 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 87 M*/ 88 PetscComplex PETSC_i; 89 MPI_Datatype MPIU___COMPLEX128 = 0; 90 #endif /* PETSC_HAVE_COMPLEX */ 91 #if defined(PETSC_USE_REAL___FLOAT128) 92 MPI_Datatype MPIU___FLOAT128 = 0; 93 #elif defined(PETSC_USE_REAL___FP16) 94 MPI_Datatype MPIU___FP16 = 0; 95 #endif 96 MPI_Datatype MPIU_2SCALAR = 0; 97 MPI_Datatype MPIU_REAL_INT = 0; 98 MPI_Datatype MPIU_SCALAR_INT = 0; 99 #if defined(PETSC_USE_64BIT_INDICES) 100 MPI_Datatype MPIU_2INT = 0; 101 #endif 102 MPI_Datatype MPIU_BOOL; 103 MPI_Datatype MPIU_ENUM; 104 MPI_Datatype MPIU_FORTRANADDR; 105 MPI_Datatype MPIU_SIZE_T; 106 107 /* 108 Function that is called to display all error messages 109 */ 110 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 111 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 112 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 113 /* 114 This is needed to turn on/off GPU synchronization 115 */ 116 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 117 118 /* ------------------------------------------------------------------------------*/ 119 /* 120 Optional file where all PETSc output from various prints is saved 121 */ 122 PETSC_INTERN FILE *petsc_history; 123 FILE *petsc_history = NULL; 124 125 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 126 { 127 PetscErrorCode ierr; 128 PetscMPIInt rank,size; 129 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 130 char version[256]; 131 132 PetscFunctionBegin; 133 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 134 if (!rank) { 135 char arch[10]; 136 int err; 137 138 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 139 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 140 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 141 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 142 if (filename) { 143 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 144 } else { 145 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 146 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 147 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 148 } 149 150 *fd = fopen(fname,"a"); 151 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 152 153 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 155 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 156 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 157 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 158 159 err = fflush(*fd); 160 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 161 } 162 PetscFunctionReturn(0); 163 } 164 165 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 166 { 167 PetscErrorCode ierr; 168 PetscMPIInt rank; 169 char date[64]; 170 int err; 171 172 PetscFunctionBegin; 173 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 174 if (!rank) { 175 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 177 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 178 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 179 err = fflush(*fd); 180 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 181 err = fclose(*fd); 182 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 183 } 184 PetscFunctionReturn(0); 185 } 186 187 /* ------------------------------------------------------------------------------*/ 188 189 /* 190 This is ugly and probably belongs somewhere else, but I want to 191 be able to put a true MPI abort error handler with command line args. 192 193 This is so MPI errors in the debugger will leave all the stack 194 frames. The default MP_Abort() cleans up and exits thus providing no useful information 195 in the debugger hence we call abort() instead of MPI_Abort(). 196 */ 197 198 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 199 { 200 PetscFunctionBegin; 201 (*PetscErrorPrintf)("MPI error %d\n",*flag); 202 abort(); 203 } 204 205 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 206 { 207 PetscErrorCode ierr; 208 209 PetscFunctionBegin; 210 (*PetscErrorPrintf)("MPI error %d\n",*flag); 211 ierr = PetscAttachDebugger(); 212 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 213 } 214 215 /*@C 216 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 217 wishes a clean exit somewhere deep in the program. 218 219 Collective on PETSC_COMM_WORLD 220 221 Options Database Keys are the same as for PetscFinalize() 222 223 Level: advanced 224 225 Note: 226 See PetscInitialize() for more general runtime options. 227 228 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 229 @*/ 230 PetscErrorCode PetscEnd(void) 231 { 232 PetscFunctionBegin; 233 PetscFinalize(); 234 exit(0); 235 return 0; 236 } 237 238 PetscBool PetscOptionsPublish = PETSC_FALSE; 239 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 240 PETSC_INTERN PetscBool petscsetmallocvisited; 241 static char emacsmachinename[256]; 242 243 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 244 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 245 246 /*@C 247 PetscSetHelpVersionFunctions - Sets functions that print help and version information 248 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 249 This routine enables a "higher-level" package that uses PETSc to print its messages first. 250 251 Input Parameter: 252 + help - the help function (may be NULL) 253 - version - the version function (may be NULL) 254 255 Level: developer 256 257 @*/ 258 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 259 { 260 PetscFunctionBegin; 261 PetscExternalHelpFunction = help; 262 PetscExternalVersionFunction = version; 263 PetscFunctionReturn(0); 264 } 265 266 #if defined(PETSC_USE_LOG) 267 PETSC_INTERN PetscBool PetscObjectsLog; 268 #endif 269 270 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 271 Use the following macros to define CUDA/HIP initialization related vars/routines. 272 */ 273 #if defined(PETSC_HAVE_CUDA) 274 typedef cudaError_t cupmError_t; 275 typedef struct cudaDeviceProp cupmDeviceProp; 276 typedef cudaStream_t cupmStream_t; 277 typedef cudaEvent_t cupmEvent_t; 278 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 279 #define cupmGetDevice(x) cudaGetDevice(x) 280 #define cupmSetDevice(x) cudaSetDevice(x) 281 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 282 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 283 #define cupmStreamCreate(x) cudaStreamCreate(x) 284 #define cupmGetLastError() cudaGetLastError() 285 #define cupmDeviceMapHost cudaDeviceMapHost 286 #define cupmSuccess cudaSuccess 287 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 288 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 289 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 290 #define CHKERRCUPM(x) CHKERRCUDA(x) 291 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 292 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 293 #define PetscCUPMInitialize PetscCUDAInitialize 294 #define PetscCUPMInitialized PetscCUDAInitialized 295 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 296 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 297 #define PetscCUPMSynchronize PetscCUDASynchronize 298 #define PetscNotUseCUPM PetscNotUseCUDA 299 #define cupmOptionsStr "CUDA options" 300 #define cupmSetDeviceStr "-cuda_device" 301 #define cupmViewStr "-cuda_view" 302 #define cupmSynchronizeStr "-cuda_synchronize" 303 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 304 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 305 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 306 #define PetscDefaultCupmStream PetscDefaultCudaStream 307 #define cupmEventCreate(x) cudaEventCreate(x) 308 #include "cupminit.inc" 309 #endif 310 311 #if defined(PETSC_HAVE_HIP) 312 typedef hipError_t cupmError_t; 313 typedef hipDeviceProp_t cupmDeviceProp; 314 typedef hipStream_t cupmStream_t; 315 typedef hipEvent_t cupmEvent_t; 316 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 317 #define cupmGetDevice(x) hipGetDevice(x) 318 #define cupmSetDevice(x) hipSetDevice(x) 319 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 320 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 321 #define cupmStreamCreate(x) hipStreamCreate(x) 322 #define cupmEventCreate(x) hipEventCreate(x); 323 #define cupmGetLastError() hipGetLastError() 324 #define cupmDeviceMapHost hipDeviceMapHost 325 #define cupmSuccess hipSuccess 326 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 327 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 328 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 329 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 330 #define PetscCUPMBLASInitializeHandle() 0 331 #define PetscCUPMSOLVERDnInitializeHandle() 0 332 #define PetscCUPMInitialize PetscHIPInitialize 333 #define PetscCUPMInitialized PetscHIPInitialized 334 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 335 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 336 #define PetscCUPMSynchronize PetscHIPSynchronize 337 #define PetscNotUseCUPM PetscNotUseHIP 338 #define cupmOptionsStr "HIP options" 339 #define cupmSetDeviceStr "-hip_device" 340 #define cupmViewStr "-hip_view" 341 #define cupmSynchronizeStr "-hip_synchronize" 342 #define PetscCUPMInitializeStr "PetscHIPInitialize" 343 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 344 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 345 #define PetscDefaultCupmStream PetscDefaultHipStream 346 #include "cupminit.inc" 347 #endif 348 349 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 350 { 351 char string[64]; 352 MPI_Comm comm = PETSC_COMM_WORLD; 353 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 354 PetscErrorCode ierr; 355 PetscReal si; 356 PetscInt intensity; 357 int i; 358 PetscMPIInt rank; 359 char version[256]; 360 #if defined(PETSC_USE_LOG) 361 char mname[PETSC_MAX_PATH_LEN]; 362 PetscViewerFormat format; 363 PetscBool flg4 = PETSC_FALSE; 364 #endif 365 366 PetscFunctionBegin; 367 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 368 369 #if !defined(PETSC_HAVE_THREADSAFETY) 370 if (!(PETSC_RUNNING_ON_VALGRIND)) { 371 /* 372 Setup the memory management; support for tracing malloc() usage 373 */ 374 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 375 376 if (PetscDefined(USE_DEBUG)) { 377 mdebug = PETSC_TRUE; 378 initializenan = PETSC_TRUE; 379 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 380 } else { 381 /* don't warn about unused option */ 382 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 383 flg1 = PETSC_FALSE; 384 } 385 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 386 if (flg1 || flg2) { 387 mdebug = PETSC_TRUE; 388 eachcall = PETSC_TRUE; 389 initializenan = PETSC_TRUE; 390 } else if (flg3 && !flg2) { 391 mdebug = PETSC_FALSE; 392 eachcall = PETSC_FALSE; 393 initializenan = PETSC_FALSE; 394 } 395 396 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 397 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 398 399 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 400 if (mlog) { 401 mdebug = PETSC_TRUE; 402 } 403 /* the next line is deprecated */ 404 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 405 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 406 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 407 if (mdebug) { 408 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 409 } 410 if (mlog) { 411 PetscReal logthreshold = 0; 412 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 413 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 414 } 415 #if defined(PETSC_USE_LOG) 416 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 417 #endif 418 } 419 420 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 421 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 422 flg1 = PETSC_FALSE; 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 424 /* ignore this option if malloc is already set */ 425 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 426 427 flg1 = PETSC_FALSE; 428 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 429 if (!flg1) { 430 flg1 = PETSC_FALSE; 431 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 432 } 433 if (flg1) { 434 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 435 } 436 #endif 437 438 #if defined(PETSC_USE_LOG) 439 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 440 #endif 441 442 /* 443 Set the display variable for graphics 444 */ 445 ierr = PetscSetDisplay();CHKERRQ(ierr); 446 447 /* 448 Print main application help message 449 */ 450 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 451 if (help && hasHelp) { 452 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 453 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 454 } 455 456 /* 457 Print the PETSc version information 458 */ 459 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 460 if (flg1 || hasHelp) { 461 /* 462 Print "higher-level" package version message 463 */ 464 if (PetscExternalVersionFunction) { 465 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 466 } 467 468 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 476 } 477 478 /* 479 Print "higher-level" package help message 480 */ 481 if (hasHelp) { 482 PetscBool hasHelpIntro; 483 484 if (PetscExternalHelpFunction) { 485 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 486 } 487 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 488 if (hasHelpIntro) { 489 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 490 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 491 ierr = MPI_Finalize();CHKERRMPI(ierr); 492 exit(0); 493 } 494 } 495 496 /* 497 Setup the error handling 498 */ 499 flg1 = PETSC_FALSE; 500 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 501 if (flg1) { 502 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 503 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 504 } 505 flg1 = PETSC_FALSE; 506 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 507 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 508 flg1 = PETSC_FALSE; 509 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 510 if (flg1) { 511 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 512 } 513 flg1 = PETSC_FALSE; 514 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 515 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 516 517 /* 518 Setup debugger information 519 */ 520 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 521 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 522 if (flg1) { 523 MPI_Errhandler err_handler; 524 525 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 526 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 527 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 528 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 529 } 530 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 531 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 532 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 533 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 534 if (flg1 || flg2) { 535 PetscMPIInt size; 536 PetscInt lsize,*ranks; 537 MPI_Errhandler err_handler; 538 /* 539 we have to make sure that all processors have opened 540 connections to all other processors, otherwise once the 541 debugger has stated it is likely to receive a SIGUSR1 542 and kill the program. 543 */ 544 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 545 if (size > 2) { 546 PetscMPIInt dummy = 0; 547 MPI_Status status; 548 for (i=0; i<size; i++) { 549 if (rank != i) { 550 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 551 } 552 } 553 for (i=0; i<size; i++) { 554 if (rank != i) { 555 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 556 } 557 } 558 } 559 /* check if this processor node should be in debugger */ 560 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 561 lsize = size; 562 /* Deprecated in 3.14 */ 563 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 564 if (flag) { 565 const char * const quietopt="-options_suppress_deprecated_warnings"; 566 char msg[4096]; 567 PetscBool quiet = PETSC_FALSE; 568 569 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 570 if (!quiet) { 571 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 572 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 573 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 574 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 576 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 577 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 579 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 580 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 581 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 582 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 583 } 584 } else { 585 lsize = size; 586 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 587 } 588 if (flag) { 589 for (i=0; i<lsize; i++) { 590 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 591 } 592 } 593 if (!flag) { 594 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 595 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 596 if (flg1) { 597 ierr = PetscAttachDebugger();CHKERRQ(ierr); 598 } else { 599 ierr = PetscStopForDebugger();CHKERRQ(ierr); 600 } 601 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 602 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 603 } else { 604 ierr = PetscWaitOnError();CHKERRQ(ierr); 605 } 606 ierr = PetscFree(ranks);CHKERRQ(ierr); 607 } 608 609 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 610 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 611 612 /* 613 Setup profiling and logging 614 */ 615 #if defined(PETSC_USE_INFO) 616 { 617 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 618 } 619 #endif 620 ierr = PetscDetermineInitialFPTrap(); 621 flg1 = PETSC_FALSE; 622 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 623 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 624 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 625 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 626 #if defined(PETSC_USE_LOG) 627 mname[0] = 0; 628 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 629 if (flg1) { 630 if (mname[0]) { 631 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 632 } else { 633 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 634 } 635 } 636 637 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 638 639 #if defined(PETSC_HAVE_MPE) 640 flg1 = PETSC_FALSE; 641 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 642 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 643 #endif 644 flg1 = PETSC_FALSE; 645 flg3 = PETSC_FALSE; 646 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 647 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 648 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 649 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 650 651 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 652 if (flg1) { 653 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 654 FILE *file; 655 if (mname[0]) { 656 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 657 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 658 file = fopen(fname,"w"); 659 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 660 } else file = PETSC_STDOUT; 661 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 662 } 663 664 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 665 if (flg4) { 666 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 667 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 668 } else { 669 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 670 } 671 } 672 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 673 PetscReal threshold = PetscRealConstant(0.01); 674 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 675 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 676 } 677 #endif 678 679 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 680 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 681 /* 682 If collecting logging information, by default, wait for device to complete its operations 683 before returning to the CPU in order to get accurate timings of each event 684 */ 685 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 686 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 687 688 #if defined(PETSC_HAVE_CUDA) 689 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 690 #endif 691 692 #if defined(PETSC_HAVE_HIP) 693 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 694 #endif 695 696 /* 697 Print basic help message 698 */ 699 if (hasHelp) { 700 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 735 #if defined(PETSC_USE_LOG) 736 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 740 #if defined(PETSC_HAVE_MPE) 741 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 742 #endif 743 #endif 744 #if defined(PETSC_USE_INFO) 745 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 746 #endif 747 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 749 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 750 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 751 } 752 753 #if defined(PETSC_HAVE_POPEN) 754 { 755 char machine[128]; 756 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 757 if (flg1) { 758 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 759 } 760 } 761 #endif 762 763 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 764 if (flg1) { 765 ierr = PetscSleep(si);CHKERRQ(ierr); 766 } 767 768 #if defined(PETSC_HAVE_VIENNACL) 769 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 770 if (!flg3) { 771 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 772 } 773 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 774 PetscViennaCLSynchronize = flg3; 775 ierr = PetscViennaCLInit();CHKERRQ(ierr); 776 #endif 777 778 /* 779 Creates the logging data structures; this is enabled even if logging is not turned on 780 This is the last thing we do before returning to the user code to prevent having the 781 logging numbers contaminated by any startup time associated with MPI and the GPUs 782 */ 783 #if defined(PETSC_USE_LOG) 784 ierr = PetscLogInitialize();CHKERRQ(ierr); 785 #endif 786 787 PetscFunctionReturn(0); 788 } 789