1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscviewer.h> 12 #if defined(PETSC_USE_LOG) 13 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 14 #endif 15 16 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 17 #include <sys/sysinfo.h> 18 #endif 19 #if defined(PETSC_HAVE_UNISTD_H) 20 #include <unistd.h> 21 #endif 22 23 #if defined(PETSC_HAVE_CUDA) || defined(PETSC_HAVE_HIP) 24 # include <petscdevice.h> 25 #endif 26 27 #if defined(PETSC_HAVE_DEVICE) 28 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 29 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 30 #endif 31 #endif 32 33 #if defined(PETSC_HAVE_VIENNACL) 34 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 35 #endif 36 37 /* ------------------------Nasty global variables -------------------------------*/ 38 /* 39 Indicates if PETSc started up MPI, or it was 40 already started before PETSc was initialized. 41 */ 42 PetscBool PetscBeganMPI = PETSC_FALSE; 43 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 44 PetscBool PetscInitializeCalled = PETSC_FALSE; 45 PetscBool PetscFinalizeCalled = PETSC_FALSE; 46 47 PetscMPIInt PetscGlobalRank = -1; 48 PetscMPIInt PetscGlobalSize = -1; 49 50 #if defined(PETSC_HAVE_KOKKOS) 51 PetscBool PetscBeganKokkos = PETSC_FALSE; 52 #endif 53 54 #if defined(PETSC_HAVE_NVSHMEM) 55 PetscBool PetscBeganNvshmem = PETSC_FALSE; 56 PetscBool PetscNvshmemInitialized = PETSC_FALSE; 57 #endif 58 59 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 60 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 61 62 #if defined(PETSC_HAVE_COMPLEX) 63 #if defined(PETSC_COMPLEX_INSTANTIATE) 64 template <> class std::complex<double>; /* instantiate complex template class */ 65 #endif 66 67 /*MC 68 PETSC_i - the imaginary number i 69 70 Synopsis: 71 #include <petscsys.h> 72 PetscComplex PETSC_i; 73 74 Level: beginner 75 76 Note: 77 Complex numbers are automatically available if PETSc located a working complex implementation 78 79 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 80 M*/ 81 PetscComplex PETSC_i; 82 MPI_Datatype MPIU___COMPLEX128 = 0; 83 #endif /* PETSC_HAVE_COMPLEX */ 84 #if defined(PETSC_USE_REAL___FLOAT128) 85 MPI_Datatype MPIU___FLOAT128 = 0; 86 #elif defined(PETSC_USE_REAL___FP16) 87 MPI_Datatype MPIU___FP16 = 0; 88 #endif 89 MPI_Datatype MPIU_2SCALAR = 0; 90 MPI_Datatype MPIU_REAL_INT = 0; 91 MPI_Datatype MPIU_SCALAR_INT = 0; 92 #if defined(PETSC_USE_64BIT_INDICES) 93 MPI_Datatype MPIU_2INT = 0; 94 #endif 95 MPI_Datatype MPI_4INT = 0; 96 MPI_Datatype MPIU_4INT = 0; 97 MPI_Datatype MPIU_BOOL; 98 MPI_Datatype MPIU_ENUM; 99 MPI_Datatype MPIU_FORTRANADDR; 100 MPI_Datatype MPIU_SIZE_T; 101 102 /* 103 Function that is called to display all error messages 104 */ 105 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 106 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 107 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 108 /* 109 This is needed to turn on/off GPU synchronization 110 */ 111 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 112 113 /* ------------------------------------------------------------------------------*/ 114 /* 115 Optional file where all PETSc output from various prints is saved 116 */ 117 PETSC_INTERN FILE *petsc_history; 118 FILE *petsc_history = NULL; 119 120 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 121 { 122 PetscErrorCode ierr; 123 PetscMPIInt rank,size; 124 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 125 char version[256]; 126 127 PetscFunctionBegin; 128 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 129 if (rank == 0) { 130 char arch[10]; 131 int err; 132 133 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 134 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 135 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 136 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 137 if (filename) { 138 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 139 } else { 140 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 141 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 142 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 143 } 144 145 *fd = fopen(fname,"a"); 146 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 147 148 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 149 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 150 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 153 154 err = fflush(*fd); 155 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 156 } 157 PetscFunctionReturn(0); 158 } 159 160 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 161 { 162 PetscErrorCode ierr; 163 PetscMPIInt rank; 164 char date[64]; 165 int err; 166 167 PetscFunctionBegin; 168 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 169 if (rank == 0) { 170 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 171 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 172 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 173 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 174 err = fflush(*fd); 175 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 176 err = fclose(*fd); 177 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 178 } 179 PetscFunctionReturn(0); 180 } 181 182 /* ------------------------------------------------------------------------------*/ 183 184 /* 185 This is ugly and probably belongs somewhere else, but I want to 186 be able to put a true MPI abort error handler with command line args. 187 188 This is so MPI errors in the debugger will leave all the stack 189 frames. The default MP_Abort() cleans up and exits thus providing no useful information 190 in the debugger hence we call abort() instead of MPI_Abort(). 191 */ 192 193 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 194 { 195 PetscFunctionBegin; 196 (*PetscErrorPrintf)("MPI error %d\n",*flag); 197 abort(); 198 } 199 200 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 201 { 202 PetscErrorCode ierr; 203 204 PetscFunctionBegin; 205 (*PetscErrorPrintf)("MPI error %d\n",*flag); 206 ierr = PetscAttachDebugger(); 207 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 208 } 209 210 /*@C 211 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 212 wishes a clean exit somewhere deep in the program. 213 214 Collective on PETSC_COMM_WORLD 215 216 Options Database Keys are the same as for PetscFinalize() 217 218 Level: advanced 219 220 Note: 221 See PetscInitialize() for more general runtime options. 222 223 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 224 @*/ 225 PetscErrorCode PetscEnd(void) 226 { 227 PetscFunctionBegin; 228 PetscFinalize(); 229 exit(0); 230 return 0; 231 } 232 233 PetscBool PetscOptionsPublish = PETSC_FALSE; 234 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 235 PETSC_INTERN PetscBool petscsetmallocvisited; 236 static char emacsmachinename[256]; 237 238 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 239 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 240 241 /*@C 242 PetscSetHelpVersionFunctions - Sets functions that print help and version information 243 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 244 This routine enables a "higher-level" package that uses PETSc to print its messages first. 245 246 Input Parameters: 247 + help - the help function (may be NULL) 248 - version - the version function (may be NULL) 249 250 Level: developer 251 252 @*/ 253 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 254 { 255 PetscFunctionBegin; 256 PetscExternalHelpFunction = help; 257 PetscExternalVersionFunction = version; 258 PetscFunctionReturn(0); 259 } 260 261 #if defined(PETSC_USE_LOG) 262 PETSC_INTERN PetscBool PetscObjectsLog; 263 #endif 264 265 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 266 Use the following macros to define CUDA/HIP initialization related vars/routines. 267 */ 268 #if defined(PETSC_HAVE_CUDA) 269 typedef cudaError_t cupmError_t; 270 typedef struct cudaDeviceProp cupmDeviceProp; 271 typedef cudaStream_t cupmStream_t; 272 typedef cudaEvent_t cupmEvent_t; 273 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 274 #define cupmGetDevice(x) cudaGetDevice(x) 275 #define cupmSetDevice(x) cudaSetDevice(x) 276 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 277 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 278 #define cupmStreamCreate(x) cudaStreamCreate(x) 279 #define cupmGetLastError() cudaGetLastError() 280 #define cupmDeviceMapHost cudaDeviceMapHost 281 #define cupmSuccess cudaSuccess 282 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 283 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 284 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 285 #define CHKERRCUPM(x) CHKERRCUDA(x) 286 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 287 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 288 #define PetscCUPMInitialize PetscCUDAInitialize 289 #define PetscCUPMInitialized PetscCUDAInitialized 290 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 291 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 292 #define PetscCUPMSynchronize PetscCUDASynchronize 293 #define PetscNotUseCUPM PetscNotUseCUDA 294 #define cupmOptionsStr "CUDA options" 295 #define cupmSetDeviceStr "-cuda_device" 296 #define cupmViewStr "-cuda_view" 297 #define cupmSynchronizeStr "-cuda_synchronize" 298 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 299 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 300 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 301 #define PetscDefaultCupmStream PetscDefaultCudaStream 302 #define cupmEventCreate(x) cudaEventCreate(x) 303 #include "cupminit.inc" 304 #endif 305 306 #if defined(PETSC_HAVE_HIP) 307 typedef hipError_t cupmError_t; 308 typedef hipDeviceProp_t cupmDeviceProp; 309 typedef hipStream_t cupmStream_t; 310 typedef hipEvent_t cupmEvent_t; 311 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 312 #define cupmGetDevice(x) hipGetDevice(x) 313 #define cupmSetDevice(x) hipSetDevice(x) 314 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 315 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 316 #define cupmStreamCreate(x) hipStreamCreate(x) 317 #define cupmEventCreate(x) hipEventCreate(x); 318 #define cupmGetLastError() hipGetLastError() 319 #define cupmDeviceMapHost hipDeviceMapHost 320 #define cupmSuccess hipSuccess 321 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 322 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 323 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 324 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 325 #define PetscCUPMBLASInitializeHandle() 0 326 #define PetscCUPMSOLVERDnInitializeHandle() 0 327 #define PetscCUPMInitialize PetscHIPInitialize 328 #define PetscCUPMInitialized PetscHIPInitialized 329 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 330 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 331 #define PetscCUPMSynchronize PetscHIPSynchronize 332 #define PetscNotUseCUPM PetscNotUseHIP 333 #define cupmOptionsStr "HIP options" 334 #define cupmSetDeviceStr "-hip_device" 335 #define cupmViewStr "-hip_view" 336 #define cupmSynchronizeStr "-hip_synchronize" 337 #define PetscCUPMInitializeStr "PetscHIPInitialize" 338 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 339 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 340 #define PetscDefaultCupmStream PetscDefaultHipStream 341 #include "cupminit.inc" 342 #endif 343 344 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 345 { 346 char string[64]; 347 MPI_Comm comm = PETSC_COMM_WORLD; 348 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 349 PetscErrorCode ierr; 350 PetscReal si; 351 PetscInt intensity; 352 int i; 353 PetscMPIInt rank; 354 char version[256]; 355 #if defined(PETSC_USE_LOG) 356 char mname[PETSC_MAX_PATH_LEN]; 357 PetscViewerFormat format; 358 PetscBool flg4 = PETSC_FALSE; 359 #endif 360 361 PetscFunctionBegin; 362 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 363 364 /* 365 Setup building of stack frames for all function calls 366 */ 367 if (PetscDefined(USE_DEBUG) && !PetscDefined(HAVE_THREADSAFETY)) { 368 ierr = PetscOptionsGetBool(NULL,NULL,"-checkstack",&flg1,NULL);CHKERRQ(ierr); 369 ierr = PetscStackSetCheck(flg1);CHKERRQ(ierr); 370 } 371 372 #if !defined(PETSC_HAVE_THREADSAFETY) 373 if (!(PETSC_RUNNING_ON_VALGRIND)) { 374 /* 375 Setup the memory management; support for tracing malloc() usage 376 */ 377 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 378 379 if (PetscDefined(USE_DEBUG)) { 380 mdebug = PETSC_TRUE; 381 initializenan = PETSC_TRUE; 382 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 383 } else { 384 /* don't warn about unused option */ 385 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 386 flg1 = PETSC_FALSE; 387 } 388 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 389 if (flg1 || flg2) { 390 mdebug = PETSC_TRUE; 391 eachcall = PETSC_TRUE; 392 initializenan = PETSC_TRUE; 393 } else if (flg3 && !flg2) { 394 mdebug = PETSC_FALSE; 395 eachcall = PETSC_FALSE; 396 initializenan = PETSC_FALSE; 397 } 398 399 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 400 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 401 402 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 403 if (mlog) { 404 mdebug = PETSC_TRUE; 405 } 406 /* the next line is deprecated */ 407 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 408 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 409 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 410 if (mdebug) { 411 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 412 } 413 if (mlog) { 414 PetscReal logthreshold = 0; 415 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 416 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 417 } 418 #if defined(PETSC_USE_LOG) 419 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 420 #endif 421 } 422 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 424 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 425 flg1 = PETSC_FALSE; 426 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 427 /* ignore this option if malloc is already set */ 428 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 429 430 flg1 = PETSC_FALSE; 431 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 432 if (!flg1) { 433 flg1 = PETSC_FALSE; 434 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 435 } 436 if (flg1) { 437 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 438 } 439 #endif 440 441 #if defined(PETSC_USE_LOG) 442 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 443 #endif 444 445 /* 446 Set the display variable for graphics 447 */ 448 ierr = PetscSetDisplay();CHKERRQ(ierr); 449 450 /* 451 Print main application help message 452 */ 453 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 454 if (help && hasHelp) { 455 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 456 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 457 } 458 459 /* 460 Print the PETSc version information 461 */ 462 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 463 if (flg1 || hasHelp) { 464 /* 465 Print "higher-level" package version message 466 */ 467 if (PetscExternalVersionFunction) { 468 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 469 } 470 471 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 477 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 478 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 479 } 480 481 /* 482 Print "higher-level" package help message 483 */ 484 if (hasHelp) { 485 PetscBool hasHelpIntro; 486 487 if (PetscExternalHelpFunction) { 488 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 489 } 490 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 491 if (hasHelpIntro) { 492 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 493 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 494 ierr = MPI_Finalize();CHKERRMPI(ierr); 495 exit(0); 496 } 497 } 498 499 /* 500 Setup the error handling 501 */ 502 flg1 = PETSC_FALSE; 503 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 504 if (flg1) { 505 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 506 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 507 } 508 flg1 = PETSC_FALSE; 509 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 510 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 511 flg1 = PETSC_FALSE; 512 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 513 if (flg1) { 514 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 515 } 516 flg1 = PETSC_FALSE; 517 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 518 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 519 520 /* 521 Setup debugger information 522 */ 523 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 524 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 525 if (flg1) { 526 MPI_Errhandler err_handler; 527 528 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 529 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 530 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 531 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 532 } 533 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 534 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 535 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 536 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 537 if (flg1 || flg2) { 538 PetscMPIInt size; 539 PetscInt lsize,*ranks; 540 MPI_Errhandler err_handler; 541 /* 542 we have to make sure that all processors have opened 543 connections to all other processors, otherwise once the 544 debugger has stated it is likely to receive a SIGUSR1 545 and kill the program. 546 */ 547 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 548 if (size > 2) { 549 PetscMPIInt dummy = 0; 550 MPI_Status status; 551 for (i=0; i<size; i++) { 552 if (rank != i) { 553 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 554 } 555 } 556 for (i=0; i<size; i++) { 557 if (rank != i) { 558 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 559 } 560 } 561 } 562 /* check if this processor node should be in debugger */ 563 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 564 lsize = size; 565 /* Deprecated in 3.14 */ 566 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 567 if (flag) { 568 const char * const quietopt="-options_suppress_deprecated_warnings"; 569 char msg[4096]; 570 PetscBool quiet = PETSC_FALSE; 571 572 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 573 if (!quiet) { 574 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 575 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 576 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 577 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 578 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 579 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 580 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 581 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 582 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 583 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 584 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 585 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 586 } 587 } else { 588 lsize = size; 589 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 590 } 591 if (flag) { 592 for (i=0; i<lsize; i++) { 593 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 594 } 595 } 596 if (!flag) { 597 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 598 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 599 if (flg1) { 600 ierr = PetscAttachDebugger();CHKERRQ(ierr); 601 } else { 602 ierr = PetscStopForDebugger();CHKERRQ(ierr); 603 } 604 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 605 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 606 } else { 607 ierr = PetscWaitOnError();CHKERRQ(ierr); 608 } 609 ierr = PetscFree(ranks);CHKERRQ(ierr); 610 } 611 612 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 613 if (flg1 && rank == 0) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 614 615 /* 616 Setup profiling and logging 617 */ 618 #if defined(PETSC_USE_INFO) 619 { 620 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 621 } 622 #endif 623 ierr = PetscDetermineInitialFPTrap(); 624 flg1 = PETSC_FALSE; 625 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 626 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 627 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 628 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 629 #if defined(PETSC_USE_LOG) 630 mname[0] = 0; 631 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 632 if (flg1) { 633 if (mname[0]) { 634 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 635 } else { 636 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 637 } 638 } 639 640 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 641 642 #if defined(PETSC_HAVE_MPE) 643 flg1 = PETSC_FALSE; 644 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 645 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 646 #endif 647 flg1 = PETSC_FALSE; 648 flg3 = PETSC_FALSE; 649 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 650 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 651 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 652 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 653 654 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 655 if (flg1) { 656 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 657 FILE *file; 658 if (mname[0]) { 659 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 660 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 661 file = fopen(fname,"w"); 662 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 663 } else file = PETSC_STDOUT; 664 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 665 } 666 667 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 668 if (flg4) { 669 if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 670 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 671 } else { 672 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 673 } 674 } 675 if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) { 676 PetscReal threshold = PetscRealConstant(0.01); 677 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 678 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 679 } 680 #endif 681 682 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 683 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 684 /* 685 If collecting logging information, by default, wait for device to complete its operations 686 before returning to the CPU in order to get accurate timings of each event 687 */ 688 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 689 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 690 691 #if defined(PETSC_HAVE_CUDA) 692 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 693 #endif 694 695 #if defined(PETSC_HAVE_HIP) 696 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 697 #endif 698 699 /* 700 Print basic help message 701 */ 702 if (hasHelp) { 703 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 738 #if defined(PETSC_USE_LOG) 739 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 743 #if defined(PETSC_HAVE_MPE) 744 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 745 #endif 746 #endif 747 #if defined(PETSC_USE_INFO) 748 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 749 #endif 750 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 751 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 752 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 753 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 754 } 755 756 #if defined(PETSC_HAVE_POPEN) 757 { 758 char machine[128]; 759 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 760 if (flg1) { 761 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 762 } 763 } 764 #endif 765 766 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 767 if (flg1) { 768 ierr = PetscSleep(si);CHKERRQ(ierr); 769 } 770 771 #if defined(PETSC_HAVE_VIENNACL) 772 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 773 if (!flg3) { 774 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 775 } 776 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 777 PetscViennaCLSynchronize = flg3; 778 ierr = PetscViennaCLInit();CHKERRQ(ierr); 779 #endif 780 781 /* 782 Creates the logging data structures; this is enabled even if logging is not turned on 783 This is the last thing we do before returning to the user code to prevent having the 784 logging numbers contaminated by any startup time associated with MPI and the GPUs 785 */ 786 #if defined(PETSC_USE_LOG) 787 ierr = PetscLogInitialize();CHKERRQ(ierr); 788 #endif 789 790 PetscFunctionReturn(0); 791 } 792