1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 63 64 #if defined(PETSC_HAVE_COMPLEX) 65 #if defined(PETSC_COMPLEX_INSTANTIATE) 66 template <> class std::complex<double>; /* instantiate complex template class */ 67 #endif 68 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 69 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 70 MPI_Datatype MPIU_C_COMPLEX; 71 #endif 72 73 /*MC 74 PETSC_i - the imaginary number i 75 76 Synopsis: 77 #include <petscsys.h> 78 PetscComplex PETSC_i; 79 80 Level: beginner 81 82 Note: 83 Complex numbers are automatically available if PETSc located a working complex implementation 84 85 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 86 M*/ 87 PetscComplex PETSC_i; 88 #endif 89 #if defined(PETSC_USE_REAL___FLOAT128) 90 MPI_Datatype MPIU___FLOAT128 = 0; 91 #if defined(PETSC_HAVE_COMPLEX) 92 MPI_Datatype MPIU___COMPLEX128 = 0; 93 #endif 94 #elif defined(PETSC_USE_REAL___FP16) 95 MPI_Datatype MPIU___FP16 = 0; 96 #endif 97 MPI_Datatype MPIU_2SCALAR = 0; 98 #if defined(PETSC_USE_64BIT_INDICES) 99 MPI_Datatype MPIU_2INT = 0; 100 #endif 101 MPI_Datatype MPIU_BOOL; 102 MPI_Datatype MPIU_ENUM; 103 MPI_Datatype MPIU_FORTRANADDR; 104 MPI_Datatype MPIU_SIZE_T; 105 106 /* 107 Function that is called to display all error messages 108 */ 109 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 110 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 111 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 112 /* 113 This is needed to turn on/off GPU synchronization 114 */ 115 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 116 117 /* ------------------------------------------------------------------------------*/ 118 /* 119 Optional file where all PETSc output from various prints is saved 120 */ 121 PETSC_INTERN FILE *petsc_history; 122 FILE *petsc_history = NULL; 123 124 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 125 { 126 PetscErrorCode ierr; 127 PetscMPIInt rank,size; 128 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 129 char version[256]; 130 131 PetscFunctionBegin; 132 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 133 if (!rank) { 134 char arch[10]; 135 int err; 136 137 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 138 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 139 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 140 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 141 if (filename) { 142 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 143 } else { 144 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 145 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 146 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 147 } 148 149 *fd = fopen(fname,"a"); 150 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 151 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 153 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 154 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 156 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 157 158 err = fflush(*fd); 159 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 160 } 161 PetscFunctionReturn(0); 162 } 163 164 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 165 { 166 PetscErrorCode ierr; 167 PetscMPIInt rank; 168 char date[64]; 169 int err; 170 171 PetscFunctionBegin; 172 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 173 if (!rank) { 174 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 177 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 178 err = fflush(*fd); 179 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 180 err = fclose(*fd); 181 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 182 } 183 PetscFunctionReturn(0); 184 } 185 186 /* ------------------------------------------------------------------------------*/ 187 188 /* 189 This is ugly and probably belongs somewhere else, but I want to 190 be able to put a true MPI abort error handler with command line args. 191 192 This is so MPI errors in the debugger will leave all the stack 193 frames. The default MP_Abort() cleans up and exits thus providing no useful information 194 in the debugger hence we call abort() instead of MPI_Abort(). 195 */ 196 197 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 198 { 199 PetscFunctionBegin; 200 (*PetscErrorPrintf)("MPI error %d\n",*flag); 201 abort(); 202 } 203 204 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 205 { 206 PetscErrorCode ierr; 207 208 PetscFunctionBegin; 209 (*PetscErrorPrintf)("MPI error %d\n",*flag); 210 ierr = PetscAttachDebugger(); 211 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 212 } 213 214 /*@C 215 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 216 wishes a clean exit somewhere deep in the program. 217 218 Collective on PETSC_COMM_WORLD 219 220 Options Database Keys are the same as for PetscFinalize() 221 222 Level: advanced 223 224 Note: 225 See PetscInitialize() for more general runtime options. 226 227 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 228 @*/ 229 PetscErrorCode PetscEnd(void) 230 { 231 PetscFunctionBegin; 232 PetscFinalize(); 233 exit(0); 234 return 0; 235 } 236 237 PetscBool PetscOptionsPublish = PETSC_FALSE; 238 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 239 PETSC_INTERN PetscBool petscsetmallocvisited; 240 static char emacsmachinename[256]; 241 242 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 243 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 244 245 /*@C 246 PetscSetHelpVersionFunctions - Sets functions that print help and version information 247 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 248 This routine enables a "higher-level" package that uses PETSc to print its messages first. 249 250 Input Parameter: 251 + help - the help function (may be NULL) 252 - version - the version function (may be NULL) 253 254 Level: developer 255 256 @*/ 257 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 258 { 259 PetscFunctionBegin; 260 PetscExternalHelpFunction = help; 261 PetscExternalVersionFunction = version; 262 PetscFunctionReturn(0); 263 } 264 265 #if defined(PETSC_USE_LOG) 266 PETSC_INTERN PetscBool PetscObjectsLog; 267 #endif 268 269 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 270 Use the following macros to define CUDA/HIP initialization related vars/routines. 271 */ 272 #if defined(PETSC_HAVE_CUDA) 273 typedef cudaError_t cupmError_t; 274 typedef struct cudaDeviceProp cupmDeviceProp; 275 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 276 #define cupmGetDevice(x) cudaGetDevice(x) 277 #define cupmSetDevice(x) cudaSetDevice(x) 278 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 279 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 280 #define cupmGetLastError() cudaGetLastError() 281 #define cupmDeviceMapHost cudaDeviceMapHost 282 #define cupmSuccess cudaSuccess 283 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 284 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 285 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 286 #define CHKERRCUPM(x) CHKERRCUDA(x) 287 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 288 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 289 #define PetscCUPMInitialize PetscCUDAInitialize 290 #define PetscCUPMInitialized PetscCUDAInitialized 291 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 292 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 293 #define PetscCUPMSynchronize PetscCUDASynchronize 294 #define PetscNotUseCUPM PetscNotUseCUDA 295 #define cupmOptionsStr "CUDA options" 296 #define cupmSetDeviceStr "-cuda_device" 297 #define cupmViewStr "-cuda_view" 298 #define cupmSynchronizeStr "-cuda_synchronize" 299 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 300 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 301 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 302 #include "cupminit.inc" 303 #endif 304 305 #if defined(PETSC_HAVE_HIP) 306 typedef hipError_t cupmError_t; 307 typedef hipDeviceProp_t cupmDeviceProp; 308 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 309 #define cupmGetDevice(x) hipGetDevice(x) 310 #define cupmSetDevice(x) hipSetDevice(x) 311 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 312 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 313 #define cupmGetLastError() hipGetLastError() 314 #define cupmDeviceMapHost hipDeviceMapHost 315 #define cupmSuccess hipSuccess 316 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 317 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 318 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 319 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 320 #define PetscCUPMBLASInitializeHandle() 0 321 #define PetscCUPMSOLVERDnInitializeHandle() 0 322 #define PetscCUPMInitialize PetscHIPInitialize 323 #define PetscCUPMInitialized PetscHIPInitialized 324 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 325 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 326 #define PetscCUPMSynchronize PetscHIPSynchronize 327 #define PetscNotUseCUPM PetscNotUseHIP 328 #define cupmOptionsStr "HIP options" 329 #define cupmSetDeviceStr "-hip_device" 330 #define cupmViewStr "-hip_view" 331 #define cupmSynchronizeStr "-hip_synchronize" 332 #define PetscCUPMInitializeStr "PetscHIPInitialize" 333 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 334 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 335 #include "cupminit.inc" 336 #endif 337 338 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 339 { 340 char string[64]; 341 MPI_Comm comm = PETSC_COMM_WORLD; 342 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 343 PetscErrorCode ierr; 344 PetscReal si; 345 PetscInt intensity; 346 int i; 347 PetscMPIInt rank; 348 char version[256]; 349 #if defined(PETSC_USE_LOG) 350 char mname[PETSC_MAX_PATH_LEN]; 351 PetscViewerFormat format; 352 PetscBool flg4 = PETSC_FALSE; 353 #endif 354 355 PetscFunctionBegin; 356 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 357 358 #if !defined(PETSC_HAVE_THREADSAFETY) 359 if (!(PETSC_RUNNING_ON_VALGRIND)) { 360 /* 361 Setup the memory management; support for tracing malloc() usage 362 */ 363 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 364 365 if (PetscDefined(USE_DEBUG)) { 366 mdebug = PETSC_TRUE; 367 initializenan = PETSC_TRUE; 368 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 369 } else { 370 /* don't warn about unused option */ 371 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 372 flg1 = PETSC_FALSE; 373 } 374 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 375 if (flg1 || flg2) { 376 mdebug = PETSC_TRUE; 377 eachcall = PETSC_TRUE; 378 initializenan = PETSC_TRUE; 379 } else if (flg3 && !flg2) { 380 mdebug = PETSC_FALSE; 381 eachcall = PETSC_FALSE; 382 initializenan = PETSC_FALSE; 383 } 384 385 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 386 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 387 388 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 389 if (mlog) { 390 mdebug = PETSC_TRUE; 391 } 392 /* the next line is deprecated */ 393 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 394 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 395 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 396 if (mdebug) { 397 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 398 } 399 if (mlog) { 400 PetscReal logthreshold = 0; 401 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 402 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 403 } 404 #if defined(PETSC_USE_LOG) 405 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 406 #endif 407 } 408 409 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 410 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 411 flg1 = PETSC_FALSE; 412 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 413 /* ignore this option if malloc is already set */ 414 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 415 416 flg1 = PETSC_FALSE; 417 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 418 if (!flg1) { 419 flg1 = PETSC_FALSE; 420 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 421 } 422 if (flg1) { 423 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 424 } 425 #endif 426 427 #if defined(PETSC_USE_LOG) 428 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 429 #endif 430 431 /* 432 Set the display variable for graphics 433 */ 434 ierr = PetscSetDisplay();CHKERRQ(ierr); 435 436 /* 437 Print main application help message 438 */ 439 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 440 if (help && hasHelp) { 441 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 442 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 443 } 444 445 /* 446 Print the PETSc version information 447 */ 448 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 449 if (flg1 || hasHelp) { 450 /* 451 Print "higher-level" package version message 452 */ 453 if (PetscExternalVersionFunction) { 454 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 455 } 456 457 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 458 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 459 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 460 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 461 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 462 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 463 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 464 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 465 } 466 467 /* 468 Print "higher-level" package help message 469 */ 470 if (hasHelp) { 471 PetscBool hasHelpIntro; 472 473 if (PetscExternalHelpFunction) { 474 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 475 } 476 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 477 if (hasHelpIntro) { 478 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 479 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 480 ierr = MPI_Finalize();CHKERRMPI(ierr); 481 exit(0); 482 } 483 } 484 485 /* 486 Setup the error handling 487 */ 488 flg1 = PETSC_FALSE; 489 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 490 if (flg1) { 491 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 492 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 493 } 494 flg1 = PETSC_FALSE; 495 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 496 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 497 flg1 = PETSC_FALSE; 498 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 499 if (flg1) { 500 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 501 } 502 flg1 = PETSC_FALSE; 503 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 504 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 505 506 /* 507 Setup debugger information 508 */ 509 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 510 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 511 if (flg1) { 512 MPI_Errhandler err_handler; 513 514 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 515 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 516 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 517 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 518 } 519 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 520 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 521 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 522 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 523 if (flg1 || flg2) { 524 PetscMPIInt size; 525 PetscInt lsize,*ranks; 526 MPI_Errhandler err_handler; 527 /* 528 we have to make sure that all processors have opened 529 connections to all other processors, otherwise once the 530 debugger has stated it is likely to receive a SIGUSR1 531 and kill the program. 532 */ 533 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 534 if (size > 2) { 535 PetscMPIInt dummy = 0; 536 MPI_Status status; 537 for (i=0; i<size; i++) { 538 if (rank != i) { 539 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 540 } 541 } 542 for (i=0; i<size; i++) { 543 if (rank != i) { 544 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 545 } 546 } 547 } 548 /* check if this processor node should be in debugger */ 549 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 550 lsize = size; 551 /* Deprecated in 3.14 */ 552 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 553 if (flag) { 554 const char * const quietopt="-options_suppress_deprecated_warnings"; 555 char msg[4096]; 556 PetscBool quiet = PETSC_FALSE; 557 558 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 559 if (!quiet) { 560 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 561 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 562 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 563 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 564 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 565 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 566 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 567 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 568 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 569 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 570 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 571 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 572 } 573 } else { 574 lsize = size; 575 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 576 } 577 if (flag) { 578 for (i=0; i<lsize; i++) { 579 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 580 } 581 } 582 if (!flag) { 583 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 584 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 585 if (flg1) { 586 ierr = PetscAttachDebugger();CHKERRQ(ierr); 587 } else { 588 ierr = PetscStopForDebugger();CHKERRQ(ierr); 589 } 590 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 591 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 592 } else { 593 ierr = PetscWaitOnError();CHKERRQ(ierr); 594 } 595 ierr = PetscFree(ranks);CHKERRQ(ierr); 596 } 597 598 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 599 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 600 601 /* 602 Setup profiling and logging 603 */ 604 #if defined(PETSC_USE_INFO) 605 { 606 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 607 } 608 #endif 609 ierr = PetscDetermineInitialFPTrap(); 610 flg1 = PETSC_FALSE; 611 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 612 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 613 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 614 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 615 #if defined(PETSC_USE_LOG) 616 mname[0] = 0; 617 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 618 if (flg1) { 619 if (mname[0]) { 620 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 621 } else { 622 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 623 } 624 } 625 626 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 627 628 #if defined(PETSC_HAVE_MPE) 629 flg1 = PETSC_FALSE; 630 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 631 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 632 #endif 633 flg1 = PETSC_FALSE; 634 flg3 = PETSC_FALSE; 635 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 636 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 637 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 638 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 639 640 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 641 if (flg1) { 642 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 643 FILE *file; 644 if (mname[0]) { 645 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 646 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 647 file = fopen(fname,"w"); 648 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 649 } else file = PETSC_STDOUT; 650 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 651 } 652 653 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 654 if (flg4) { 655 if (format == PETSC_VIEWER_ASCII_XML) { 656 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 657 } else { 658 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 659 } 660 } 661 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 662 PetscReal threshold = PetscRealConstant(0.01); 663 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 664 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 665 } 666 #endif 667 668 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 669 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 670 /* 671 If collecting logging information, by default, wait for device to complete its operations 672 before returning to the CPU in order to get accurate timings of each event 673 */ 674 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 675 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 676 677 #if defined(PETSC_HAVE_CUDA) 678 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 679 #endif 680 681 #if defined(PETSC_HAVE_HIP) 682 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 683 #endif 684 685 /* 686 Print basic help message 687 */ 688 if (hasHelp) { 689 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 690 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 693 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 694 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 724 #if defined(PETSC_USE_LOG) 725 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 729 #if defined(PETSC_HAVE_MPE) 730 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 731 #endif 732 #endif 733 #if defined(PETSC_USE_INFO) 734 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 735 #endif 736 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 740 } 741 742 #if defined(PETSC_HAVE_POPEN) 743 { 744 char machine[128]; 745 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 746 if (flg1) { 747 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 748 } 749 } 750 #endif 751 752 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 753 if (flg1) { 754 ierr = PetscSleep(si);CHKERRQ(ierr); 755 } 756 757 #if defined(PETSC_HAVE_VIENNACL) 758 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 759 if (!flg3) { 760 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 761 } 762 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 763 PetscViennaCLSynchronize = flg3; 764 ierr = PetscViennaCLInit();CHKERRQ(ierr); 765 #endif 766 767 /* 768 Creates the logging data structures; this is enabled even if logging is not turned on 769 This is the last thing we do before returning to the user code to prevent having the 770 logging numbers contaminated by any startup time associated with MPI and the GPUs 771 */ 772 #if defined(PETSC_USE_LOG) 773 ierr = PetscLogInitialize();CHKERRQ(ierr); 774 #endif 775 776 PetscFunctionReturn(0); 777 } 778