1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 24 #if defined(PETSC_HAVE_CUDA) 25 #include <cuda_runtime.h> 26 #include <petsccublas.h> 27 #endif 28 29 #if defined(PETSC_HAVE_HIP) 30 #include <hip/hip_runtime.h> 31 #endif 32 33 #if defined(PETSC_HAVE_DEVICE) 34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 36 #endif 37 #endif 38 39 #if defined(PETSC_HAVE_VIENNACL) 40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 41 #endif 42 43 44 /* ------------------------Nasty global variables -------------------------------*/ 45 /* 46 Indicates if PETSc started up MPI, or it was 47 already started before PETSc was initialized. 48 */ 49 PetscBool PetscBeganMPI = PETSC_FALSE; 50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 51 PetscBool PetscInitializeCalled = PETSC_FALSE; 52 PetscBool PetscFinalizeCalled = PETSC_FALSE; 53 54 PetscMPIInt PetscGlobalRank = -1; 55 PetscMPIInt PetscGlobalSize = -1; 56 57 #if defined(PETSC_HAVE_KOKKOS) 58 PetscBool PetscBeganKokkos = PETSC_FALSE; 59 #endif 60 61 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 62 PetscBool PetscCreatedGpuObjects = PETSC_FALSE; 63 64 #if defined(PETSC_HAVE_COMPLEX) 65 #if defined(PETSC_COMPLEX_INSTANTIATE) 66 template <> class std::complex<double>; /* instantiate complex template class */ 67 #endif 68 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 69 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 70 MPI_Datatype MPIU_C_COMPLEX; 71 #endif 72 73 /*MC 74 PETSC_i - the imaginary number i 75 76 Synopsis: 77 #include <petscsys.h> 78 PetscComplex PETSC_i; 79 80 Level: beginner 81 82 Note: 83 Complex numbers are automatically available if PETSc located a working complex implementation 84 85 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 86 M*/ 87 PetscComplex PETSC_i; 88 MPI_Datatype MPIU___COMPLEX128 = 0; 89 #endif /* PETSC_HAVE_COMPLEX */ 90 #if defined(PETSC_USE_REAL___FLOAT128) 91 MPI_Datatype MPIU___FLOAT128 = 0; 92 #elif defined(PETSC_USE_REAL___FP16) 93 MPI_Datatype MPIU___FP16 = 0; 94 #endif 95 MPI_Datatype MPIU_2SCALAR = 0; 96 #if defined(PETSC_USE_64BIT_INDICES) 97 MPI_Datatype MPIU_2INT = 0; 98 #endif 99 MPI_Datatype MPIU_BOOL; 100 MPI_Datatype MPIU_ENUM; 101 MPI_Datatype MPIU_FORTRANADDR; 102 MPI_Datatype MPIU_SIZE_T; 103 104 /* 105 Function that is called to display all error messages 106 */ 107 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 108 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 109 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 110 /* 111 This is needed to turn on/off GPU synchronization 112 */ 113 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 114 115 /* ------------------------------------------------------------------------------*/ 116 /* 117 Optional file where all PETSc output from various prints is saved 118 */ 119 PETSC_INTERN FILE *petsc_history; 120 FILE *petsc_history = NULL; 121 122 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 123 { 124 PetscErrorCode ierr; 125 PetscMPIInt rank,size; 126 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 127 char version[256]; 128 129 PetscFunctionBegin; 130 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 131 if (!rank) { 132 char arch[10]; 133 int err; 134 135 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 136 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 137 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 138 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 139 if (filename) { 140 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 141 } else { 142 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 143 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 144 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 145 } 146 147 *fd = fopen(fname,"a"); 148 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 149 150 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 152 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 153 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 154 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 155 156 err = fflush(*fd); 157 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 158 } 159 PetscFunctionReturn(0); 160 } 161 162 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 163 { 164 PetscErrorCode ierr; 165 PetscMPIInt rank; 166 char date[64]; 167 int err; 168 169 PetscFunctionBegin; 170 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 171 if (!rank) { 172 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 173 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 174 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 176 err = fflush(*fd); 177 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 178 err = fclose(*fd); 179 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 180 } 181 PetscFunctionReturn(0); 182 } 183 184 /* ------------------------------------------------------------------------------*/ 185 186 /* 187 This is ugly and probably belongs somewhere else, but I want to 188 be able to put a true MPI abort error handler with command line args. 189 190 This is so MPI errors in the debugger will leave all the stack 191 frames. The default MP_Abort() cleans up and exits thus providing no useful information 192 in the debugger hence we call abort() instead of MPI_Abort(). 193 */ 194 195 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 196 { 197 PetscFunctionBegin; 198 (*PetscErrorPrintf)("MPI error %d\n",*flag); 199 abort(); 200 } 201 202 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 203 { 204 PetscErrorCode ierr; 205 206 PetscFunctionBegin; 207 (*PetscErrorPrintf)("MPI error %d\n",*flag); 208 ierr = PetscAttachDebugger(); 209 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 210 } 211 212 /*@C 213 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 214 wishes a clean exit somewhere deep in the program. 215 216 Collective on PETSC_COMM_WORLD 217 218 Options Database Keys are the same as for PetscFinalize() 219 220 Level: advanced 221 222 Note: 223 See PetscInitialize() for more general runtime options. 224 225 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 226 @*/ 227 PetscErrorCode PetscEnd(void) 228 { 229 PetscFunctionBegin; 230 PetscFinalize(); 231 exit(0); 232 return 0; 233 } 234 235 PetscBool PetscOptionsPublish = PETSC_FALSE; 236 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 237 PETSC_INTERN PetscBool petscsetmallocvisited; 238 static char emacsmachinename[256]; 239 240 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 241 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 242 243 /*@C 244 PetscSetHelpVersionFunctions - Sets functions that print help and version information 245 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 246 This routine enables a "higher-level" package that uses PETSc to print its messages first. 247 248 Input Parameter: 249 + help - the help function (may be NULL) 250 - version - the version function (may be NULL) 251 252 Level: developer 253 254 @*/ 255 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 256 { 257 PetscFunctionBegin; 258 PetscExternalHelpFunction = help; 259 PetscExternalVersionFunction = version; 260 PetscFunctionReturn(0); 261 } 262 263 #if defined(PETSC_USE_LOG) 264 PETSC_INTERN PetscBool PetscObjectsLog; 265 #endif 266 267 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP. 268 Use the following macros to define CUDA/HIP initialization related vars/routines. 269 */ 270 #if defined(PETSC_HAVE_CUDA) 271 typedef cudaError_t cupmError_t; 272 typedef struct cudaDeviceProp cupmDeviceProp; 273 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x) 274 #define cupmGetDevice(x) cudaGetDevice(x) 275 #define cupmSetDevice(x) cudaSetDevice(x) 276 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x) 277 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y) 278 #define cupmGetLastError() cudaGetLastError() 279 #define cupmDeviceMapHost cudaDeviceMapHost 280 #define cupmSuccess cudaSuccess 281 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation 282 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources 283 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess 284 #define CHKERRCUPM(x) CHKERRCUDA(x) 285 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle() 286 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle() 287 #define PetscCUPMInitialize PetscCUDAInitialize 288 #define PetscCUPMInitialized PetscCUDAInitialized 289 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck 290 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView 291 #define PetscCUPMSynchronize PetscCUDASynchronize 292 #define PetscNotUseCUPM PetscNotUseCUDA 293 #define cupmOptionsStr "CUDA options" 294 #define cupmSetDeviceStr "-cuda_device" 295 #define cupmViewStr "-cuda_view" 296 #define cupmSynchronizeStr "-cuda_synchronize" 297 #define PetscCUPMInitializeStr "PetscCUDAInitialize" 298 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA 299 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck 300 #include "cupminit.inc" 301 #endif 302 303 #if defined(PETSC_HAVE_HIP) 304 typedef hipError_t cupmError_t; 305 typedef hipDeviceProp_t cupmDeviceProp; 306 #define cupmGetDeviceCount(x) hipGetDeviceCount(x) 307 #define cupmGetDevice(x) hipGetDevice(x) 308 #define cupmSetDevice(x) hipSetDevice(x) 309 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x) 310 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y) 311 #define cupmGetLastError() hipGetLastError() 312 #define cupmDeviceMapHost hipDeviceMapHost 313 #define cupmSuccess hipSuccess 314 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation 315 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources 316 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess 317 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB) 318 #define PetscCUPMBLASInitializeHandle() 0 319 #define PetscCUPMSOLVERDnInitializeHandle() 0 320 #define PetscCUPMInitialize PetscHIPInitialize 321 #define PetscCUPMInitialized PetscHIPInitialized 322 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck 323 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView 324 #define PetscCUPMSynchronize PetscHIPSynchronize 325 #define PetscNotUseCUPM PetscNotUseHIP 326 #define cupmOptionsStr "HIP options" 327 #define cupmSetDeviceStr "-hip_device" 328 #define cupmViewStr "-hip_view" 329 #define cupmSynchronizeStr "-hip_synchronize" 330 #define PetscCUPMInitializeStr "PetscHIPInitialize" 331 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP 332 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck 333 #include "cupminit.inc" 334 #endif 335 336 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 337 { 338 char string[64]; 339 MPI_Comm comm = PETSC_COMM_WORLD; 340 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView; 341 PetscErrorCode ierr; 342 PetscReal si; 343 PetscInt intensity; 344 int i; 345 PetscMPIInt rank; 346 char version[256]; 347 #if defined(PETSC_USE_LOG) 348 char mname[PETSC_MAX_PATH_LEN]; 349 PetscViewerFormat format; 350 PetscBool flg4 = PETSC_FALSE; 351 #endif 352 353 PetscFunctionBegin; 354 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 355 356 #if !defined(PETSC_HAVE_THREADSAFETY) 357 if (!(PETSC_RUNNING_ON_VALGRIND)) { 358 /* 359 Setup the memory management; support for tracing malloc() usage 360 */ 361 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 362 363 if (PetscDefined(USE_DEBUG)) { 364 mdebug = PETSC_TRUE; 365 initializenan = PETSC_TRUE; 366 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 367 } else { 368 /* don't warn about unused option */ 369 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 370 flg1 = PETSC_FALSE; 371 } 372 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 373 if (flg1 || flg2) { 374 mdebug = PETSC_TRUE; 375 eachcall = PETSC_TRUE; 376 initializenan = PETSC_TRUE; 377 } else if (flg3 && !flg2) { 378 mdebug = PETSC_FALSE; 379 eachcall = PETSC_FALSE; 380 initializenan = PETSC_FALSE; 381 } 382 383 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr); 384 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);} 385 386 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 387 if (mlog) { 388 mdebug = PETSC_TRUE; 389 } 390 /* the next line is deprecated */ 391 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 392 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 393 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 394 if (mdebug) { 395 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 396 } 397 if (mlog) { 398 PetscReal logthreshold = 0; 399 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 400 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 401 } 402 #if defined(PETSC_USE_LOG) 403 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 404 #endif 405 } 406 407 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 408 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 409 flg1 = PETSC_FALSE; 410 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 411 /* ignore this option if malloc is already set */ 412 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 413 414 flg1 = PETSC_FALSE; 415 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 416 if (!flg1) { 417 flg1 = PETSC_FALSE; 418 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 419 } 420 if (flg1) { 421 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 422 } 423 #endif 424 425 #if defined(PETSC_USE_LOG) 426 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 427 #endif 428 429 /* 430 Set the display variable for graphics 431 */ 432 ierr = PetscSetDisplay();CHKERRQ(ierr); 433 434 /* 435 Print main application help message 436 */ 437 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 438 if (help && hasHelp) { 439 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 440 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 441 } 442 443 /* 444 Print the PETSc version information 445 */ 446 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 447 if (flg1 || hasHelp) { 448 /* 449 Print "higher-level" package version message 450 */ 451 if (PetscExternalVersionFunction) { 452 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 453 } 454 455 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 456 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 457 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 458 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 459 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 460 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 461 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 462 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 463 } 464 465 /* 466 Print "higher-level" package help message 467 */ 468 if (hasHelp) { 469 PetscBool hasHelpIntro; 470 471 if (PetscExternalHelpFunction) { 472 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 473 } 474 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 475 if (hasHelpIntro) { 476 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 477 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 478 ierr = MPI_Finalize();CHKERRMPI(ierr); 479 exit(0); 480 } 481 } 482 483 /* 484 Setup the error handling 485 */ 486 flg1 = PETSC_FALSE; 487 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 488 if (flg1) { 489 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRMPI(ierr); 490 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 491 } 492 flg1 = PETSC_FALSE; 493 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 494 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 495 flg1 = PETSC_FALSE; 496 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 497 if (flg1) { 498 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRMPI(ierr); 499 } 500 flg1 = PETSC_FALSE; 501 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 502 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 503 504 /* 505 Setup debugger information 506 */ 507 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 508 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 509 if (flg1) { 510 MPI_Errhandler err_handler; 511 512 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 513 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRMPI(ierr); 514 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 515 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 516 } 517 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 518 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 519 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 520 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 521 if (flg1 || flg2) { 522 PetscMPIInt size; 523 PetscInt lsize,*ranks; 524 MPI_Errhandler err_handler; 525 /* 526 we have to make sure that all processors have opened 527 connections to all other processors, otherwise once the 528 debugger has stated it is likely to receive a SIGUSR1 529 and kill the program. 530 */ 531 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 532 if (size > 2) { 533 PetscMPIInt dummy = 0; 534 MPI_Status status; 535 for (i=0; i<size; i++) { 536 if (rank != i) { 537 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRMPI(ierr); 538 } 539 } 540 for (i=0; i<size; i++) { 541 if (rank != i) { 542 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRMPI(ierr); 543 } 544 } 545 } 546 /* check if this processor node should be in debugger */ 547 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr); 548 lsize = size; 549 /* Deprecated in 3.14 */ 550 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr); 551 if (flag) { 552 const char * const quietopt="-options_suppress_deprecated_warnings"; 553 char msg[4096]; 554 PetscBool quiet = PETSC_FALSE; 555 556 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr); 557 if (!quiet) { 558 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr); 559 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr); 560 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr); 561 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr); 562 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr); 563 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr); 564 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr); 565 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr); 566 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr); 567 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr); 568 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr); 569 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr); 570 } 571 } else { 572 lsize = size; 573 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr); 574 } 575 if (flag) { 576 for (i=0; i<lsize; i++) { 577 if (ranks[i] == rank) { flag = PETSC_FALSE; break; } 578 } 579 } 580 if (!flag) { 581 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 582 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 583 if (flg1) { 584 ierr = PetscAttachDebugger();CHKERRQ(ierr); 585 } else { 586 ierr = PetscStopForDebugger();CHKERRQ(ierr); 587 } 588 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRMPI(ierr); 589 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRMPI(ierr); 590 } else { 591 ierr = PetscWaitOnError();CHKERRQ(ierr); 592 } 593 ierr = PetscFree(ranks);CHKERRQ(ierr); 594 } 595 596 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 597 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 598 599 /* 600 Setup profiling and logging 601 */ 602 #if defined(PETSC_USE_INFO) 603 { 604 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 605 } 606 #endif 607 ierr = PetscDetermineInitialFPTrap(); 608 flg1 = PETSC_FALSE; 609 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 610 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 611 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 612 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 613 #if defined(PETSC_USE_LOG) 614 mname[0] = 0; 615 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 616 if (flg1) { 617 if (mname[0]) { 618 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 619 } else { 620 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 621 } 622 } 623 624 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 625 626 #if defined(PETSC_HAVE_MPE) 627 flg1 = PETSC_FALSE; 628 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 629 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 630 #endif 631 flg1 = PETSC_FALSE; 632 flg3 = PETSC_FALSE; 633 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 634 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 635 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 636 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 637 638 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 639 if (flg1) { 640 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 641 FILE *file; 642 if (mname[0]) { 643 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 644 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 645 file = fopen(fname,"w"); 646 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 647 } else file = PETSC_STDOUT; 648 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 649 } 650 651 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 652 if (flg4) { 653 if (format == PETSC_VIEWER_ASCII_XML) { 654 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 655 } else { 656 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 657 } 658 } 659 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 660 PetscReal threshold = PetscRealConstant(0.01); 661 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 662 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 663 } 664 #endif 665 666 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 667 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 668 /* 669 If collecting logging information, by default, wait for device to complete its operations 670 before returning to the CPU in order to get accurate timings of each event 671 */ 672 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 673 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);} 674 675 #if defined(PETSC_HAVE_CUDA) 676 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr); 677 #endif 678 679 #if defined(PETSC_HAVE_HIP) 680 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr); 681 #endif 682 683 /* 684 Print basic help message 685 */ 686 if (hasHelp) { 687 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 688 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 689 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 690 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 693 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 694 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 695 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 696 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 697 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 698 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 702 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 703 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 704 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 705 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 706 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 707 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 708 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 709 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 722 #if defined(PETSC_USE_LOG) 723 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 727 #if defined(PETSC_HAVE_MPE) 728 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 729 #endif 730 #endif 731 #if defined(PETSC_USE_INFO) 732 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 733 #endif 734 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 738 } 739 740 #if defined(PETSC_HAVE_POPEN) 741 { 742 char machine[128]; 743 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 744 if (flg1) { 745 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 746 } 747 } 748 #endif 749 750 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 751 if (flg1) { 752 ierr = PetscSleep(si);CHKERRQ(ierr); 753 } 754 755 #if defined(PETSC_HAVE_VIENNACL) 756 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 757 if (!flg3) { 758 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 759 } 760 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 761 PetscViennaCLSynchronize = flg3; 762 ierr = PetscViennaCLInit();CHKERRQ(ierr); 763 #endif 764 765 /* 766 Creates the logging data structures; this is enabled even if logging is not turned on 767 This is the last thing we do before returning to the user code to prevent having the 768 logging numbers contaminated by any startup time associated with MPI and the GPUs 769 */ 770 #if defined(PETSC_USE_LOG) 771 ierr = PetscLogInitialize();CHKERRQ(ierr); 772 #endif 773 774 PetscFunctionReturn(0); 775 } 776