1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <petsccublas.h> 26 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 27 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 28 #endif 29 #endif 30 31 #if defined(PETSC_HAVE_VIENNACL) 32 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 33 #endif 34 35 /* ------------------------Nasty global variables -------------------------------*/ 36 /* 37 Indicates if PETSc started up MPI, or it was 38 already started before PETSc was initialized. 39 */ 40 PetscBool PetscBeganMPI = PETSC_FALSE; 41 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 42 PetscBool PetscInitializeCalled = PETSC_FALSE; 43 PetscBool PetscFinalizeCalled = PETSC_FALSE; 44 PetscBool PetscCUDAInitialized = PETSC_FALSE; 45 46 PetscMPIInt PetscGlobalRank = -1; 47 PetscMPIInt PetscGlobalSize = -1; 48 49 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 50 51 #if defined(PETSC_HAVE_COMPLEX) 52 #if defined(PETSC_COMPLEX_INSTANTIATE) 53 template <> class std::complex<double>; /* instantiate complex template class */ 54 #endif 55 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 56 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 57 MPI_Datatype MPIU_C_COMPLEX; 58 #endif 59 60 /*MC 61 PETSC_i - the imaginary number i 62 63 Synopsis: 64 #include <petscsys.h> 65 PetscComplex PETSC_i; 66 67 Level: beginner 68 69 Note: 70 Complex numbers are automatically available if PETSc located a working complex implementation 71 72 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 73 M*/ 74 PetscComplex PETSC_i; 75 #endif 76 #if defined(PETSC_USE_REAL___FLOAT128) 77 MPI_Datatype MPIU___FLOAT128 = 0; 78 #if defined(PETSC_HAVE_COMPLEX) 79 MPI_Datatype MPIU___COMPLEX128 = 0; 80 #endif 81 #elif defined(PETSC_USE_REAL___FP16) 82 MPI_Datatype MPIU___FP16 = 0; 83 #endif 84 MPI_Datatype MPIU_2SCALAR = 0; 85 #if defined(PETSC_USE_64BIT_INDICES) 86 MPI_Datatype MPIU_2INT = 0; 87 #endif 88 MPI_Datatype MPIU_BOOL; 89 MPI_Datatype MPIU_ENUM; 90 MPI_Datatype MPIU_FORTRANADDR; 91 MPI_Datatype MPIU_SIZE_T; 92 93 /* 94 Function that is called to display all error messages 95 */ 96 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 97 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 98 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 99 /* 100 This is needed to turn on/off GPU synchronization 101 */ 102 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 103 PetscBool PetscCUDASynchronize = PETSC_FALSE; 104 105 /* ------------------------------------------------------------------------------*/ 106 /* 107 Optional file where all PETSc output from various prints is saved 108 */ 109 PETSC_INTERN FILE *petsc_history; 110 FILE *petsc_history = NULL; 111 112 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 113 { 114 PetscErrorCode ierr; 115 PetscMPIInt rank,size; 116 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 117 char version[256]; 118 119 PetscFunctionBegin; 120 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 121 if (!rank) { 122 char arch[10]; 123 int err; 124 125 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 126 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 127 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 128 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 129 if (filename) { 130 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 131 } else { 132 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 133 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 134 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 135 } 136 137 *fd = fopen(fname,"a"); 138 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 139 140 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 141 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 142 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 143 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 144 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 145 146 err = fflush(*fd); 147 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 148 } 149 PetscFunctionReturn(0); 150 } 151 152 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 153 { 154 PetscErrorCode ierr; 155 PetscMPIInt rank; 156 char date[64]; 157 int err; 158 159 PetscFunctionBegin; 160 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 161 if (!rank) { 162 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 165 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 166 err = fflush(*fd); 167 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 168 err = fclose(*fd); 169 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 170 } 171 PetscFunctionReturn(0); 172 } 173 174 /* ------------------------------------------------------------------------------*/ 175 176 /* 177 This is ugly and probably belongs somewhere else, but I want to 178 be able to put a true MPI abort error handler with command line args. 179 180 This is so MPI errors in the debugger will leave all the stack 181 frames. The default MP_Abort() cleans up and exits thus providing no useful information 182 in the debugger hence we call abort() instead of MPI_Abort(). 183 */ 184 185 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 186 { 187 PetscFunctionBegin; 188 (*PetscErrorPrintf)("MPI error %d\n",*flag); 189 abort(); 190 } 191 192 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 193 { 194 PetscErrorCode ierr; 195 196 PetscFunctionBegin; 197 (*PetscErrorPrintf)("MPI error %d\n",*flag); 198 ierr = PetscAttachDebugger(); 199 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 200 } 201 202 #if defined(PETSC_HAVE_CUDA) 203 /*@C 204 PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device 205 206 Logically collective 207 208 Input Parameter: 209 comm - the MPI communicator that will utilize the CUDA devices 210 211 Options Database: 212 + -cuda_initialize <yes,no> - Default no. Do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically 213 when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed 214 . -cuda_view - view information about the CUDA devices 215 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view 216 . -cuda_set_device <gpu> - integer number of the device 217 - -use_gpu_aware_mpi - Assume the MPI is GPU-aware when communicating data on GPU 218 219 Level: beginner 220 221 Notes: 222 Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins 223 224 @*/ 225 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm) 226 { 227 PetscErrorCode ierr; 228 PetscInt deviceOpt = 0; 229 PetscBool cuda_view_flag = PETSC_FALSE,flg; 230 struct cudaDeviceProp prop; 231 int devCount,device,devicecnt; 232 cudaError_t err = cudaSuccess; 233 PetscMPIInt rank,size; 234 235 PetscFunctionBegin; 236 ierr = PetscOptionsBegin(comm,NULL,"CUDA options","Sys");CHKERRQ(ierr); 237 ierr = PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);CHKERRQ(ierr); 238 device = (int)deviceOpt; 239 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 240 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);CHKERRQ(ierr); 241 ierr = PetscOptionsEnd();CHKERRQ(ierr); 242 if (!PetscCUDAInitialized) { 243 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 244 245 if (size>1 && !flg) { 246 /* check to see if we force multiple ranks to hit the same GPU */ 247 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 248 249 /* First get the device count */ 250 err = cudaGetDeviceCount(&devCount); 251 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 252 253 /* next determine the rank and then set the device via a mod */ 254 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 255 device = rank % devCount; 256 } 257 err = cudaSetDevice(device); 258 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 259 260 /* set the device flags so that it can map host memory */ 261 err = cudaSetDeviceFlags(cudaDeviceMapHost); 262 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 263 264 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 265 ierr = PetscCUSOLVERDnInitializeHandle();CHKERRQ(ierr); 266 PetscCUDAInitialized = PETSC_TRUE; 267 } 268 if (cuda_view_flag) { 269 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 270 err = cudaGetDeviceCount(&devCount); 271 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 272 for (devicecnt = 0; devicecnt < devCount; ++devicecnt) { 273 err = cudaGetDeviceProperties(&prop,devicecnt); 274 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 275 ierr = PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);CHKERRQ(ierr); 276 } 277 ierr = PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);CHKERRQ(ierr); 278 ierr = PetscSynchronizedFlush(comm,PETSC_STDOUT);CHKERRQ(ierr); 279 } 280 PetscFunctionReturn(0); 281 } 282 #endif 283 284 /*@C 285 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 286 wishes a clean exit somewhere deep in the program. 287 288 Collective on PETSC_COMM_WORLD 289 290 Options Database Keys are the same as for PetscFinalize() 291 292 Level: advanced 293 294 Note: 295 See PetscInitialize() for more general runtime options. 296 297 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 298 @*/ 299 PetscErrorCode PetscEnd(void) 300 { 301 PetscFunctionBegin; 302 PetscFinalize(); 303 exit(0); 304 return 0; 305 } 306 307 PetscBool PetscOptionsPublish = PETSC_FALSE; 308 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 309 PETSC_INTERN PetscBool petscsetmallocvisited; 310 static char emacsmachinename[256]; 311 312 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 313 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 314 315 /*@C 316 PetscSetHelpVersionFunctions - Sets functions that print help and version information 317 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 318 This routine enables a "higher-level" package that uses PETSc to print its messages first. 319 320 Input Parameter: 321 + help - the help function (may be NULL) 322 - version - the version function (may be NULL) 323 324 Level: developer 325 326 @*/ 327 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 328 { 329 PetscFunctionBegin; 330 PetscExternalHelpFunction = help; 331 PetscExternalVersionFunction = version; 332 PetscFunctionReturn(0); 333 } 334 335 #if defined(PETSC_USE_LOG) 336 PETSC_INTERN PetscBool PetscObjectsLog; 337 #endif 338 339 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 340 { 341 if (PetscUnlikely(*err)) { 342 PetscMPIInt len; 343 char errstring[MPI_MAX_ERROR_STRING]; 344 345 MPI_Error_string(*err,errstring,&len); 346 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 347 } 348 return; 349 } 350 351 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 352 { 353 char string[64]; 354 MPI_Comm comm = PETSC_COMM_WORLD; 355 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp; 356 PetscErrorCode ierr; 357 PetscReal si; 358 PetscInt intensity; 359 int i; 360 PetscMPIInt rank; 361 char version[256],helpoptions[256]; 362 #if defined(PETSC_USE_LOG) 363 char mname[PETSC_MAX_PATH_LEN]; 364 PetscViewerFormat format; 365 PetscBool flg4 = PETSC_FALSE; 366 #endif 367 #if defined(PETSC_HAVE_CUDA) 368 PetscBool initCUDA = PETSC_FALSE,mpi_gpu_awareness; 369 cudaError_t cerr; 370 int devCount = 0; 371 #endif 372 373 PetscFunctionBegin; 374 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 375 376 #if !defined(PETSC_HAVE_THREADSAFETY) 377 if (!(PETSC_RUNNING_ON_VALGRIND)) { 378 /* 379 Setup the memory management; support for tracing malloc() usage 380 */ 381 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 382 383 if (PetscDefined(USE_DEBUG)) { 384 mdebug = PETSC_TRUE; 385 initializenan = PETSC_TRUE; 386 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 387 } else { 388 /* don't warn about unused option */ 389 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 390 flg1 = PETSC_FALSE; 391 } 392 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 393 if (flg1 || flg2) { 394 mdebug = PETSC_TRUE; 395 eachcall = PETSC_TRUE; 396 initializenan = PETSC_TRUE; 397 } else if (flg3 && !flg2) { 398 mdebug = PETSC_FALSE; 399 eachcall = PETSC_FALSE; 400 initializenan = PETSC_FALSE; 401 } 402 403 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 404 if (mlog) { 405 mdebug = PETSC_TRUE; 406 } 407 /* the next line is deprecated */ 408 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 409 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 410 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 411 if (mdebug) { 412 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 413 } 414 if (mlog) { 415 PetscReal logthreshold = 0; 416 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 417 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 418 } 419 #if defined(PETSC_USE_LOG) 420 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 421 #endif 422 } 423 424 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 425 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 426 flg1 = PETSC_FALSE; 427 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 428 /* ignore this option if malloc is already set */ 429 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 430 431 flg1 = PETSC_FALSE; 432 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 433 if (!flg1) { 434 flg1 = PETSC_FALSE; 435 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 436 } 437 if (flg1) { 438 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 439 } 440 #endif 441 442 #if defined(PETSC_USE_LOG) 443 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 444 #endif 445 446 /* 447 Set the display variable for graphics 448 */ 449 ierr = PetscSetDisplay();CHKERRQ(ierr); 450 451 /* 452 Print main application help message 453 */ 454 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 455 if (help && hasHelp) { 456 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 457 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 458 } 459 460 /* 461 Print the PETSc version information 462 */ 463 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 464 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 465 if (flg1 || flg2 || hasHelp) { 466 /* 467 Print "higher-level" package version message 468 */ 469 if (PetscExternalVersionFunction) { 470 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 471 } 472 473 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 477 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 478 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 479 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 480 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 481 } 482 483 /* 484 Print "higher-level" package help message 485 */ 486 if (hasHelp) { 487 if (PetscExternalHelpFunction) { 488 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 489 } 490 } 491 492 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 493 if (flg1) { 494 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 495 if (flg2) { 496 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 497 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 498 ierr = MPI_Finalize();CHKERRQ(ierr); 499 exit(0); 500 } 501 } 502 503 /* 504 Setup the error handling 505 */ 506 flg1 = PETSC_FALSE; 507 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 508 if (flg1) { 509 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 510 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 511 } 512 flg1 = PETSC_FALSE; 513 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 514 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 515 flg1 = PETSC_FALSE; 516 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 517 if (flg1) { 518 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 519 } 520 /* experimental */ 521 flg1 = PETSC_FALSE; 522 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 523 if (flg1) { 524 MPI_Errhandler eh; 525 526 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 527 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 528 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 529 } 530 flg1 = PETSC_FALSE; 531 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 532 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 533 534 /* 535 Setup debugger information 536 */ 537 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 538 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 539 if (flg1) { 540 MPI_Errhandler err_handler; 541 542 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 543 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 544 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 545 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 546 } 547 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 548 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 549 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 550 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 551 if (flg1 || flg2) { 552 PetscMPIInt size; 553 PetscInt lsize,*nodes; 554 MPI_Errhandler err_handler; 555 /* 556 we have to make sure that all processors have opened 557 connections to all other processors, otherwise once the 558 debugger has stated it is likely to receive a SIGUSR1 559 and kill the program. 560 */ 561 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 562 if (size > 2) { 563 PetscMPIInt dummy = 0; 564 MPI_Status status; 565 for (i=0; i<size; i++) { 566 if (rank != i) { 567 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 568 } 569 } 570 for (i=0; i<size; i++) { 571 if (rank != i) { 572 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 573 } 574 } 575 } 576 /* check if this processor node should be in debugger */ 577 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 578 lsize = size; 579 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 580 if (flag) { 581 for (i=0; i<lsize; i++) { 582 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 583 } 584 } 585 if (!flag) { 586 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 587 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 588 if (flg1) { 589 ierr = PetscAttachDebugger();CHKERRQ(ierr); 590 } else { 591 ierr = PetscStopForDebugger();CHKERRQ(ierr); 592 } 593 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 594 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 595 } 596 ierr = PetscFree(nodes);CHKERRQ(ierr); 597 } 598 599 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 600 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 601 602 /* 603 Setup profiling and logging 604 */ 605 #if defined(PETSC_USE_INFO) 606 { 607 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 608 } 609 #endif 610 ierr = PetscDetermineInitialFPTrap(); 611 flg1 = PETSC_FALSE; 612 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 613 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 614 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 615 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 616 #if defined(PETSC_USE_LOG) 617 mname[0] = 0; 618 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 619 if (flg1) { 620 if (mname[0]) { 621 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 622 } else { 623 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 624 } 625 } 626 627 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 628 629 #if defined(PETSC_HAVE_MPE) 630 flg1 = PETSC_FALSE; 631 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 632 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 633 #endif 634 flg1 = PETSC_FALSE; 635 flg3 = PETSC_FALSE; 636 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 637 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 638 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 639 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 640 641 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 642 if (flg1) { 643 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 644 FILE *file; 645 if (mname[0]) { 646 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 647 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 648 file = fopen(fname,"w"); 649 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 650 } else file = PETSC_STDOUT; 651 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 652 } 653 654 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 655 if (flg4) { 656 if (format == PETSC_VIEWER_ASCII_XML) { 657 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 658 } else { 659 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 660 } 661 } 662 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 663 PetscReal threshold = PetscRealConstant(0.01); 664 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 665 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 666 } 667 #endif 668 669 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 670 671 #if defined(PETSC_HAVE_CUDA) 672 /* 673 If collecting logging information, by default, wait for GPU to complete its operations 674 before returning to the CPU in order to get accurate timings of each event 675 */ 676 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);CHKERRQ(ierr); 677 if (!PetscCUDASynchronize) { 678 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);CHKERRQ(ierr); 679 } 680 681 ierr = PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");CHKERRQ(ierr); 682 ierr = PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCUDA,&initCUDA,NULL);CHKERRQ(ierr); 683 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU (on by default with -log_summary or -log_view)",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 684 ierr = PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 685 ierr = PetscOptionsEnd();CHKERRQ(ierr); 686 if (initCUDA) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD);CHKERRQ(ierr);} 687 if (use_gpu_aware_mpi) { 688 cerr = cudaGetDeviceCount(&devCount);{if (cerr != cudaErrorNoDevice) CHKERRCUDA(cerr);} /* Catch other errors */ 689 if (cerr == cudaErrorNoDevice) devCount = 0; /* CUDA does not say what devCount is under this error */ 690 } 691 if (devCount > 0 && use_gpu_aware_mpi) { /* Only do the MPI GPU awareness check when there are GPU(s) */ 692 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT 693 /* Trust OpenMPI's compile time cuda query interface */ 694 mpi_gpu_awareness = PETSC_TRUE; 695 #else 696 /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults. 697 Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT. 698 */ 699 mpi_gpu_awareness = PetscCheckMpiGpuAwareness(); 700 #endif 701 if (!mpi_gpu_awareness) { 702 (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n"); 703 (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n"); 704 (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n"); 705 (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n"); 706 (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n"); 707 (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n"); 708 PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB); 709 } 710 } 711 #endif 712 713 /* 714 Print basic help message 715 */ 716 if (hasHelp) { 717 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 746 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 747 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 749 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 750 #if defined(PETSC_USE_LOG) 751 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 752 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 753 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 754 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 755 #if defined(PETSC_HAVE_MPE) 756 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 757 #endif 758 #endif 759 #if defined(PETSC_USE_INFO) 760 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 761 #endif 762 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 763 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 764 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 765 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 766 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 767 } 768 769 #if defined(PETSC_HAVE_POPEN) 770 { 771 char machine[128]; 772 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 773 if (flg1) { 774 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 775 } 776 } 777 #endif 778 779 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 780 if (flg1) { 781 ierr = PetscSleep(si);CHKERRQ(ierr); 782 } 783 784 #if defined(PETSC_HAVE_VIENNACL) 785 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 786 if (!flg3) { 787 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 788 } 789 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 790 PetscViennaCLSynchronize = flg3; 791 ierr = PetscViennaCLInit();CHKERRQ(ierr); 792 #endif 793 794 /* 795 Creates the logging data structures; this is enabled even if logging is not turned on 796 This is the last thing we do before returning to the user code to prevent having the 797 logging numbers contaminated by any startup time associated with MPI and the GPUs 798 */ 799 #if defined(PETSC_USE_LOG) 800 ierr = PetscLogInitialize();CHKERRQ(ierr); 801 #endif 802 803 PetscFunctionReturn(0); 804 } 805