1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <petsccublas.h> 26 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 27 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 28 #endif 29 #endif 30 31 #if defined(PETSC_HAVE_VIENNACL) 32 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 33 #endif 34 35 /* ------------------------Nasty global variables -------------------------------*/ 36 /* 37 Indicates if PETSc started up MPI, or it was 38 already started before PETSc was initialized. 39 */ 40 PetscBool PetscBeganMPI = PETSC_FALSE; 41 PetscBool PetscInitializeCalled = PETSC_FALSE; 42 PetscBool PetscFinalizeCalled = PETSC_FALSE; 43 PetscBool PetscCUDAInitialized = PETSC_FALSE; 44 45 PetscMPIInt PetscGlobalRank = -1; 46 PetscMPIInt PetscGlobalSize = -1; 47 48 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 49 50 #if defined(PETSC_HAVE_COMPLEX) 51 #if defined(PETSC_COMPLEX_INSTANTIATE) 52 template <> class std::complex<double>; /* instantiate complex template class */ 53 #endif 54 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 55 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 56 MPI_Datatype MPIU_C_COMPLEX; 57 #endif 58 59 /*MC 60 PETSC_i - the imaginary number i 61 62 Synopsis: 63 #include <petscsys.h> 64 PetscComplex PETSC_i; 65 66 Level: beginner 67 68 Note: 69 Complex numbers are automatically available if PETSc located a working complex implementation 70 71 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 72 M*/ 73 PetscComplex PETSC_i; 74 #endif 75 #if defined(PETSC_USE_REAL___FLOAT128) 76 MPI_Datatype MPIU___FLOAT128 = 0; 77 #if defined(PETSC_HAVE_COMPLEX) 78 MPI_Datatype MPIU___COMPLEX128 = 0; 79 #endif 80 #elif defined(PETSC_USE_REAL___FP16) 81 MPI_Datatype MPIU___FP16 = 0; 82 #endif 83 MPI_Datatype MPIU_2SCALAR = 0; 84 #if defined(PETSC_USE_64BIT_INDICES) 85 MPI_Datatype MPIU_2INT = 0; 86 #endif 87 MPI_Datatype MPIU_BOOL; 88 MPI_Datatype MPIU_ENUM; 89 MPI_Datatype MPIU_FORTRANADDR; 90 MPI_Datatype MPIU_SIZE_T; 91 92 /* 93 Function that is called to display all error messages 94 */ 95 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 96 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 97 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 98 /* 99 This is needed to turn on/off GPU synchronization 100 */ 101 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 102 PetscBool PetscCUDASynchronize = PETSC_FALSE; 103 104 /* ------------------------------------------------------------------------------*/ 105 /* 106 Optional file where all PETSc output from various prints is saved 107 */ 108 PETSC_INTERN FILE *petsc_history; 109 FILE *petsc_history = NULL; 110 111 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 112 { 113 PetscErrorCode ierr; 114 PetscMPIInt rank,size; 115 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 116 char version[256]; 117 118 PetscFunctionBegin; 119 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 120 if (!rank) { 121 char arch[10]; 122 int err; 123 124 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 125 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 126 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 127 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 128 if (filename) { 129 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 130 } else { 131 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 132 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 133 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 134 } 135 136 *fd = fopen(fname,"a"); 137 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 138 139 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 140 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 141 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 142 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 143 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 144 145 err = fflush(*fd); 146 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 147 } 148 PetscFunctionReturn(0); 149 } 150 151 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 152 { 153 PetscErrorCode ierr; 154 PetscMPIInt rank; 155 char date[64]; 156 int err; 157 158 PetscFunctionBegin; 159 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 160 if (!rank) { 161 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 162 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 165 err = fflush(*fd); 166 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 167 err = fclose(*fd); 168 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 169 } 170 PetscFunctionReturn(0); 171 } 172 173 /* ------------------------------------------------------------------------------*/ 174 175 /* 176 This is ugly and probably belongs somewhere else, but I want to 177 be able to put a true MPI abort error handler with command line args. 178 179 This is so MPI errors in the debugger will leave all the stack 180 frames. The default MP_Abort() cleans up and exits thus providing no useful information 181 in the debugger hence we call abort() instead of MPI_Abort(). 182 */ 183 184 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 185 { 186 PetscFunctionBegin; 187 (*PetscErrorPrintf)("MPI error %d\n",*flag); 188 abort(); 189 } 190 191 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 192 { 193 PetscErrorCode ierr; 194 195 PetscFunctionBegin; 196 (*PetscErrorPrintf)("MPI error %d\n",*flag); 197 ierr = PetscAttachDebugger(); 198 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 199 } 200 201 #if defined(PETSC_HAVE_CUDA) 202 /*@C 203 PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device 204 205 Logically collective 206 207 Input Parameter: 208 comm - the MPI communicator that will utilize the CUDA devices 209 210 Options Database: 211 + -cuda_initialize <yes,no> - Default no. Do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically 212 when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed 213 . -cuda_view - view information about the CUDA devices 214 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view 215 . -cuda_set_device <gpu> - integer number of the device 216 - -use_gpu_aware_mpi - Assume the MPI is GPU-aware when communicating data on GPU 217 218 Level: beginner 219 220 Notes: 221 Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins 222 223 @*/ 224 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm) 225 { 226 PetscErrorCode ierr; 227 PetscInt deviceOpt = 0; 228 PetscBool cuda_view_flag = PETSC_FALSE,flg; 229 struct cudaDeviceProp prop; 230 int devCount,device,devicecnt; 231 cudaError_t err = cudaSuccess; 232 PetscMPIInt rank,size; 233 234 PetscFunctionBegin; 235 ierr = PetscOptionsBegin(comm,NULL,"CUDA options","Sys");CHKERRQ(ierr); 236 ierr = PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);CHKERRQ(ierr); 237 device = (int)deviceOpt; 238 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 239 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);CHKERRQ(ierr); 240 ierr = PetscOptionsEnd();CHKERRQ(ierr); 241 if (!PetscCUDAInitialized) { 242 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 243 244 if (size>1 && !flg) { 245 /* check to see if we force multiple ranks to hit the same GPU */ 246 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 247 248 /* First get the device count */ 249 err = cudaGetDeviceCount(&devCount); 250 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 251 252 /* next determine the rank and then set the device via a mod */ 253 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 254 device = rank % devCount; 255 } 256 err = cudaSetDevice(device); 257 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 258 259 /* set the device flags so that it can map host memory */ 260 err = cudaSetDeviceFlags(cudaDeviceMapHost); 261 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 262 263 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 264 ierr = PetscCUSOLVERDnInitializeHandle();CHKERRQ(ierr); 265 PetscCUDAInitialized = PETSC_TRUE; 266 } 267 if (cuda_view_flag) { 268 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 269 err = cudaGetDeviceCount(&devCount); 270 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 271 for (devicecnt = 0; devicecnt < devCount; ++devicecnt) { 272 err = cudaGetDeviceProperties(&prop,devicecnt); 273 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 274 ierr = PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);CHKERRQ(ierr); 275 } 276 ierr = PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);CHKERRQ(ierr); 277 ierr = PetscSynchronizedFlush(comm,PETSC_STDOUT);CHKERRQ(ierr); 278 } 279 PetscFunctionReturn(0); 280 } 281 #endif 282 283 /*@C 284 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 285 wishes a clean exit somewhere deep in the program. 286 287 Collective on PETSC_COMM_WORLD 288 289 Options Database Keys are the same as for PetscFinalize() 290 291 Level: advanced 292 293 Note: 294 See PetscInitialize() for more general runtime options. 295 296 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 297 @*/ 298 PetscErrorCode PetscEnd(void) 299 { 300 PetscFunctionBegin; 301 PetscFinalize(); 302 exit(0); 303 return 0; 304 } 305 306 PetscBool PetscOptionsPublish = PETSC_FALSE; 307 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 308 PETSC_INTERN PetscBool petscsetmallocvisited; 309 static char emacsmachinename[256]; 310 311 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 312 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 313 314 /*@C 315 PetscSetHelpVersionFunctions - Sets functions that print help and version information 316 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 317 This routine enables a "higher-level" package that uses PETSc to print its messages first. 318 319 Input Parameter: 320 + help - the help function (may be NULL) 321 - version - the version function (may be NULL) 322 323 Level: developer 324 325 @*/ 326 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 327 { 328 PetscFunctionBegin; 329 PetscExternalHelpFunction = help; 330 PetscExternalVersionFunction = version; 331 PetscFunctionReturn(0); 332 } 333 334 #if defined(PETSC_USE_LOG) 335 PETSC_INTERN PetscBool PetscObjectsLog; 336 #endif 337 338 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 339 { 340 if (PetscUnlikely(*err)) { 341 PetscMPIInt len; 342 char errstring[MPI_MAX_ERROR_STRING]; 343 344 MPI_Error_string(*err,errstring,&len); 345 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 346 } 347 return; 348 } 349 350 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(void) 351 { 352 char string[64]; 353 MPI_Comm comm = PETSC_COMM_WORLD; 354 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag; 355 PetscErrorCode ierr; 356 PetscReal si; 357 PetscInt intensity; 358 int i; 359 PetscMPIInt rank; 360 char version[256],helpoptions[256]; 361 #if defined(PETSC_USE_LOG) 362 char mname[PETSC_MAX_PATH_LEN]; 363 PetscViewerFormat format; 364 PetscBool flg4 = PETSC_FALSE; 365 #endif 366 #if defined(PETSC_HAVE_CUDA) 367 PetscBool initCUDA = PETSC_FALSE,mpi_gpu_awareness; 368 cudaError_t cerr; 369 int devCount = 0; 370 #endif 371 372 PetscFunctionBegin; 373 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 374 375 #if !defined(PETSC_HAVE_THREADSAFETY) 376 if (!(PETSC_RUNNING_ON_VALGRIND)) { 377 /* 378 Setup the memory management; support for tracing malloc() usage 379 */ 380 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 381 382 if (PetscDefined(USE_DEBUG)) { 383 mdebug = PETSC_TRUE; 384 initializenan = PETSC_TRUE; 385 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 386 } else { 387 /* don't warn about unused option */ 388 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 389 flg1 = PETSC_FALSE; 390 } 391 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 392 if (flg1 || flg2) { 393 mdebug = PETSC_TRUE; 394 eachcall = PETSC_TRUE; 395 initializenan = PETSC_TRUE; 396 } else if (flg3 && !flg2) { 397 mdebug = PETSC_FALSE; 398 eachcall = PETSC_FALSE; 399 initializenan = PETSC_FALSE; 400 } 401 402 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 403 if (mlog) { 404 mdebug = PETSC_TRUE; 405 } 406 /* the next line is deprecated */ 407 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 408 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 409 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 410 if (mdebug) { 411 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 412 } 413 if (mlog) { 414 PetscReal logthreshold = 0; 415 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 416 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 417 } 418 #if defined(PETSC_USE_LOG) 419 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 420 #endif 421 } 422 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 424 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 425 flg1 = PETSC_FALSE; 426 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 427 /* ignore this option if malloc is already set */ 428 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 429 430 flg1 = PETSC_FALSE; 431 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 432 if (!flg1) { 433 flg1 = PETSC_FALSE; 434 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 435 } 436 if (flg1) { 437 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 438 } 439 #endif 440 441 #if defined(PETSC_USE_LOG) 442 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 443 #endif 444 445 /* 446 Set the display variable for graphics 447 */ 448 ierr = PetscSetDisplay();CHKERRQ(ierr); 449 450 /* 451 Print the PETSc version information 452 */ 453 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 454 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 455 ierr = PetscOptionsHasHelp(NULL,&flg3);CHKERRQ(ierr); 456 if (flg1 || flg2 || flg3) { 457 458 /* 459 Print "higher-level" package version message 460 */ 461 if (PetscExternalVersionFunction) { 462 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 463 } 464 465 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 473 } 474 475 /* 476 Print "higher-level" package help message 477 */ 478 if (flg3) { 479 if (PetscExternalHelpFunction) { 480 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 481 } 482 } 483 484 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 485 if (flg1) { 486 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 487 if (flg2) { 488 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 489 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 490 ierr = MPI_Finalize();CHKERRQ(ierr); 491 exit(0); 492 } 493 } 494 495 /* 496 Setup the error handling 497 */ 498 flg1 = PETSC_FALSE; 499 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 500 if (flg1) { 501 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 502 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 503 } 504 flg1 = PETSC_FALSE; 505 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 506 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 507 flg1 = PETSC_FALSE; 508 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 509 if (flg1) { 510 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 511 } 512 /* experimental */ 513 flg1 = PETSC_FALSE; 514 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 515 if (flg1) { 516 MPI_Errhandler eh; 517 518 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 519 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 520 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 521 } 522 flg1 = PETSC_FALSE; 523 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 524 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 525 flg1 = PETSC_FALSE; 526 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 527 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 528 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 529 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 530 531 /* 532 Setup debugger information 533 */ 534 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 535 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 536 if (flg1) { 537 MPI_Errhandler err_handler; 538 539 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 540 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 541 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 542 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 543 } 544 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 545 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 546 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 547 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 548 if (flg1 || flg2) { 549 PetscMPIInt size; 550 PetscInt lsize,*nodes; 551 MPI_Errhandler err_handler; 552 /* 553 we have to make sure that all processors have opened 554 connections to all other processors, otherwise once the 555 debugger has stated it is likely to receive a SIGUSR1 556 and kill the program. 557 */ 558 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 559 if (size > 2) { 560 PetscMPIInt dummy = 0; 561 MPI_Status status; 562 for (i=0; i<size; i++) { 563 if (rank != i) { 564 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 565 } 566 } 567 for (i=0; i<size; i++) { 568 if (rank != i) { 569 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 570 } 571 } 572 } 573 /* check if this processor node should be in debugger */ 574 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 575 lsize = size; 576 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 577 if (flag) { 578 for (i=0; i<lsize; i++) { 579 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 580 } 581 } 582 if (!flag) { 583 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 584 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 585 if (flg1) { 586 ierr = PetscAttachDebugger();CHKERRQ(ierr); 587 } else { 588 ierr = PetscStopForDebugger();CHKERRQ(ierr); 589 } 590 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 591 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 592 } 593 ierr = PetscFree(nodes);CHKERRQ(ierr); 594 } 595 596 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 597 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 598 599 /* 600 Setup profiling and logging 601 */ 602 #if defined(PETSC_USE_INFO) 603 { 604 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 605 } 606 #endif 607 #if defined(PETSC_USE_LOG) 608 mname[0] = 0; 609 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 610 if (flg1) { 611 if (mname[0]) { 612 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 613 } else { 614 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 615 } 616 } 617 618 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 619 620 #if defined(PETSC_HAVE_MPE) 621 flg1 = PETSC_FALSE; 622 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 623 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 624 #endif 625 flg1 = PETSC_FALSE; 626 flg3 = PETSC_FALSE; 627 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 628 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 629 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 630 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 631 632 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 633 if (flg1) { 634 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 635 FILE *file; 636 if (mname[0]) { 637 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 638 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 639 file = fopen(fname,"w"); 640 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 641 } else file = PETSC_STDOUT; 642 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 643 } 644 645 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 646 if (flg4) { 647 if (format == PETSC_VIEWER_ASCII_XML) { 648 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 649 } else { 650 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 651 } 652 } 653 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 654 PetscReal threshold = PetscRealConstant(0.01); 655 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 656 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 657 } 658 #endif 659 660 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 661 662 #if defined(PETSC_HAVE_CUDA) 663 /* 664 If collecting logging information, by default, wait for GPU to complete its operations 665 before returning to the CPU in order to get accurate timings of each event 666 */ 667 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);CHKERRQ(ierr); 668 if (!PetscCUDASynchronize) { 669 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);CHKERRQ(ierr); 670 } 671 672 ierr = PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");CHKERRQ(ierr); 673 ierr = PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCUDA,&initCUDA,NULL);CHKERRQ(ierr); 674 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU (on by default with -log_summary or -log_view)",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 675 ierr = PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 676 ierr = PetscOptionsEnd();CHKERRQ(ierr); 677 if (initCUDA) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD);CHKERRQ(ierr);} 678 if (use_gpu_aware_mpi) { 679 cerr = cudaGetDeviceCount(&devCount);{if (cerr != cudaErrorNoDevice) CHKERRCUDA(cerr);} /* Catch other errors */ 680 if (cerr == cudaErrorNoDevice) devCount = 0; /* CUDA does not say what devCount is under this error */ 681 } 682 if (devCount > 0 && use_gpu_aware_mpi) { /* Only do the MPI GPU awareness check when there are GPU(s) */ 683 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT 684 /* Trust OpenMPI's compile time cuda query interface */ 685 mpi_gpu_awareness = PETSC_TRUE; 686 #else 687 /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults. 688 Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT. 689 */ 690 mpi_gpu_awareness = PetscCheckMpiGpuAwareness(); 691 #endif 692 if (!mpi_gpu_awareness) { 693 (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n"); 694 (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n"); 695 (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n"); 696 (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n"); 697 (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n"); 698 (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n"); 699 PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB); 700 } 701 } 702 #endif 703 704 /* 705 Print basic help message 706 */ 707 ierr = PetscOptionsHasHelp(NULL,&flg1);CHKERRQ(ierr); 708 if (flg1) { 709 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 742 #if defined(PETSC_USE_LOG) 743 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 746 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 747 #if defined(PETSC_HAVE_MPE) 748 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 749 #endif 750 #endif 751 #if defined(PETSC_USE_INFO) 752 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 753 #endif 754 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 755 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 756 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 757 } 758 759 #if defined(PETSC_HAVE_POPEN) 760 { 761 char machine[128]; 762 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 763 if (flg1) { 764 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 765 } 766 } 767 #endif 768 769 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 770 if (flg1) { 771 ierr = PetscSleep(si);CHKERRQ(ierr); 772 } 773 774 #if defined(PETSC_HAVE_VIENNACL) 775 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 776 if (!flg3) { 777 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 778 } 779 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 780 PetscViennaCLSynchronize = flg3; 781 ierr = PetscViennaCLInit();CHKERRQ(ierr); 782 #endif 783 784 /* 785 Creates the logging data structures; this is enabled even if logging is not turned on 786 This is the last thing we do before returning to the user code to prevent having the 787 logging numbers contaminated by any startup time associated with MPI and the GPUs 788 */ 789 #if defined(PETSC_USE_LOG) 790 ierr = PetscLogInitialize();CHKERRQ(ierr); 791 #endif 792 793 PetscFunctionReturn(0); 794 } 795