1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <petsccublas.h> 26 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 27 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 28 #endif 29 #endif 30 31 #if defined(PETSC_HAVE_VIENNACL) 32 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 33 #endif 34 35 /* ------------------------Nasty global variables -------------------------------*/ 36 /* 37 Indicates if PETSc started up MPI, or it was 38 already started before PETSc was initialized. 39 */ 40 PetscBool PetscBeganMPI = PETSC_FALSE; 41 PetscBool PetscInitializeCalled = PETSC_FALSE; 42 PetscBool PetscFinalizeCalled = PETSC_FALSE; 43 PetscBool PetscCUDAInitialized = PETSC_FALSE; 44 45 PetscMPIInt PetscGlobalRank = -1; 46 PetscMPIInt PetscGlobalSize = -1; 47 48 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 49 50 #if defined(PETSC_HAVE_COMPLEX) 51 #if defined(PETSC_COMPLEX_INSTANTIATE) 52 template <> class std::complex<double>; /* instantiate complex template class */ 53 #endif 54 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 55 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 56 MPI_Datatype MPIU_C_COMPLEX; 57 #endif 58 59 /*MC 60 PETSC_i - the imaginary number i 61 62 Synopsis: 63 #include <petscsys.h> 64 PetscComplex PETSC_i; 65 66 Level: beginner 67 68 Note: 69 Complex numbers are automatically available if PETSc located a working complex implementation 70 71 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 72 M*/ 73 PetscComplex PETSC_i; 74 #endif 75 #if defined(PETSC_USE_REAL___FLOAT128) 76 MPI_Datatype MPIU___FLOAT128 = 0; 77 #if defined(PETSC_HAVE_COMPLEX) 78 MPI_Datatype MPIU___COMPLEX128 = 0; 79 #endif 80 #elif defined(PETSC_USE_REAL___FP16) 81 MPI_Datatype MPIU___FP16 = 0; 82 #endif 83 MPI_Datatype MPIU_2SCALAR = 0; 84 #if defined(PETSC_USE_64BIT_INDICES) 85 MPI_Datatype MPIU_2INT = 0; 86 #endif 87 MPI_Datatype MPIU_BOOL; 88 MPI_Datatype MPIU_ENUM; 89 MPI_Datatype MPIU_FORTRANADDR; 90 MPI_Datatype MPIU_SIZE_T; 91 92 /* 93 Function that is called to display all error messages 94 */ 95 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 96 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 97 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 98 /* 99 This is needed to turn on/off GPU synchronization 100 */ 101 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 102 PetscBool PetscCUDASynchronize = PETSC_FALSE; 103 104 /* ------------------------------------------------------------------------------*/ 105 /* 106 Optional file where all PETSc output from various prints is saved 107 */ 108 PETSC_INTERN FILE *petsc_history; 109 FILE *petsc_history = NULL; 110 111 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 112 { 113 PetscErrorCode ierr; 114 PetscMPIInt rank,size; 115 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 116 char version[256]; 117 118 PetscFunctionBegin; 119 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 120 if (!rank) { 121 char arch[10]; 122 int err; 123 124 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 125 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 126 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 127 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 128 if (filename) { 129 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 130 } else { 131 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 132 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 133 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 134 } 135 136 *fd = fopen(fname,"a"); 137 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 138 139 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 140 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 141 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 142 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 143 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 144 145 err = fflush(*fd); 146 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 147 } 148 PetscFunctionReturn(0); 149 } 150 151 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 152 { 153 PetscErrorCode ierr; 154 PetscMPIInt rank; 155 char date[64]; 156 int err; 157 158 PetscFunctionBegin; 159 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 160 if (!rank) { 161 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 162 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 165 err = fflush(*fd); 166 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 167 err = fclose(*fd); 168 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 169 } 170 PetscFunctionReturn(0); 171 } 172 173 /* ------------------------------------------------------------------------------*/ 174 175 /* 176 This is ugly and probably belongs somewhere else, but I want to 177 be able to put a true MPI abort error handler with command line args. 178 179 This is so MPI errors in the debugger will leave all the stack 180 frames. The default MP_Abort() cleans up and exits thus providing no useful information 181 in the debugger hence we call abort() instead of MPI_Abort(). 182 */ 183 184 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 185 { 186 PetscFunctionBegin; 187 (*PetscErrorPrintf)("MPI error %d\n",*flag); 188 abort(); 189 } 190 191 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 192 { 193 PetscErrorCode ierr; 194 195 PetscFunctionBegin; 196 (*PetscErrorPrintf)("MPI error %d\n",*flag); 197 ierr = PetscAttachDebugger(); 198 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 199 } 200 201 #if defined(PETSC_HAVE_CUDA) 202 /*@C 203 PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device 204 205 Logically collective 206 207 Input Parameter: 208 comm - the MPI communicator that will utilize the CUDA devices 209 210 Options Database: 211 + -cuda_initialize <yes,no> - Default no. Do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically 212 when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed 213 . -cuda_view - view information about the CUDA devices 214 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view 215 . -cuda_set_device <gpu> - integer number of the device 216 - -use_gpu_aware_mpi - Assume the MPI is GPU-aware when communicating data on GPU 217 218 Level: beginner 219 220 Notes: 221 Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins 222 223 @*/ 224 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm) 225 { 226 PetscErrorCode ierr; 227 PetscInt deviceOpt = 0; 228 PetscBool cuda_view_flag = PETSC_FALSE,flg; 229 struct cudaDeviceProp prop; 230 int devCount,device,devicecnt; 231 cudaError_t err = cudaSuccess; 232 PetscMPIInt rank,size; 233 234 PetscFunctionBegin; 235 ierr = PetscOptionsBegin(comm,NULL,"CUDA options","Sys");CHKERRQ(ierr); 236 ierr = PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);CHKERRQ(ierr); 237 device = (int)deviceOpt; 238 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 239 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);CHKERRQ(ierr); 240 ierr = PetscOptionsEnd();CHKERRQ(ierr); 241 if (!PetscCUDAInitialized) { 242 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 243 244 if (size>1 && !flg) { 245 /* check to see if we force multiple ranks to hit the same GPU */ 246 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 247 248 /* First get the device count */ 249 err = cudaGetDeviceCount(&devCount); 250 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 251 252 /* next determine the rank and then set the device via a mod */ 253 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 254 device = rank % devCount; 255 } 256 err = cudaSetDevice(device); 257 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 258 259 /* set the device flags so that it can map host memory */ 260 err = cudaSetDeviceFlags(cudaDeviceMapHost); 261 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 262 263 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 264 ierr = PetscCUSOLVERDnInitializeHandle();CHKERRQ(ierr); 265 PetscCUDAInitialized = PETSC_TRUE; 266 } 267 if (cuda_view_flag) { 268 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 269 err = cudaGetDeviceCount(&devCount); 270 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 271 for (devicecnt = 0; devicecnt < devCount; ++devicecnt) { 272 err = cudaGetDeviceProperties(&prop,devicecnt); 273 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 274 ierr = PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);CHKERRQ(ierr); 275 } 276 ierr = PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);CHKERRQ(ierr); 277 ierr = PetscSynchronizedFlush(comm,PETSC_STDOUT);CHKERRQ(ierr); 278 } 279 PetscFunctionReturn(0); 280 } 281 #endif 282 283 /*@C 284 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 285 wishes a clean exit somewhere deep in the program. 286 287 Collective on PETSC_COMM_WORLD 288 289 Options Database Keys are the same as for PetscFinalize() 290 291 Level: advanced 292 293 Note: 294 See PetscInitialize() for more general runtime options. 295 296 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 297 @*/ 298 PetscErrorCode PetscEnd(void) 299 { 300 PetscFunctionBegin; 301 PetscFinalize(); 302 exit(0); 303 return 0; 304 } 305 306 PetscBool PetscOptionsPublish = PETSC_FALSE; 307 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 308 PETSC_INTERN PetscBool petscsetmallocvisited; 309 static char emacsmachinename[256]; 310 311 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 312 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 313 314 /*@C 315 PetscSetHelpVersionFunctions - Sets functions that print help and version information 316 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 317 This routine enables a "higher-level" package that uses PETSc to print its messages first. 318 319 Input Parameter: 320 + help - the help function (may be NULL) 321 - version - the version function (may be NULL) 322 323 Level: developer 324 325 @*/ 326 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 327 { 328 PetscFunctionBegin; 329 PetscExternalHelpFunction = help; 330 PetscExternalVersionFunction = version; 331 PetscFunctionReturn(0); 332 } 333 334 #if defined(PETSC_USE_LOG) 335 PETSC_INTERN PetscBool PetscObjectsLog; 336 #endif 337 338 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 339 { 340 if (PetscUnlikely(*err)) { 341 PetscMPIInt len; 342 char errstring[MPI_MAX_ERROR_STRING]; 343 344 MPI_Error_string(*err,errstring,&len); 345 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 346 } 347 return; 348 } 349 350 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(void) 351 { 352 char string[64]; 353 MPI_Comm comm = PETSC_COMM_WORLD; 354 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag; 355 PetscErrorCode ierr; 356 PetscReal si; 357 PetscInt intensity; 358 int i; 359 PetscMPIInt rank; 360 char version[256],helpoptions[256]; 361 #if defined(PETSC_USE_LOG) 362 char mname[PETSC_MAX_PATH_LEN]; 363 PetscViewerFormat format; 364 PetscBool flg4 = PETSC_FALSE; 365 #endif 366 #if defined(PETSC_HAVE_CUDA) 367 PetscBool initCUDA = PETSC_FALSE,mpi_gpu_awareness; 368 cudaError_t cerr; 369 int devCount = 0; 370 #endif 371 372 PetscFunctionBegin; 373 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 374 375 #if !defined(PETSC_HAVE_THREADSAFETY) 376 if (!(PETSC_RUNNING_ON_VALGRIND)) { 377 /* 378 Setup the memory management; support for tracing malloc() usage 379 */ 380 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 381 382 if (PetscDefined(USE_DEBUG)) { 383 mdebug = PETSC_TRUE; 384 initializenan = PETSC_TRUE; 385 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 386 } else { 387 /* don't warn about unused option */ 388 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 389 flg1 = PETSC_FALSE; 390 } 391 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 392 if (flg1 || flg2) { 393 mdebug = PETSC_TRUE; 394 eachcall = PETSC_TRUE; 395 initializenan = PETSC_TRUE; 396 } else if (flg3 && !flg2) { 397 mdebug = PETSC_FALSE; 398 eachcall = PETSC_FALSE; 399 initializenan = PETSC_FALSE; 400 } 401 402 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 403 if (mlog) { 404 mdebug = PETSC_TRUE; 405 } 406 /* the next line is deprecated */ 407 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 408 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 409 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 410 if (mdebug) { 411 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 412 } 413 if (mlog) { 414 PetscReal logthreshold = 0; 415 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 416 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 417 } 418 #if defined(PETSC_USE_LOG) 419 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 420 #endif 421 } 422 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 424 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 425 flg1 = PETSC_FALSE; 426 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 427 /* ignore this option if malloc is already set */ 428 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 429 430 flg1 = PETSC_FALSE; 431 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 432 if (!flg1) { 433 flg1 = PETSC_FALSE; 434 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 435 } 436 if (flg1) { 437 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 438 } 439 #endif 440 441 #if defined(PETSC_USE_LOG) 442 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 443 #endif 444 445 /* 446 Set the display variable for graphics 447 */ 448 ierr = PetscSetDisplay();CHKERRQ(ierr); 449 450 /* 451 Print the PETSc version information 452 */ 453 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 454 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 455 ierr = PetscOptionsHasHelp(NULL,&flg3);CHKERRQ(ierr); 456 if (flg1 || flg2 || flg3) { 457 458 /* 459 Print "higher-level" package version message 460 */ 461 if (PetscExternalVersionFunction) { 462 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 463 } 464 465 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 466 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 467 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 468 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 469 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 470 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 471 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 472 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 473 } 474 475 /* 476 Print "higher-level" package help message 477 */ 478 if (flg3) { 479 if (PetscExternalHelpFunction) { 480 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 481 } 482 } 483 484 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 485 if (flg1) { 486 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 487 if (flg2) { 488 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 489 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 490 ierr = MPI_Finalize();CHKERRQ(ierr); 491 exit(0); 492 } 493 } 494 495 /* 496 Setup the error handling 497 */ 498 flg1 = PETSC_FALSE; 499 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 500 if (flg1) { 501 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 502 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 503 } 504 flg1 = PETSC_FALSE; 505 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 506 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 507 flg1 = PETSC_FALSE; 508 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 509 if (flg1) { 510 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 511 } 512 /* experimental */ 513 flg1 = PETSC_FALSE; 514 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 515 if (flg1) { 516 MPI_Errhandler eh; 517 518 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 519 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 520 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 521 } 522 flg1 = PETSC_FALSE; 523 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 524 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 525 526 /* 527 Setup debugger information 528 */ 529 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 530 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 531 if (flg1) { 532 MPI_Errhandler err_handler; 533 534 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 535 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 536 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 537 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 538 } 539 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 540 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 541 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 542 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 543 if (flg1 || flg2) { 544 PetscMPIInt size; 545 PetscInt lsize,*nodes; 546 MPI_Errhandler err_handler; 547 /* 548 we have to make sure that all processors have opened 549 connections to all other processors, otherwise once the 550 debugger has stated it is likely to receive a SIGUSR1 551 and kill the program. 552 */ 553 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 554 if (size > 2) { 555 PetscMPIInt dummy = 0; 556 MPI_Status status; 557 for (i=0; i<size; i++) { 558 if (rank != i) { 559 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 560 } 561 } 562 for (i=0; i<size; i++) { 563 if (rank != i) { 564 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 565 } 566 } 567 } 568 /* check if this processor node should be in debugger */ 569 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 570 lsize = size; 571 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 572 if (flag) { 573 for (i=0; i<lsize; i++) { 574 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 575 } 576 } 577 if (!flag) { 578 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 579 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 580 if (flg1) { 581 ierr = PetscAttachDebugger();CHKERRQ(ierr); 582 } else { 583 ierr = PetscStopForDebugger();CHKERRQ(ierr); 584 } 585 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 586 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 587 } 588 ierr = PetscFree(nodes);CHKERRQ(ierr); 589 } 590 591 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 592 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 593 594 /* 595 Setup profiling and logging 596 */ 597 #if defined(PETSC_USE_INFO) 598 { 599 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 600 } 601 #endif 602 ierr = PetscDetermineInitialFPTrap(); 603 flg1 = PETSC_FALSE; 604 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 605 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 606 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 607 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 608 #if defined(PETSC_USE_LOG) 609 mname[0] = 0; 610 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 611 if (flg1) { 612 if (mname[0]) { 613 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 614 } else { 615 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 616 } 617 } 618 619 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 620 621 #if defined(PETSC_HAVE_MPE) 622 flg1 = PETSC_FALSE; 623 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 624 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 625 #endif 626 flg1 = PETSC_FALSE; 627 flg3 = PETSC_FALSE; 628 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 629 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 630 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 631 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 632 633 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 634 if (flg1) { 635 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 636 FILE *file; 637 if (mname[0]) { 638 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 639 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 640 file = fopen(fname,"w"); 641 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 642 } else file = PETSC_STDOUT; 643 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 644 } 645 646 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 647 if (flg4) { 648 if (format == PETSC_VIEWER_ASCII_XML) { 649 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 650 } else { 651 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 652 } 653 } 654 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 655 PetscReal threshold = PetscRealConstant(0.01); 656 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 657 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 658 } 659 #endif 660 661 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 662 663 #if defined(PETSC_HAVE_CUDA) 664 /* 665 If collecting logging information, by default, wait for GPU to complete its operations 666 before returning to the CPU in order to get accurate timings of each event 667 */ 668 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);CHKERRQ(ierr); 669 if (!PetscCUDASynchronize) { 670 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);CHKERRQ(ierr); 671 } 672 673 ierr = PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");CHKERRQ(ierr); 674 ierr = PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCUDA,&initCUDA,NULL);CHKERRQ(ierr); 675 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU (on by default with -log_summary or -log_view)",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 676 ierr = PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 677 ierr = PetscOptionsEnd();CHKERRQ(ierr); 678 if (initCUDA) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD);CHKERRQ(ierr);} 679 if (use_gpu_aware_mpi) { 680 cerr = cudaGetDeviceCount(&devCount);{if (cerr != cudaErrorNoDevice) CHKERRCUDA(cerr);} /* Catch other errors */ 681 if (cerr == cudaErrorNoDevice) devCount = 0; /* CUDA does not say what devCount is under this error */ 682 } 683 if (devCount > 0 && use_gpu_aware_mpi) { /* Only do the MPI GPU awareness check when there are GPU(s) */ 684 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT 685 /* Trust OpenMPI's compile time cuda query interface */ 686 mpi_gpu_awareness = PETSC_TRUE; 687 #else 688 /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults. 689 Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT. 690 */ 691 mpi_gpu_awareness = PetscCheckMpiGpuAwareness(); 692 #endif 693 if (!mpi_gpu_awareness) { 694 (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n"); 695 (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n"); 696 (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n"); 697 (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n"); 698 (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n"); 699 (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n"); 700 PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB); 701 } 702 } 703 #endif 704 705 /* 706 Print basic help message 707 */ 708 ierr = PetscOptionsHasHelp(NULL,&flg1);CHKERRQ(ierr); 709 if (flg1) { 710 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 743 #if defined(PETSC_USE_LOG) 744 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 746 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 747 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 748 #if defined(PETSC_HAVE_MPE) 749 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 750 #endif 751 #endif 752 #if defined(PETSC_USE_INFO) 753 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 754 #endif 755 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 756 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 757 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 758 } 759 760 #if defined(PETSC_HAVE_POPEN) 761 { 762 char machine[128]; 763 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 764 if (flg1) { 765 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 766 } 767 } 768 #endif 769 770 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 771 if (flg1) { 772 ierr = PetscSleep(si);CHKERRQ(ierr); 773 } 774 775 #if defined(PETSC_HAVE_VIENNACL) 776 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 777 if (!flg3) { 778 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 779 } 780 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 781 PetscViennaCLSynchronize = flg3; 782 ierr = PetscViennaCLInit();CHKERRQ(ierr); 783 #endif 784 785 /* 786 Creates the logging data structures; this is enabled even if logging is not turned on 787 This is the last thing we do before returning to the user code to prevent having the 788 logging numbers contaminated by any startup time associated with MPI and the GPUs 789 */ 790 #if defined(PETSC_USE_LOG) 791 ierr = PetscLogInitialize();CHKERRQ(ierr); 792 #endif 793 794 PetscFunctionReturn(0); 795 } 796