1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <petsccublas.h> 26 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 27 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 28 #endif 29 #endif 30 31 #if defined(PETSC_HAVE_VIENNACL) 32 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 33 #endif 34 35 /* ------------------------Nasty global variables -------------------------------*/ 36 /* 37 Indicates if PETSc started up MPI, or it was 38 already started before PETSc was initialized. 39 */ 40 PetscBool PetscBeganMPI = PETSC_FALSE; 41 PetscBool PetscInitializeCalled = PETSC_FALSE; 42 PetscBool PetscFinalizeCalled = PETSC_FALSE; 43 PetscBool PetscCUDAInitialized = PETSC_FALSE; 44 45 PetscMPIInt PetscGlobalRank = -1; 46 PetscMPIInt PetscGlobalSize = -1; 47 48 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 49 50 #if defined(PETSC_HAVE_COMPLEX) 51 #if defined(PETSC_COMPLEX_INSTANTIATE) 52 template <> class std::complex<double>; /* instantiate complex template class */ 53 #endif 54 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 55 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 56 MPI_Datatype MPIU_C_COMPLEX; 57 #endif 58 59 /*MC 60 PETSC_i - the imaginary number i 61 62 Synopsis: 63 #include <petscsys.h> 64 PetscComplex PETSC_i; 65 66 Level: beginner 67 68 Note: 69 Complex numbers are automatically available if PETSc located a working complex implementation 70 71 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 72 M*/ 73 PetscComplex PETSC_i; 74 #endif 75 #if defined(PETSC_USE_REAL___FLOAT128) 76 MPI_Datatype MPIU___FLOAT128 = 0; 77 #if defined(PETSC_HAVE_COMPLEX) 78 MPI_Datatype MPIU___COMPLEX128 = 0; 79 #endif 80 #elif defined(PETSC_USE_REAL___FP16) 81 MPI_Datatype MPIU___FP16 = 0; 82 #endif 83 MPI_Datatype MPIU_2SCALAR = 0; 84 #if defined(PETSC_USE_64BIT_INDICES) 85 MPI_Datatype MPIU_2INT = 0; 86 #endif 87 MPI_Datatype MPIU_BOOL; 88 MPI_Datatype MPIU_ENUM; 89 MPI_Datatype MPIU_FORTRANADDR; 90 MPI_Datatype MPIU_SIZE_T; 91 92 /* 93 Function that is called to display all error messages 94 */ 95 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 96 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 97 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 98 /* 99 This is needed to turn on/off GPU synchronization 100 */ 101 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 102 PetscBool PetscCUDASynchronize = PETSC_FALSE; 103 104 /* ------------------------------------------------------------------------------*/ 105 /* 106 Optional file where all PETSc output from various prints is saved 107 */ 108 PETSC_INTERN FILE *petsc_history; 109 FILE *petsc_history = NULL; 110 111 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 112 { 113 PetscErrorCode ierr; 114 PetscMPIInt rank,size; 115 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 116 char version[256]; 117 118 PetscFunctionBegin; 119 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 120 if (!rank) { 121 char arch[10]; 122 int err; 123 124 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 125 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 126 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 127 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 128 if (filename) { 129 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 130 } else { 131 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 132 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 133 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 134 } 135 136 *fd = fopen(fname,"a"); 137 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 138 139 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 140 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 141 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 142 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 143 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 144 145 err = fflush(*fd); 146 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 147 } 148 PetscFunctionReturn(0); 149 } 150 151 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 152 { 153 PetscErrorCode ierr; 154 PetscMPIInt rank; 155 char date[64]; 156 int err; 157 158 PetscFunctionBegin; 159 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 160 if (!rank) { 161 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 162 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 165 err = fflush(*fd); 166 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 167 err = fclose(*fd); 168 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 169 } 170 PetscFunctionReturn(0); 171 } 172 173 /* ------------------------------------------------------------------------------*/ 174 175 /* 176 This is ugly and probably belongs somewhere else, but I want to 177 be able to put a true MPI abort error handler with command line args. 178 179 This is so MPI errors in the debugger will leave all the stack 180 frames. The default MP_Abort() cleans up and exits thus providing no useful information 181 in the debugger hence we call abort() instead of MPI_Abort(). 182 */ 183 184 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 185 { 186 PetscFunctionBegin; 187 (*PetscErrorPrintf)("MPI error %d\n",*flag); 188 abort(); 189 } 190 191 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 192 { 193 PetscErrorCode ierr; 194 195 PetscFunctionBegin; 196 (*PetscErrorPrintf)("MPI error %d\n",*flag); 197 ierr = PetscAttachDebugger(); 198 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 199 } 200 201 #if defined(PETSC_HAVE_CUDA) 202 /*@C 203 PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device 204 205 Logically collective 206 207 Input Parameter: 208 comm - the MPI communicator that will utilize the CUDA devices 209 210 Options Database: 211 + -cuda_initialize <yes,no> - Default no. Do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically 212 when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed 213 . -cuda_view - view information about the CUDA devices 214 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view 215 . -cuda_set_device <gpu> - integer number of the device 216 - -use_gpu_aware_mpi - Assume the MPI is GPU-aware when communicating data on GPU 217 218 Level: beginner 219 220 Notes: 221 Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins 222 223 @*/ 224 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm) 225 { 226 PetscErrorCode ierr; 227 PetscInt deviceOpt = 0; 228 PetscBool cuda_view_flag = PETSC_FALSE,flg; 229 struct cudaDeviceProp prop; 230 int devCount,device,devicecnt; 231 cudaError_t err = cudaSuccess; 232 PetscMPIInt rank,size; 233 234 PetscFunctionBegin; 235 ierr = PetscOptionsBegin(comm,NULL,"CUDA options","Sys");CHKERRQ(ierr); 236 ierr = PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);CHKERRQ(ierr); 237 device = (int)deviceOpt; 238 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 239 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);CHKERRQ(ierr); 240 ierr = PetscOptionsEnd();CHKERRQ(ierr); 241 if (!PetscCUDAInitialized) { 242 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 243 244 if (size>1 && !flg) { 245 /* check to see if we force multiple ranks to hit the same GPU */ 246 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 247 248 /* First get the device count */ 249 err = cudaGetDeviceCount(&devCount); 250 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 251 252 /* next determine the rank and then set the device via a mod */ 253 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 254 device = rank % devCount; 255 } 256 err = cudaSetDevice(device); 257 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 258 259 /* set the device flags so that it can map host memory */ 260 err = cudaSetDeviceFlags(cudaDeviceMapHost); 261 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 262 263 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 264 ierr = PetscCUSOLVERDnInitializeHandle();CHKERRQ(ierr); 265 PetscCUDAInitialized = PETSC_TRUE; 266 } 267 if (cuda_view_flag) { 268 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 269 err = cudaGetDeviceCount(&devCount); 270 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 271 for (devicecnt = 0; devicecnt < devCount; ++devicecnt) { 272 err = cudaGetDeviceProperties(&prop,devicecnt); 273 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 274 ierr = PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);CHKERRQ(ierr); 275 } 276 ierr = PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);CHKERRQ(ierr); 277 ierr = PetscSynchronizedFlush(comm,PETSC_STDOUT);CHKERRQ(ierr); 278 } 279 PetscFunctionReturn(0); 280 } 281 #endif 282 283 /*@C 284 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 285 wishes a clean exit somewhere deep in the program. 286 287 Collective on PETSC_COMM_WORLD 288 289 Options Database Keys are the same as for PetscFinalize() 290 291 Level: advanced 292 293 Note: 294 See PetscInitialize() for more general runtime options. 295 296 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 297 @*/ 298 PetscErrorCode PetscEnd(void) 299 { 300 PetscFunctionBegin; 301 PetscFinalize(); 302 exit(0); 303 return 0; 304 } 305 306 PetscBool PetscOptionsPublish = PETSC_FALSE; 307 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 308 PETSC_INTERN PetscBool petscsetmallocvisited; 309 static char emacsmachinename[256]; 310 311 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 312 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 313 314 /*@C 315 PetscSetHelpVersionFunctions - Sets functions that print help and version information 316 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 317 This routine enables a "higher-level" package that uses PETSc to print its messages first. 318 319 Input Parameter: 320 + help - the help function (may be NULL) 321 - version - the version function (may be NULL) 322 323 Level: developer 324 325 @*/ 326 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 327 { 328 PetscFunctionBegin; 329 PetscExternalHelpFunction = help; 330 PetscExternalVersionFunction = version; 331 PetscFunctionReturn(0); 332 } 333 334 #if defined(PETSC_USE_LOG) 335 PETSC_INTERN PetscBool PetscObjectsLog; 336 #endif 337 338 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 339 { 340 if (PetscUnlikely(*err)) { 341 PetscMPIInt len; 342 char errstring[MPI_MAX_ERROR_STRING]; 343 344 MPI_Error_string(*err,errstring,&len); 345 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 346 } 347 return; 348 } 349 350 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 351 { 352 char string[64]; 353 MPI_Comm comm = PETSC_COMM_WORLD; 354 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp; 355 PetscErrorCode ierr; 356 PetscReal si; 357 PetscInt intensity; 358 int i; 359 PetscMPIInt rank; 360 char version[256],helpoptions[256]; 361 #if defined(PETSC_USE_LOG) 362 char mname[PETSC_MAX_PATH_LEN]; 363 PetscViewerFormat format; 364 PetscBool flg4 = PETSC_FALSE; 365 #endif 366 #if defined(PETSC_HAVE_CUDA) 367 PetscBool initCUDA = PETSC_FALSE,mpi_gpu_awareness; 368 cudaError_t cerr; 369 int devCount = 0; 370 #endif 371 372 PetscFunctionBegin; 373 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 374 375 #if !defined(PETSC_HAVE_THREADSAFETY) 376 if (!(PETSC_RUNNING_ON_VALGRIND)) { 377 /* 378 Setup the memory management; support for tracing malloc() usage 379 */ 380 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 381 382 if (PetscDefined(USE_DEBUG)) { 383 mdebug = PETSC_TRUE; 384 initializenan = PETSC_TRUE; 385 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 386 } else { 387 /* don't warn about unused option */ 388 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 389 flg1 = PETSC_FALSE; 390 } 391 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 392 if (flg1 || flg2) { 393 mdebug = PETSC_TRUE; 394 eachcall = PETSC_TRUE; 395 initializenan = PETSC_TRUE; 396 } else if (flg3 && !flg2) { 397 mdebug = PETSC_FALSE; 398 eachcall = PETSC_FALSE; 399 initializenan = PETSC_FALSE; 400 } 401 402 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 403 if (mlog) { 404 mdebug = PETSC_TRUE; 405 } 406 /* the next line is deprecated */ 407 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 408 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 409 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 410 if (mdebug) { 411 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 412 } 413 if (mlog) { 414 PetscReal logthreshold = 0; 415 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 416 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 417 } 418 #if defined(PETSC_USE_LOG) 419 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 420 #endif 421 } 422 423 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 424 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 425 flg1 = PETSC_FALSE; 426 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 427 /* ignore this option if malloc is already set */ 428 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 429 430 flg1 = PETSC_FALSE; 431 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 432 if (!flg1) { 433 flg1 = PETSC_FALSE; 434 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 435 } 436 if (flg1) { 437 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 438 } 439 #endif 440 441 #if defined(PETSC_USE_LOG) 442 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 443 #endif 444 445 /* 446 Set the display variable for graphics 447 */ 448 ierr = PetscSetDisplay();CHKERRQ(ierr); 449 450 /* 451 Print main application help message 452 */ 453 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 454 if (help && hasHelp) { 455 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 456 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 457 } 458 459 /* 460 Print the PETSc version information 461 */ 462 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 463 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 464 if (flg1 || flg2 || hasHelp) { 465 /* 466 Print "higher-level" package version message 467 */ 468 if (PetscExternalVersionFunction) { 469 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 470 } 471 472 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 477 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 478 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 479 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 480 } 481 482 /* 483 Print "higher-level" package help message 484 */ 485 if (hasHelp) { 486 if (PetscExternalHelpFunction) { 487 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 488 } 489 } 490 491 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 492 if (flg1) { 493 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 494 if (flg2) { 495 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 496 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 497 ierr = MPI_Finalize();CHKERRQ(ierr); 498 exit(0); 499 } 500 } 501 502 /* 503 Setup the error handling 504 */ 505 flg1 = PETSC_FALSE; 506 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 507 if (flg1) { 508 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 509 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 510 } 511 flg1 = PETSC_FALSE; 512 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 513 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 514 flg1 = PETSC_FALSE; 515 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 516 if (flg1) { 517 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 518 } 519 /* experimental */ 520 flg1 = PETSC_FALSE; 521 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 522 if (flg1) { 523 MPI_Errhandler eh; 524 525 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 526 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 527 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 528 } 529 flg1 = PETSC_FALSE; 530 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 531 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 532 533 /* 534 Setup debugger information 535 */ 536 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 537 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 538 if (flg1) { 539 MPI_Errhandler err_handler; 540 541 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 542 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 543 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 544 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 545 } 546 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 547 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 548 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 549 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 550 if (flg1 || flg2) { 551 PetscMPIInt size; 552 PetscInt lsize,*nodes; 553 MPI_Errhandler err_handler; 554 /* 555 we have to make sure that all processors have opened 556 connections to all other processors, otherwise once the 557 debugger has stated it is likely to receive a SIGUSR1 558 and kill the program. 559 */ 560 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 561 if (size > 2) { 562 PetscMPIInt dummy = 0; 563 MPI_Status status; 564 for (i=0; i<size; i++) { 565 if (rank != i) { 566 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 567 } 568 } 569 for (i=0; i<size; i++) { 570 if (rank != i) { 571 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 572 } 573 } 574 } 575 /* check if this processor node should be in debugger */ 576 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 577 lsize = size; 578 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 579 if (flag) { 580 for (i=0; i<lsize; i++) { 581 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 582 } 583 } 584 if (!flag) { 585 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 586 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 587 if (flg1) { 588 ierr = PetscAttachDebugger();CHKERRQ(ierr); 589 } else { 590 ierr = PetscStopForDebugger();CHKERRQ(ierr); 591 } 592 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 593 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 594 } 595 ierr = PetscFree(nodes);CHKERRQ(ierr); 596 } 597 598 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 599 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 600 601 /* 602 Setup profiling and logging 603 */ 604 #if defined(PETSC_USE_INFO) 605 { 606 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 607 } 608 #endif 609 ierr = PetscDetermineInitialFPTrap(); 610 flg1 = PETSC_FALSE; 611 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 612 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 613 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 614 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 615 #if defined(PETSC_USE_LOG) 616 mname[0] = 0; 617 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 618 if (flg1) { 619 if (mname[0]) { 620 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 621 } else { 622 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 623 } 624 } 625 626 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 627 628 #if defined(PETSC_HAVE_MPE) 629 flg1 = PETSC_FALSE; 630 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 631 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 632 #endif 633 flg1 = PETSC_FALSE; 634 flg3 = PETSC_FALSE; 635 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 636 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 637 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 638 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 639 640 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 641 if (flg1) { 642 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 643 FILE *file; 644 if (mname[0]) { 645 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 646 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 647 file = fopen(fname,"w"); 648 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 649 } else file = PETSC_STDOUT; 650 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 651 } 652 653 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 654 if (flg4) { 655 if (format == PETSC_VIEWER_ASCII_XML) { 656 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 657 } else { 658 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 659 } 660 } 661 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 662 PetscReal threshold = PetscRealConstant(0.01); 663 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 664 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 665 } 666 #endif 667 668 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 669 670 #if defined(PETSC_HAVE_CUDA) 671 /* 672 If collecting logging information, by default, wait for GPU to complete its operations 673 before returning to the CPU in order to get accurate timings of each event 674 */ 675 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);CHKERRQ(ierr); 676 if (!PetscCUDASynchronize) { 677 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);CHKERRQ(ierr); 678 } 679 680 ierr = PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");CHKERRQ(ierr); 681 ierr = PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCUDA,&initCUDA,NULL);CHKERRQ(ierr); 682 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU (on by default with -log_summary or -log_view)",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 683 ierr = PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 684 ierr = PetscOptionsEnd();CHKERRQ(ierr); 685 if (initCUDA) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD);CHKERRQ(ierr);} 686 if (use_gpu_aware_mpi) { 687 cerr = cudaGetDeviceCount(&devCount);{if (cerr != cudaErrorNoDevice) CHKERRCUDA(cerr);} /* Catch other errors */ 688 if (cerr == cudaErrorNoDevice) devCount = 0; /* CUDA does not say what devCount is under this error */ 689 } 690 if (devCount > 0 && use_gpu_aware_mpi) { /* Only do the MPI GPU awareness check when there are GPU(s) */ 691 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT 692 /* Trust OpenMPI's compile time cuda query interface */ 693 mpi_gpu_awareness = PETSC_TRUE; 694 #else 695 /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults. 696 Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT. 697 */ 698 mpi_gpu_awareness = PetscCheckMpiGpuAwareness(); 699 #endif 700 if (!mpi_gpu_awareness) { 701 (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n"); 702 (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n"); 703 (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n"); 704 (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n"); 705 (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n"); 706 (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n"); 707 PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB); 708 } 709 } 710 #endif 711 712 /* 713 Print basic help message 714 */ 715 if (hasHelp) { 716 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 742 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 743 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 746 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 747 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 748 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 749 #if defined(PETSC_USE_LOG) 750 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 751 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 752 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 753 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 754 #if defined(PETSC_HAVE_MPE) 755 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 756 #endif 757 #endif 758 #if defined(PETSC_USE_INFO) 759 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 760 #endif 761 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 762 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 763 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 764 } 765 766 #if defined(PETSC_HAVE_POPEN) 767 { 768 char machine[128]; 769 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 770 if (flg1) { 771 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 772 } 773 } 774 #endif 775 776 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 777 if (flg1) { 778 ierr = PetscSleep(si);CHKERRQ(ierr); 779 } 780 781 #if defined(PETSC_HAVE_VIENNACL) 782 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 783 if (!flg3) { 784 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 785 } 786 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 787 PetscViennaCLSynchronize = flg3; 788 ierr = PetscViennaCLInit();CHKERRQ(ierr); 789 #endif 790 791 /* 792 Creates the logging data structures; this is enabled even if logging is not turned on 793 This is the last thing we do before returning to the user code to prevent having the 794 logging numbers contaminated by any startup time associated with MPI and the GPUs 795 */ 796 #if defined(PETSC_USE_LOG) 797 ierr = PetscLogInitialize();CHKERRQ(ierr); 798 #endif 799 800 PetscFunctionReturn(0); 801 } 802