1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <petsccublas.h> 26 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 27 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 28 #endif 29 #endif 30 31 #if defined(PETSC_HAVE_VIENNACL) 32 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 33 #endif 34 35 /* ------------------------Nasty global variables -------------------------------*/ 36 /* 37 Indicates if PETSc started up MPI, or it was 38 already started before PETSc was initialized. 39 */ 40 PetscBool PetscBeganMPI = PETSC_FALSE; 41 PetscBool PetscInitializeCalled = PETSC_FALSE; 42 PetscBool PetscFinalizeCalled = PETSC_FALSE; 43 PetscBool PetscCUDAInitialized = PETSC_FALSE; 44 45 PetscMPIInt PetscGlobalRank = -1; 46 PetscMPIInt PetscGlobalSize = -1; 47 48 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 49 50 #if defined(PETSC_HAVE_COMPLEX) 51 #if defined(PETSC_COMPLEX_INSTANTIATE) 52 template <> class std::complex<double>; /* instantiate complex template class */ 53 #endif 54 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 55 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 56 MPI_Datatype MPIU_C_COMPLEX; 57 #endif 58 59 /*MC 60 PETSC_i - the imaginary number i 61 62 Synopsis: 63 #include <petscsys.h> 64 PetscComplex PETSC_i; 65 66 Level: beginner 67 68 Note: 69 Complex numbers are automatically available if PETSc located a working complex implementation 70 71 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 72 M*/ 73 PetscComplex PETSC_i; 74 #endif 75 #if defined(PETSC_USE_REAL___FLOAT128) 76 MPI_Datatype MPIU___FLOAT128 = 0; 77 #if defined(PETSC_HAVE_COMPLEX) 78 MPI_Datatype MPIU___COMPLEX128 = 0; 79 #endif 80 #elif defined(PETSC_USE_REAL___FP16) 81 MPI_Datatype MPIU___FP16 = 0; 82 #endif 83 MPI_Datatype MPIU_2SCALAR = 0; 84 #if defined(PETSC_USE_64BIT_INDICES) 85 MPI_Datatype MPIU_2INT = 0; 86 #endif 87 MPI_Datatype MPIU_BOOL; 88 MPI_Datatype MPIU_ENUM; 89 MPI_Datatype MPIU_FORTRANADDR; 90 MPI_Datatype MPIU_SIZE_T; 91 92 /* 93 Function that is called to display all error messages 94 */ 95 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 96 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 97 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 98 /* 99 This is needed to turn on/off GPU synchronization 100 */ 101 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 102 PetscBool PetscCUDASynchronize = PETSC_FALSE; 103 104 /* ------------------------------------------------------------------------------*/ 105 /* 106 Optional file where all PETSc output from various prints is saved 107 */ 108 PETSC_INTERN FILE *petsc_history; 109 FILE *petsc_history = NULL; 110 111 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 112 { 113 PetscErrorCode ierr; 114 PetscMPIInt rank,size; 115 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 116 char version[256]; 117 118 PetscFunctionBegin; 119 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 120 if (!rank) { 121 char arch[10]; 122 int err; 123 124 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 125 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 126 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 127 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 128 if (filename) { 129 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 130 } else { 131 ierr = PetscGetHomeDirectory(pfile,240);CHKERRQ(ierr); 132 ierr = PetscStrcat(pfile,"/.petschistory");CHKERRQ(ierr); 133 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 134 } 135 136 *fd = fopen(fname,"a"); 137 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 138 139 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 140 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 141 ierr = PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr); 142 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 143 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 144 145 err = fflush(*fd); 146 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 147 } 148 PetscFunctionReturn(0); 149 } 150 151 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 152 { 153 PetscErrorCode ierr; 154 PetscMPIInt rank; 155 char date[64]; 156 int err; 157 158 PetscFunctionBegin; 159 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 160 if (!rank) { 161 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 162 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 165 err = fflush(*fd); 166 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 167 err = fclose(*fd); 168 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 169 } 170 PetscFunctionReturn(0); 171 } 172 173 /* ------------------------------------------------------------------------------*/ 174 175 /* 176 This is ugly and probably belongs somewhere else, but I want to 177 be able to put a true MPI abort error handler with command line args. 178 179 This is so MPI errors in the debugger will leave all the stack 180 frames. The default MP_Abort() cleans up and exits thus providing no useful information 181 in the debugger hence we call abort() instead of MPI_Abort(). 182 */ 183 184 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 185 { 186 PetscFunctionBegin; 187 (*PetscErrorPrintf)("MPI error %d\n",*flag); 188 abort(); 189 } 190 191 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 192 { 193 PetscErrorCode ierr; 194 195 PetscFunctionBegin; 196 (*PetscErrorPrintf)("MPI error %d\n",*flag); 197 ierr = PetscAttachDebugger(); 198 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 199 } 200 201 #if defined(PETSC_HAVE_CUDA) 202 /*@C 203 PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device 204 205 Logically collective 206 207 Input Parameter: 208 comm - the MPI communicator that will utilize the CUDA devices 209 210 Options Database: 211 + -cuda_initialize <default yes,no> - do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically 212 when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed 213 . -cuda_view - view information about the CUDA devices 214 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view 215 . -cuda_set_device <gpu> - integer number of the device 216 - -use_gpu_aware_mpi - Assume the MPI is GPU-aware when communicating data on GPU 217 218 Level: beginner 219 220 Notes: 221 Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins 222 223 @*/ 224 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm) 225 { 226 PetscErrorCode ierr; 227 PetscInt deviceOpt = 0; 228 PetscBool cuda_view_flag = PETSC_FALSE,flg; 229 struct cudaDeviceProp prop; 230 int devCount,device,devicecnt; 231 cudaError_t err = cudaSuccess; 232 PetscMPIInt rank,size; 233 234 PetscFunctionBegin; 235 /* 236 If collecting logging information, by default, wait for GPU to complete its operations 237 before returning to the CPU in order to get accurate timings of each event 238 */ 239 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);CHKERRQ(ierr); 240 if (!PetscCUDASynchronize) { 241 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);CHKERRQ(ierr); 242 } 243 244 ierr = PetscOptionsBegin(comm,NULL,"CUDA options","Sys");CHKERRQ(ierr); 245 ierr = PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);CHKERRQ(ierr); 246 device = (int)deviceOpt; 247 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 248 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 249 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);CHKERRQ(ierr); 250 ierr = PetscOptionsEnd();CHKERRQ(ierr); 251 if (!PetscCUDAInitialized) { 252 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 253 254 if (size>1 && !flg) { 255 /* check to see if we force multiple ranks to hit the same GPU */ 256 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 257 258 /* First get the device count */ 259 err = cudaGetDeviceCount(&devCount); 260 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 261 262 /* next determine the rank and then set the device via a mod */ 263 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 264 device = rank % devCount; 265 } 266 err = cudaSetDevice(device); 267 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 268 269 /* set the device flags so that it can map host memory */ 270 err = cudaSetDeviceFlags(cudaDeviceMapHost); 271 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 272 273 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 274 ierr = PetscCUSOLVERDnInitializeHandle();CHKERRQ(ierr); 275 PetscCUDAInitialized = PETSC_TRUE; 276 } 277 if (cuda_view_flag) { 278 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 279 err = cudaGetDeviceCount(&devCount); 280 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 281 for (devicecnt = 0; devicecnt < devCount; ++devicecnt) { 282 err = cudaGetDeviceProperties(&prop,devicecnt); 283 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 284 ierr = PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);CHKERRQ(ierr); 285 } 286 ierr = PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);CHKERRQ(ierr); 287 ierr = PetscSynchronizedFlush(comm,PETSC_STDOUT);CHKERRQ(ierr); 288 } 289 PetscFunctionReturn(0); 290 } 291 #endif 292 293 /*@C 294 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 295 wishes a clean exit somewhere deep in the program. 296 297 Collective on PETSC_COMM_WORLD 298 299 Options Database Keys are the same as for PetscFinalize() 300 301 Level: advanced 302 303 Note: 304 See PetscInitialize() for more general runtime options. 305 306 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 307 @*/ 308 PetscErrorCode PetscEnd(void) 309 { 310 PetscFunctionBegin; 311 PetscFinalize(); 312 exit(0); 313 return 0; 314 } 315 316 PetscBool PetscOptionsPublish = PETSC_FALSE; 317 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 318 PETSC_INTERN PetscBool petscsetmallocvisited; 319 static char emacsmachinename[256]; 320 321 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 322 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 323 324 /*@C 325 PetscSetHelpVersionFunctions - Sets functions that print help and version information 326 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 327 This routine enables a "higher-level" package that uses PETSc to print its messages first. 328 329 Input Parameter: 330 + help - the help function (may be NULL) 331 - version - the version function (may be NULL) 332 333 Level: developer 334 335 @*/ 336 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 337 { 338 PetscFunctionBegin; 339 PetscExternalHelpFunction = help; 340 PetscExternalVersionFunction = version; 341 PetscFunctionReturn(0); 342 } 343 344 #if defined(PETSC_USE_LOG) 345 PETSC_INTERN PetscBool PetscObjectsLog; 346 #endif 347 348 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 349 { 350 if (PetscUnlikely(*err)) { 351 PetscMPIInt len; 352 char errstring[MPI_MAX_ERROR_STRING]; 353 354 MPI_Error_string(*err,errstring,&len); 355 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 356 } 357 return; 358 } 359 360 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(void) 361 { 362 char string[64],mname[PETSC_MAX_PATH_LEN],*f; 363 MPI_Comm comm = PETSC_COMM_WORLD; 364 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag; 365 PetscErrorCode ierr; 366 PetscReal si; 367 PetscInt intensity; 368 int i; 369 PetscMPIInt rank; 370 char version[256],helpoptions[256]; 371 #if defined(PETSC_USE_LOG) 372 PetscViewerFormat format; 373 PetscBool flg4 = PETSC_FALSE; 374 #endif 375 #if defined(PETSC_HAVE_CUDA) 376 PetscBool initCuda = PETSC_TRUE,mpi_gpu_awareness; 377 #endif 378 379 PetscFunctionBegin; 380 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 381 382 #if !defined(PETSC_HAVE_THREADSAFETY) 383 if (!(PETSC_RUNNING_ON_VALGRIND)) { 384 /* 385 Setup the memory management; support for tracing malloc() usage 386 */ 387 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 388 389 #if defined(PETSC_USE_DEBUG) 390 mdebug = PETSC_TRUE; 391 initializenan = PETSC_TRUE; 392 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 393 #else 394 /* don't warn about unused option */ 395 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 396 flg1 = PETSC_FALSE; 397 #endif 398 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 399 if (flg1 || flg2) { 400 mdebug = PETSC_TRUE; 401 eachcall = PETSC_TRUE; 402 initializenan = PETSC_TRUE; 403 } else if (flg3 && !flg2) { 404 mdebug = PETSC_FALSE; 405 eachcall = PETSC_FALSE; 406 initializenan = PETSC_FALSE; 407 } 408 409 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 410 if (mlog) { 411 mdebug = PETSC_TRUE; 412 } 413 /* the next line is deprecated */ 414 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 415 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 416 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 417 if (mdebug) { 418 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 419 } 420 if (mlog) { 421 PetscReal logthreshold = 0; 422 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 423 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 424 } 425 #if defined(PETSC_USE_LOG) 426 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 427 #endif 428 } 429 430 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 431 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 432 flg1 = PETSC_FALSE; 433 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 434 /* ignore this option if malloc is already set */ 435 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 436 437 flg1 = PETSC_FALSE; 438 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 439 if (!flg1) { 440 flg1 = PETSC_FALSE; 441 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 442 } 443 if (flg1) { 444 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 445 } 446 #endif 447 448 #if defined(PETSC_USE_LOG) 449 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 450 #endif 451 452 /* 453 Set the display variable for graphics 454 */ 455 ierr = PetscSetDisplay();CHKERRQ(ierr); 456 457 /* 458 Print the PETSc version information 459 */ 460 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 461 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 462 ierr = PetscOptionsHasHelp(NULL,&flg3);CHKERRQ(ierr); 463 if (flg1 || flg2 || flg3) { 464 465 /* 466 Print "higher-level" package version message 467 */ 468 if (PetscExternalVersionFunction) { 469 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 470 } 471 472 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 473 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");CHKERRQ(ierr); 474 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 475 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 476 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 477 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 478 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 479 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 480 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");CHKERRQ(ierr); 481 } 482 483 /* 484 Print "higher-level" package help message 485 */ 486 if (flg3) { 487 if (PetscExternalHelpFunction) { 488 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 489 } 490 } 491 492 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 493 if (flg1) { 494 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 495 if (flg2) { 496 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 497 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 498 ierr = MPI_Finalize();CHKERRQ(ierr); 499 exit(0); 500 } 501 } 502 503 /* 504 Setup the error handling 505 */ 506 flg1 = PETSC_FALSE; 507 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 508 if (flg1) { 509 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 510 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 511 } 512 flg1 = PETSC_FALSE; 513 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 514 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 515 flg1 = PETSC_FALSE; 516 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 517 if (flg1) { 518 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 519 } 520 /* experimental */ 521 flg1 = PETSC_FALSE; 522 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 523 if (flg1) { 524 MPI_Errhandler eh; 525 526 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 527 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 528 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 529 } 530 flg1 = PETSC_FALSE; 531 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 532 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 533 flg1 = PETSC_FALSE; 534 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 535 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 536 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 537 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 538 539 /* 540 Setup debugger information 541 */ 542 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 543 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);CHKERRQ(ierr); 544 if (flg1) { 545 MPI_Errhandler err_handler; 546 547 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 548 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 549 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 550 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 551 } 552 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);CHKERRQ(ierr); 553 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 554 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);CHKERRQ(ierr); 555 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);CHKERRQ(ierr); 556 if (flg1 || flg2) { 557 PetscMPIInt size; 558 PetscInt lsize,*nodes; 559 MPI_Errhandler err_handler; 560 /* 561 we have to make sure that all processors have opened 562 connections to all other processors, otherwise once the 563 debugger has stated it is likely to receive a SIGUSR1 564 and kill the program. 565 */ 566 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 567 if (size > 2) { 568 PetscMPIInt dummy = 0; 569 MPI_Status status; 570 for (i=0; i<size; i++) { 571 if (rank != i) { 572 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 573 } 574 } 575 for (i=0; i<size; i++) { 576 if (rank != i) { 577 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 578 } 579 } 580 } 581 /* check if this processor node should be in debugger */ 582 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 583 lsize = size; 584 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 585 if (flag) { 586 for (i=0; i<lsize; i++) { 587 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 588 } 589 } 590 if (!flag) { 591 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 592 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 593 if (flg1) { 594 ierr = PetscAttachDebugger();CHKERRQ(ierr); 595 } else { 596 ierr = PetscStopForDebugger();CHKERRQ(ierr); 597 } 598 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 599 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 600 } 601 ierr = PetscFree(nodes);CHKERRQ(ierr); 602 } 603 604 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);CHKERRQ(ierr); 605 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 606 607 /* 608 Setup profiling and logging 609 */ 610 #if defined(PETSC_USE_INFO) 611 { 612 char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0; 613 ierr = PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);CHKERRQ(ierr); 614 if (flg1 && logname[0]) { 615 ierr = PetscInfoAllow(PETSC_TRUE,logname);CHKERRQ(ierr); 616 } else if (flg1) { 617 ierr = PetscInfoAllow(PETSC_TRUE,NULL);CHKERRQ(ierr); 618 } 619 } 620 #endif 621 #if defined(PETSC_USE_LOG) 622 mname[0] = 0; 623 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 624 if (flg1) { 625 if (mname[0]) { 626 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 627 } else { 628 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 629 } 630 } 631 632 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 633 634 #if defined(PETSC_HAVE_MPE) 635 flg1 = PETSC_FALSE; 636 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 637 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 638 #endif 639 flg1 = PETSC_FALSE; 640 flg3 = PETSC_FALSE; 641 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 642 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 643 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 644 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 645 646 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);CHKERRQ(ierr); 647 if (flg1) { 648 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 649 FILE *file; 650 if (mname[0]) { 651 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 652 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 653 file = fopen(fname,"w"); 654 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 655 } else file = PETSC_STDOUT; 656 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 657 } 658 659 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 660 if (flg4) { 661 if (format == PETSC_VIEWER_ASCII_XML) { 662 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 663 } else { 664 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 665 } 666 } 667 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 668 PetscReal threshold = PetscRealConstant(0.01); 669 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 670 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 671 } 672 #endif 673 674 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 675 676 #if defined(PETSC_HAVE_CUDA) 677 ierr = PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");CHKERRQ(ierr); 678 ierr = PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCuda,&initCuda,NULL);CHKERRQ(ierr); 679 ierr = PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 680 ierr = PetscOptionsEnd();CHKERRQ(ierr); 681 if (initCuda) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD);CHKERRQ(ierr);} 682 if (use_gpu_aware_mpi) { 683 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) /* Use OpenMPI's compile time cuda query interface */ 684 #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT 685 mpi_gpu_awareness = PETSC_TRUE; 686 #else 687 mpi_gpu_awareness = PETSC_FALSE; 688 #endif 689 #else 690 mpi_gpu_awareness = PetscCheckMpiGpuAwareness(); /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults */ 691 #endif 692 if (!mpi_gpu_awareness) { 693 (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n"); 694 (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n"); 695 (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n"); 696 (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n"); 697 (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n"); 698 (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n"); 699 PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB); 700 } 701 } 702 #endif 703 704 /* 705 Print basic help message 706 */ 707 ierr = PetscOptionsHasHelp(NULL,&flg1);CHKERRQ(ierr); 708 if (flg1) { 709 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 710 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 711 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 712 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 713 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 714 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 715 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 716 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 717 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 718 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 719 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 720 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 721 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 722 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 723 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 724 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 725 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 726 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 727 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 728 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 729 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 730 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 731 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 732 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 733 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 734 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 735 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 736 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 737 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 738 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 739 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 740 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 741 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 742 #if defined(PETSC_USE_LOG) 743 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 744 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 745 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 746 #if defined(PETSC_HAVE_MPE) 747 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 748 #endif 749 ierr = (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");CHKERRQ(ierr); 750 #endif 751 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 752 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 753 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 754 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 755 } 756 757 #if defined(PETSC_HAVE_POPEN) 758 { 759 char machine[128]; 760 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);CHKERRQ(ierr); 761 if (flg1) { 762 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 763 } 764 } 765 #endif 766 767 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 768 if (flg1) { 769 ierr = PetscSleep(si);CHKERRQ(ierr); 770 } 771 772 ierr = PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 773 if (flg1) { 774 ierr = PetscStrstr(mname,"null",&f);CHKERRQ(ierr); 775 if (f) { 776 ierr = PetscInfoDeactivateClass(0);CHKERRQ(ierr); 777 } 778 } 779 780 #if defined(PETSC_HAVE_VIENNACL) 781 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 782 if (!flg3) { 783 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 784 } 785 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 786 PetscViennaCLSynchronize = flg3; 787 ierr = PetscViennaCLInit();CHKERRQ(ierr); 788 #endif 789 790 /* 791 Creates the logging data structures; this is enabled even if logging is not turned on 792 This is the last thing we do before returning to the user code to prevent having the 793 logging numbers contaminated by any startup time associated with MPI and the GPUs 794 */ 795 #if defined(PETSC_USE_LOG) 796 ierr = PetscLogInitialize();CHKERRQ(ierr); 797 #endif 798 799 PetscFunctionReturn(0); 800 } 801