1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot be known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 #include <petsccublas.h> 26 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) 27 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */ 28 #endif 29 #endif 30 31 #if defined(PETSC_HAVE_VIENNACL) 32 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 33 #endif 34 35 /* ------------------------Nasty global variables -------------------------------*/ 36 /* 37 Indicates if PETSc started up MPI, or it was 38 already started before PETSc was initialized. 39 */ 40 PetscBool PetscBeganMPI = PETSC_FALSE; 41 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE; 42 PetscBool PetscInitializeCalled = PETSC_FALSE; 43 PetscBool PetscFinalizeCalled = PETSC_FALSE; 44 PetscBool PetscCUDAInitialized = PETSC_FALSE; 45 46 PetscMPIInt PetscGlobalRank = -1; 47 PetscMPIInt PetscGlobalSize = -1; 48 49 PetscBool use_gpu_aware_mpi = PETSC_TRUE; 50 51 #if defined(PETSC_HAVE_COMPLEX) 52 #if defined(PETSC_COMPLEX_INSTANTIATE) 53 template <> class std::complex<double>; /* instantiate complex template class */ 54 #endif 55 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 56 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 57 MPI_Datatype MPIU_C_COMPLEX; 58 #endif 59 60 /*MC 61 PETSC_i - the imaginary number i 62 63 Synopsis: 64 #include <petscsys.h> 65 PetscComplex PETSC_i; 66 67 Level: beginner 68 69 Note: 70 Complex numbers are automatically available if PETSc located a working complex implementation 71 72 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 73 M*/ 74 PetscComplex PETSC_i; 75 #endif 76 #if defined(PETSC_USE_REAL___FLOAT128) 77 MPI_Datatype MPIU___FLOAT128 = 0; 78 #if defined(PETSC_HAVE_COMPLEX) 79 MPI_Datatype MPIU___COMPLEX128 = 0; 80 #endif 81 #elif defined(PETSC_USE_REAL___FP16) 82 MPI_Datatype MPIU___FP16 = 0; 83 #endif 84 MPI_Datatype MPIU_2SCALAR = 0; 85 #if defined(PETSC_USE_64BIT_INDICES) 86 MPI_Datatype MPIU_2INT = 0; 87 #endif 88 MPI_Datatype MPIU_BOOL; 89 MPI_Datatype MPIU_ENUM; 90 MPI_Datatype MPIU_FORTRANADDR; 91 MPI_Datatype MPIU_SIZE_T; 92 93 /* 94 Function that is called to display all error messages 95 */ 96 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 97 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 98 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 99 /* 100 This is needed to turn on/off GPU synchronization 101 */ 102 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 103 PetscBool PetscCUDASynchronize = PETSC_FALSE; 104 105 /* ------------------------------------------------------------------------------*/ 106 /* 107 Optional file where all PETSc output from various prints is saved 108 */ 109 PETSC_INTERN FILE *petsc_history; 110 FILE *petsc_history = NULL; 111 112 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 113 { 114 PetscErrorCode ierr; 115 PetscMPIInt rank,size; 116 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 117 char version[256]; 118 119 PetscFunctionBegin; 120 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 121 if (!rank) { 122 char arch[10]; 123 int err; 124 125 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 126 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 127 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 128 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 129 if (filename) { 130 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 131 } else { 132 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr); 133 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr); 134 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 135 } 136 137 *fd = fopen(fname,"a"); 138 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 139 140 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 141 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 142 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr); 143 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 144 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 145 146 err = fflush(*fd); 147 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 148 } 149 PetscFunctionReturn(0); 150 } 151 152 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 153 { 154 PetscErrorCode ierr; 155 PetscMPIInt rank; 156 char date[64]; 157 int err; 158 159 PetscFunctionBegin; 160 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 161 if (!rank) { 162 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 165 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr); 166 err = fflush(*fd); 167 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 168 err = fclose(*fd); 169 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 170 } 171 PetscFunctionReturn(0); 172 } 173 174 /* ------------------------------------------------------------------------------*/ 175 176 /* 177 This is ugly and probably belongs somewhere else, but I want to 178 be able to put a true MPI abort error handler with command line args. 179 180 This is so MPI errors in the debugger will leave all the stack 181 frames. The default MP_Abort() cleans up and exits thus providing no useful information 182 in the debugger hence we call abort() instead of MPI_Abort(). 183 */ 184 185 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 186 { 187 PetscFunctionBegin; 188 (*PetscErrorPrintf)("MPI error %d\n",*flag); 189 abort(); 190 } 191 192 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 193 { 194 PetscErrorCode ierr; 195 196 PetscFunctionBegin; 197 (*PetscErrorPrintf)("MPI error %d\n",*flag); 198 ierr = PetscAttachDebugger(); 199 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */ 200 } 201 202 #if defined(PETSC_HAVE_CUDA) 203 /* CUDA validation after it is lazily initialized */ 204 static PetscErrorCode PetscCUDAValidate(void) 205 { 206 PetscBool mpi_gpu_awareness; 207 208 PetscFunctionBegin; 209 if (use_gpu_aware_mpi) { 210 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT 211 /* Trust OpenMPI's compile time cuda query interface */ 212 mpi_gpu_awareness = PETSC_TRUE; 213 #else 214 /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults. 215 Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT. 216 */ 217 mpi_gpu_awareness = PetscCheckMpiGpuAwareness(); 218 #endif 219 if (!mpi_gpu_awareness) { 220 (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n"); 221 (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n"); 222 (*PetscErrorPrintf)("If you do care, for IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n"); 223 (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n"); 224 (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n"); 225 (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n"); 226 PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB); 227 } 228 } 229 PetscFunctionReturn(0); 230 } 231 232 /* Initialize the CUDA device lazily just before creating the first CUDA object. */ 233 static PetscBool PetscNotUseGpu = PETSC_FALSE; /* Assert the code will not use GPUs */ 234 PetscErrorCode PetscCUDAInitializeLazily(void) 235 { 236 PetscErrorCode ierr; 237 cudaError_t cerr; 238 int devId,devCount; 239 PetscMPIInt rank; 240 static PetscBool cudaValdidateChecked = PETSC_FALSE; 241 242 PetscFunctionBegin; 243 if (PetscNotUseGpu) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"You asserted the code won't use GPUs with -cuda_set_device none, but it is trying to create a cuda object. Remove this option or see manpage of PetscCUDAInitialize()."); 244 if (!PetscCUDAInitialized) { 245 cerr = cudaGetDeviceCount(&devCount);CHKERRCUDA(cerr); 246 if (devCount > 1) { 247 cerr = cudaSetDeviceFlags(cudaDeviceMapHost); 248 cudaGetLastError(); /* Reset the last error */ 249 if (cerr == cudaSuccess) { /* It implies cuda runtime has not been initialized? */ 250 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 251 devId = rank % devCount; 252 cerr = cudaSetDevice(devId);CHKERRCUDA(cerr); 253 } else if (cerr == cudaErrorSetOnActiveProcess) { 254 /* It implies user has initialized cuda runtime outside of petsc. We do nothing to respect the device choice. */ 255 } 256 } 257 PetscCUDAInitialized = PETSC_TRUE; 258 } 259 if (!cudaValdidateChecked) { 260 ierr = PetscCUDAValidate();CHKERRQ(ierr); 261 cudaValdidateChecked = PETSC_TRUE; 262 } 263 PetscFunctionReturn(0); 264 } 265 266 /*@C 267 PetscCUDAInitialize - Initializes the CUDA device (eagerly in PetscInitialize()) and cuBLAS/cuSPARSE on the device 268 269 Logically collective 270 271 Input Parameter: 272 comm - the MPI communicator that will utilize the CUDA devices 273 device - the device assigned to current MPI process. Special values like PETSC_DECIDE/DEFAULT have special meanings (see details below) 274 275 Options Database: 276 + -cuda_set_device <device> - the device assigned to current MPI rank. <device> is case-insensitive and can be: 277 NONE (or none, or -3) : the code will not use GPUs, otherwise it will error out; 278 PETSC_DEFAULT(or DEFAULT, or -2) : do not explicitly set device, i.e., use whatever device already set by user (probably before PetscInitialize()). Init cuda runtime etc; 279 PETSC_DECIDE (or DECIDE, or -1) : assign MPI ranks in comm to available devices in round-robin, and init cuda runtime etc on the selected device; 280 >= 0 integer : assign the device with this id to current MPI process. Error out if <device> is invalid. Init cuda runtime etc on this device; 281 With PETSC_{DECIDE, DEFAULT}, if there are actually no GPUs, the code can still run, but it will error out when trying to create cuda objects. 282 . -cuda_view - view information about the CUDA devices. 283 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view. 284 . -log_view - logging, however if alone or combined with `-cuda_set_device DEFAULT | DECIDE | >=0 int`, will int cuda; if combined with `-cuda_set_device none`, won't init cuda. 285 - -use_gpu_aware_mpi - assume the MPI is GPU-aware when communicating data on GPUs. 286 287 Level: beginner 288 289 Notes: 290 Initializing cuBLAS takes about 1/2 second therefore it is done by default in PetscCUDAInitialize() before logging begins. 291 292 @*/ 293 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm,PetscInt device) 294 { 295 PetscErrorCode ierr; 296 cudaError_t cerr; 297 int devId,devCount=0; 298 const PetscInt PETSC_NONE=-3; /* Unlike PETSC_DECIDE, we don't have a macro PETSC_NONE in petsc headers */ 299 PetscMPIInt rank; 300 301 PetscFunctionBegin; 302 if (!PetscCUDAInitialized) { 303 cerr = cudaGetDeviceCount(&devCount); 304 cudaGetLastError(); /* Reset the last error */ 305 if (cerr != cudaSuccess) devCount = 0; 306 if (device >= 0) { /* User wants to use this specific device */ 307 cerr = cudaSetDeviceFlags(cudaDeviceMapHost); /* Allow it to fail since user might have already initialized the device. */ 308 cudaGetLastError(); /* Reset the last error */ 309 cerr = cudaSetDevice((int)device);CHKERRCUDA(cerr); 310 } else if (device == PETSC_DECIDE) { /* Assign MPI ranks to available devices in round-robin */ 311 if (devCount > 0) { /* Allow no-GPU as long as user does not use GPUs */ 312 /* Set the device flags so that it can map host memory */ 313 cerr = cudaSetDeviceFlags(cudaDeviceMapHost);CHKERRCUDA(cerr); 314 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 315 devId = rank % devCount; 316 cerr = cudaSetDevice(devId);CHKERRCUDA(cerr); 317 } 318 } else if (device == PETSC_DEFAULT) { 319 /* Do nothing, i.e., use whatever device set by user before PetscInitialize() */ 320 } else if (device == PETSC_NONE) { 321 PetscNotUseGpu = PETSC_TRUE; /* Assert the code won't use GPUs even there are */ 322 } else SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Wrong device (%D) passed to -cuda_set_device <dev>. Must be NONE(-3),PETSC_DEFAULT(-2),PETSC_DECIDE(-1) or a non-negative integer.",device); 323 324 if (devCount > 0 && device != PETSC_NONE) { 325 /* Do costly CUDA handles initialization here to not to distort petsc logging later */ 326 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 327 ierr = PetscCUSOLVERDnInitializeHandle();CHKERRQ(ierr); 328 PetscCUDAInitialized = PETSC_TRUE; 329 } 330 } 331 PetscFunctionReturn(0); 332 } 333 #endif 334 335 /*@C 336 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 337 wishes a clean exit somewhere deep in the program. 338 339 Collective on PETSC_COMM_WORLD 340 341 Options Database Keys are the same as for PetscFinalize() 342 343 Level: advanced 344 345 Note: 346 See PetscInitialize() for more general runtime options. 347 348 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 349 @*/ 350 PetscErrorCode PetscEnd(void) 351 { 352 PetscFunctionBegin; 353 PetscFinalize(); 354 exit(0); 355 return 0; 356 } 357 358 PetscBool PetscOptionsPublish = PETSC_FALSE; 359 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 360 PETSC_INTERN PetscBool petscsetmallocvisited; 361 static char emacsmachinename[256]; 362 363 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL; 364 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL; 365 366 /*@C 367 PetscSetHelpVersionFunctions - Sets functions that print help and version information 368 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 369 This routine enables a "higher-level" package that uses PETSc to print its messages first. 370 371 Input Parameter: 372 + help - the help function (may be NULL) 373 - version - the version function (may be NULL) 374 375 Level: developer 376 377 @*/ 378 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 379 { 380 PetscFunctionBegin; 381 PetscExternalHelpFunction = help; 382 PetscExternalVersionFunction = version; 383 PetscFunctionReturn(0); 384 } 385 386 #if defined(PETSC_USE_LOG) 387 PETSC_INTERN PetscBool PetscObjectsLog; 388 #endif 389 390 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...) 391 { 392 if (PetscUnlikely(*err)) { 393 PetscMPIInt len; 394 char errstring[MPI_MAX_ERROR_STRING]; 395 396 MPI_Error_string(*err,errstring,&len); 397 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring); 398 } 399 return; 400 } 401 402 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[]) 403 { 404 char string[64]; 405 MPI_Comm comm = PETSC_COMM_WORLD; 406 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp; 407 PetscErrorCode ierr; 408 PetscReal si; 409 PetscInt intensity; 410 int i; 411 PetscMPIInt rank; 412 char version[256]; 413 #if defined(PETSC_USE_LOG) 414 char mname[PETSC_MAX_PATH_LEN]; 415 PetscViewerFormat format; 416 PetscBool flg4 = PETSC_FALSE; 417 #endif 418 419 PetscFunctionBegin; 420 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 421 422 #if !defined(PETSC_HAVE_THREADSAFETY) 423 if (!(PETSC_RUNNING_ON_VALGRIND)) { 424 /* 425 Setup the memory management; support for tracing malloc() usage 426 */ 427 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE; 428 429 if (PetscDefined(USE_DEBUG)) { 430 mdebug = PETSC_TRUE; 431 initializenan = PETSC_TRUE; 432 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 433 } else { 434 /* don't warn about unused option */ 435 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr); 436 flg1 = PETSC_FALSE; 437 } 438 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr); 439 if (flg1 || flg2) { 440 mdebug = PETSC_TRUE; 441 eachcall = PETSC_TRUE; 442 initializenan = PETSC_TRUE; 443 } else if (flg3 && !flg2) { 444 mdebug = PETSC_FALSE; 445 eachcall = PETSC_FALSE; 446 initializenan = PETSC_FALSE; 447 } 448 449 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr); 450 if (mlog) { 451 mdebug = PETSC_TRUE; 452 } 453 /* the next line is deprecated */ 454 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr); 455 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr); 456 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr); 457 if (mdebug) { 458 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr); 459 } 460 if (mlog) { 461 PetscReal logthreshold = 0; 462 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr); 463 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr); 464 } 465 #if defined(PETSC_USE_LOG) 466 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 467 #endif 468 } 469 470 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 471 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 472 flg1 = PETSC_FALSE; 473 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 474 /* ignore this option if malloc is already set */ 475 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 476 477 flg1 = PETSC_FALSE; 478 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 479 if (!flg1) { 480 flg1 = PETSC_FALSE; 481 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 482 } 483 if (flg1) { 484 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 485 } 486 #endif 487 488 #if defined(PETSC_USE_LOG) 489 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 490 #endif 491 492 /* 493 Set the display variable for graphics 494 */ 495 ierr = PetscSetDisplay();CHKERRQ(ierr); 496 497 /* 498 Print main application help message 499 */ 500 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr); 501 if (help && hasHelp) { 502 ierr = PetscPrintf(comm,help);CHKERRQ(ierr); 503 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr); 504 } 505 506 /* 507 Print the PETSc version information 508 */ 509 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr); 510 if (flg1 || hasHelp) { 511 /* 512 Print "higher-level" package version message 513 */ 514 if (PetscExternalVersionFunction) { 515 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 516 } 517 518 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 519 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 520 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 521 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 522 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 523 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 524 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 525 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr); 526 } 527 528 /* 529 Print "higher-level" package help message 530 */ 531 if (hasHelp) { 532 PetscBool hasHelpIntro; 533 534 if (PetscExternalHelpFunction) { 535 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 536 } 537 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr); 538 if (hasHelpIntro) { 539 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 540 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 541 ierr = MPI_Finalize();CHKERRQ(ierr); 542 exit(0); 543 } 544 } 545 546 /* 547 Setup the error handling 548 */ 549 flg1 = PETSC_FALSE; 550 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 551 if (flg1) { 552 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 553 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 554 } 555 flg1 = PETSC_FALSE; 556 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 557 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);} 558 flg1 = PETSC_FALSE; 559 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 560 if (flg1) { 561 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 562 } 563 /* experimental */ 564 flg1 = PETSC_FALSE; 565 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr); 566 if (flg1) { 567 MPI_Errhandler eh; 568 569 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr); 570 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr); 571 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr); 572 } 573 flg1 = PETSC_FALSE; 574 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 575 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 576 577 /* 578 Setup debugger information 579 */ 580 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 581 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 582 if (flg1) { 583 MPI_Errhandler err_handler; 584 585 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 586 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 587 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 588 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr); 589 } 590 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr); 591 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 592 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr); 593 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr); 594 if (flg1 || flg2) { 595 PetscMPIInt size; 596 PetscInt lsize,*nodes; 597 MPI_Errhandler err_handler; 598 /* 599 we have to make sure that all processors have opened 600 connections to all other processors, otherwise once the 601 debugger has stated it is likely to receive a SIGUSR1 602 and kill the program. 603 */ 604 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 605 if (size > 2) { 606 PetscMPIInt dummy = 0; 607 MPI_Status status; 608 for (i=0; i<size; i++) { 609 if (rank != i) { 610 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 611 } 612 } 613 for (i=0; i<size; i++) { 614 if (rank != i) { 615 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 616 } 617 } 618 } 619 /* check if this processor node should be in debugger */ 620 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 621 lsize = size; 622 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 623 if (flag) { 624 for (i=0; i<lsize; i++) { 625 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 626 } 627 } 628 if (!flag) { 629 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 630 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr); 631 if (flg1) { 632 ierr = PetscAttachDebugger();CHKERRQ(ierr); 633 } else { 634 ierr = PetscStopForDebugger();CHKERRQ(ierr); 635 } 636 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 637 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 638 } 639 ierr = PetscFree(nodes);CHKERRQ(ierr); 640 } 641 642 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr); 643 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 644 645 /* 646 Setup profiling and logging 647 */ 648 #if defined(PETSC_USE_INFO) 649 { 650 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr); 651 } 652 #endif 653 ierr = PetscDetermineInitialFPTrap(); 654 flg1 = PETSC_FALSE; 655 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr); 656 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);} 657 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 658 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 659 #if defined(PETSC_USE_LOG) 660 mname[0] = 0; 661 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 662 if (flg1) { 663 if (mname[0]) { 664 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 665 } else { 666 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 667 } 668 } 669 670 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 671 672 #if defined(PETSC_HAVE_MPE) 673 flg1 = PETSC_FALSE; 674 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 675 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 676 #endif 677 flg1 = PETSC_FALSE; 678 flg3 = PETSC_FALSE; 679 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 680 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 681 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 682 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 683 684 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr); 685 if (flg1) { 686 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 687 FILE *file; 688 if (mname[0]) { 689 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 690 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 691 file = fopen(fname,"w"); 692 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 693 } else file = PETSC_STDOUT; 694 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 695 } 696 697 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 698 if (flg4) { 699 if (format == PETSC_VIEWER_ASCII_XML) { 700 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 701 } else { 702 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 703 } 704 } 705 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 706 PetscReal threshold = PetscRealConstant(0.01); 707 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 708 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 709 } 710 #endif 711 712 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 713 714 #if defined(PETSC_HAVE_CUDA) 715 { 716 cudaError_t cerr; 717 PetscBool initCUDA = PETSC_FALSE,cudaView = PETSC_FALSE,logView = PETSC_FALSE,devNone = PETSC_FALSE; 718 struct cudaDeviceProp prop; 719 PetscInt device; 720 char devStr[16]={0}; 721 int devId,devCount; 722 /* 723 If collecting logging information, by default, wait for GPU to complete its operations 724 before returning to the CPU in order to get accurate timings of each event 725 */ 726 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr); 727 if (!logView) { 728 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr); 729 } 730 PetscCUDASynchronize = logView; 731 732 ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"CUDA options","Sys");CHKERRQ(ierr); 733 ierr = PetscOptionsString("-cuda_set_device","Set which MPI ranks to use which CUDA devices","PetscCUDAInitialize",devStr,devStr,sizeof(devStr),&initCUDA);CHKERRQ(ierr); 734 ierr = PetscStrcasecmp("none",devStr,&devNone);CHKERRQ(ierr); 735 if (devNone) device = -3; /* -3 is the locally used PETSC_NONE in PetscCUDAInitialize() */ 736 else {ierr = PetscOptionsInt("-cuda_set_device","Set which MPI ranks to use which CUDA devices","PetscCUDAInitialize",device,&device,&initCUDA);CHKERRQ(ierr);} 737 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU (on by default with -log_summary or -log_view)",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 738 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 739 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cudaView);CHKERRQ(ierr); 740 /* Get use_gpu_aware_mpi ASAP since it might be accessed even before lazy CUDA initialization */ 741 ierr = PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);CHKERRQ(ierr); 742 ierr = PetscOptionsEnd();CHKERRQ(ierr); 743 744 if (initCUDA) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD,device);CHKERRQ(ierr);} 745 else if (logView) { /* With -log_view, we want to do costly cuda runtime initialization early so that not to distort the timing later. */ 746 devCount = 0; 747 cerr = cudaGetDeviceCount(&devCount); 748 cudaGetLastError(); /* Reset the last error */ 749 if (cerr == cudaSuccess && devCount >= 1) { /* There are GPU(s) */ 750 devId = 0; 751 if (devCount > 1) { /* Decide which GPU to init when there are multiple GPUs */ 752 cerr = cudaSetDeviceFlags(cudaDeviceMapHost); 753 cudaGetLastError(); /* Reset the last error */ 754 if (cerr == cudaSuccess) { /* It implies cuda runtime has not been initialized */ 755 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 756 devId = rank % devCount; 757 cerr = cudaSetDevice(devId);CHKERRCUDA(cerr); 758 } else if (cerr == cudaErrorSetOnActiveProcess) { 759 /* It means user initialized cuda runtime outside of petsc. We respect the device choice. */ 760 cerr = cudaGetDevice(&devId);CHKERRCUDA(cerr); 761 } 762 } 763 ierr = PetscCUDAInitialize(PETSC_COMM_WORLD,(PetscInt)devId);CHKERRQ(ierr); 764 } 765 } 766 767 if (cudaView) { 768 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 769 cerr = cudaGetDeviceCount(&devCount);CHKERRCUDA(cerr); 770 for (devId = 0; devId < devCount; ++devId) { 771 cerr = cudaGetDeviceProperties(&prop,devId);CHKERRCUDA(cerr); 772 ierr = PetscPrintf(PETSC_COMM_WORLD, "CUDA device %d: %s\n", devId, prop.name);CHKERRQ(ierr); 773 } 774 cerr = cudaGetDevice(&devId);CHKERRCUDA(cerr); 775 ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Using CUDA device %d.\n",rank,devId);CHKERRQ(ierr); 776 ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr); 777 } 778 } 779 #endif 780 781 /* 782 Print basic help message 783 */ 784 if (hasHelp) { 785 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 786 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr); 787 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr); 788 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr); 789 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 790 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 791 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 792 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 793 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 794 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 795 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 796 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 797 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 798 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 799 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 800 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 801 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 802 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 803 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 804 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 805 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 806 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 807 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 808 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr); 809 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr); 810 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 811 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr); 812 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr); 813 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 814 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 815 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 816 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 817 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 818 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 819 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 820 #if defined(PETSC_USE_LOG) 821 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 822 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 823 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 824 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr); 825 #if defined(PETSC_HAVE_MPE) 826 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 827 #endif 828 #endif 829 #if defined(PETSC_USE_INFO) 830 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr); 831 #endif 832 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 833 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr); 834 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr); 835 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 836 } 837 838 #if defined(PETSC_HAVE_POPEN) 839 { 840 char machine[128]; 841 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr); 842 if (flg1) { 843 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 844 } 845 } 846 #endif 847 848 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 849 if (flg1) { 850 ierr = PetscSleep(si);CHKERRQ(ierr); 851 } 852 853 #if defined(PETSC_HAVE_VIENNACL) 854 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 855 if (!flg3) { 856 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 857 } 858 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 859 PetscViennaCLSynchronize = flg3; 860 ierr = PetscViennaCLInit();CHKERRQ(ierr); 861 #endif 862 863 /* 864 Creates the logging data structures; this is enabled even if logging is not turned on 865 This is the last thing we do before returning to the user code to prevent having the 866 logging numbers contaminated by any startup time associated with MPI and the GPUs 867 */ 868 #if defined(PETSC_USE_LOG) 869 ierr = PetscLogInitialize();CHKERRQ(ierr); 870 #endif 871 872 PetscFunctionReturn(0); 873 } 874