1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 #if defined(PETSC_USE_LOG) 14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 15 #endif 16 17 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 18 #include <sys/sysinfo.h> 19 #endif 20 #if defined(PETSC_HAVE_UNISTD_H) 21 #include <unistd.h> 22 #endif 23 #if defined(PETSC_HAVE_CUDA) 24 #include <cuda_runtime.h> 25 extern PetscErrorCode PetscCUBLASInitializeHandle(void); 26 #endif 27 28 #if defined(PETSC_HAVE_VIENNACL) 29 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 30 #endif 31 32 /* ------------------------Nasty global variables -------------------------------*/ 33 /* 34 Indicates if PETSc started up MPI, or it was 35 already started before PETSc was initialized. 36 */ 37 PetscBool PetscBeganMPI = PETSC_FALSE; 38 PetscBool PetscInitializeCalled = PETSC_FALSE; 39 PetscBool PetscFinalizeCalled = PETSC_FALSE; 40 PetscBool PetscCUDAInitialized = PETSC_FALSE; 41 42 PetscMPIInt PetscGlobalRank = -1; 43 PetscMPIInt PetscGlobalSize = -1; 44 45 #if defined(PETSC_HAVE_COMPLEX) 46 #if defined(PETSC_COMPLEX_INSTANTIATE) 47 template <> class std::complex<double>; /* instantiate complex template class */ 48 #endif 49 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 50 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 51 MPI_Datatype MPIU_C_COMPLEX; 52 #endif 53 54 /*MC 55 PETSC_i - the imaginary number i 56 57 Synopsis: 58 #include <petscsys.h> 59 PetscComplex PETSC_i; 60 61 Level: beginner 62 63 Note: 64 Complex numbers are automatically available if PETSc located a working complex implementation 65 66 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 67 M*/ 68 PetscComplex PETSC_i; 69 #endif 70 #if defined(PETSC_USE_REAL___FLOAT128) 71 MPI_Datatype MPIU___FLOAT128 = 0; 72 #if defined(PETSC_HAVE_COMPLEX) 73 MPI_Datatype MPIU___COMPLEX128 = 0; 74 #endif 75 #elif defined(PETSC_USE_REAL___FP16) 76 MPI_Datatype MPIU___FP16 = 0; 77 #endif 78 MPI_Datatype MPIU_2SCALAR = 0; 79 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 80 MPI_Datatype MPIU_2INT = 0; 81 #endif 82 MPI_Datatype MPIU_BOOL; 83 MPI_Datatype MPIU_ENUM; 84 MPI_Datatype MPIU_FORTRANADDR; 85 MPI_Datatype MPIU_SIZE_T; 86 87 /* 88 Function that is called to display all error messages 89 */ 90 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 91 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 92 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 93 /* 94 This is needed to turn on/off GPU synchronization 95 */ 96 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 97 PetscBool PetscCUDASynchronize = PETSC_FALSE; 98 99 /* ------------------------------------------------------------------------------*/ 100 /* 101 Optional file where all PETSc output from various prints is saved 102 */ 103 PETSC_INTERN FILE *petsc_history; 104 FILE *petsc_history = NULL; 105 106 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 107 { 108 PetscErrorCode ierr; 109 PetscMPIInt rank,size; 110 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 111 char version[256]; 112 113 PetscFunctionBegin; 114 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 115 if (!rank) { 116 char arch[10]; 117 int err; 118 119 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 120 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 121 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 122 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 123 if (filename) { 124 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 125 } else { 126 ierr = PetscGetHomeDirectory(pfile,240);CHKERRQ(ierr); 127 ierr = PetscStrcat(pfile,"/.petschistory");CHKERRQ(ierr); 128 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 129 } 130 131 *fd = fopen(fname,"a"); 132 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 133 134 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 135 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 136 ierr = PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr); 137 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 138 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 139 140 err = fflush(*fd); 141 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 142 } 143 PetscFunctionReturn(0); 144 } 145 146 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd) 147 { 148 PetscErrorCode ierr; 149 PetscMPIInt rank; 150 char date[64]; 151 int err; 152 153 PetscFunctionBegin; 154 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 155 if (!rank) { 156 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 157 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 158 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 159 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 160 err = fflush(*fd); 161 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 162 err = fclose(*fd); 163 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 164 } 165 PetscFunctionReturn(0); 166 } 167 168 /* ------------------------------------------------------------------------------*/ 169 170 /* 171 This is ugly and probably belongs somewhere else, but I want to 172 be able to put a true MPI abort error handler with command line args. 173 174 This is so MPI errors in the debugger will leave all the stack 175 frames. The default MP_Abort() cleans up and exits thus providing no useful information 176 in the debugger hence we call abort() instead of MPI_Abort(). 177 */ 178 179 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 180 { 181 PetscFunctionBegin; 182 (*PetscErrorPrintf)("MPI error %d\n",*flag); 183 abort(); 184 } 185 186 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...) 187 { 188 PetscErrorCode ierr; 189 190 PetscFunctionBegin; 191 (*PetscErrorPrintf)("MPI error %d\n",*flag); 192 ierr = PetscAttachDebugger(); 193 if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */ 194 } 195 196 #if defined(PETSC_HAVE_CUDA) 197 /*@C 198 PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device 199 200 Logically collective 201 202 Input Parameter: 203 comm - the MPI communicator that will utilize the CUDA devices 204 205 Options Database: 206 + -cuda_initialize - do the initialization in PetscInitialize() 207 . -cuda_view - view information about the CUDA devices 208 . -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view 209 - -cuda_set_device <gpu> - integer number of the device 210 211 Notes: 212 Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins 213 214 @*/ 215 PetscErrorCode PetscCUDAInitialize(MPI_Comm comm) 216 { 217 PetscErrorCode ierr; 218 PetscInt deviceOpt = 0; 219 PetscBool cuda_view_flag = PETSC_FALSE,flg; 220 struct cudaDeviceProp prop; 221 int devCount,device,devicecnt; 222 cudaError_t err = cudaSuccess; 223 PetscMPIInt rank,size; 224 225 PetscFunctionBegin; 226 /* 227 If collecting logging information, by default, wait for GPU to complete its operations 228 before returning to the CPU in order to get accurate timings of each event 229 */ 230 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);CHKERRQ(ierr); 231 if (!PetscCUDASynchronize) { 232 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);CHKERRQ(ierr); 233 } 234 235 ierr = PetscOptionsBegin(comm,NULL,"CUDA options","Sys");CHKERRQ(ierr); 236 ierr = PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);CHKERRQ(ierr); 237 device = (int)deviceOpt; 238 ierr = PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);CHKERRQ(ierr); 239 ierr = PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);CHKERRQ(ierr); 240 ierr = PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);CHKERRQ(ierr); 241 ierr = PetscOptionsEnd();CHKERRQ(ierr); 242 if (!PetscCUDAInitialized) { 243 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 244 245 if (size>1 && !flg) { 246 /* check to see if we force multiple ranks to hit the same GPU */ 247 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 248 249 /* First get the device count */ 250 err = cudaGetDeviceCount(&devCount); 251 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 252 253 /* next determine the rank and then set the device via a mod */ 254 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 255 device = rank % devCount; 256 } 257 err = cudaSetDevice(device); 258 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 259 260 /* set the device flags so that it can map host memory */ 261 err = cudaSetDeviceFlags(cudaDeviceMapHost); 262 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 263 264 ierr = PetscCUBLASInitializeHandle();CHKERRQ(ierr); 265 PetscCUDAInitialized = PETSC_TRUE; 266 } 267 if (cuda_view_flag) { 268 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 269 err = cudaGetDeviceCount(&devCount); 270 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 271 for (devicecnt = 0; devicecnt < devCount; ++devicecnt) { 272 err = cudaGetDeviceProperties(&prop,devicecnt); 273 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 274 ierr = PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);CHKERRQ(ierr); 275 } 276 ierr = PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);CHKERRQ(ierr); 277 ierr = PetscSynchronizedFlush(comm,PETSC_STDOUT);CHKERRQ(ierr); 278 } 279 PetscFunctionReturn(0); 280 } 281 #endif 282 283 /*@C 284 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 285 wishes a clean exit somewhere deep in the program. 286 287 Collective on PETSC_COMM_WORLD 288 289 Options Database Keys are the same as for PetscFinalize() 290 291 Level: advanced 292 293 Note: 294 See PetscInitialize() for more general runtime options. 295 296 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 297 @*/ 298 PetscErrorCode PetscEnd(void) 299 { 300 PetscFunctionBegin; 301 PetscFinalize(); 302 exit(0); 303 return 0; 304 } 305 306 PetscBool PetscOptionsPublish = PETSC_FALSE; 307 PETSC_INTERN PetscErrorCode PetscSetUseTrMalloc_Private(void); 308 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void); 309 PETSC_INTERN PetscBool petscsetmallocvisited; 310 static char emacsmachinename[256]; 311 312 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0; 313 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0; 314 315 /*@C 316 PetscSetHelpVersionFunctions - Sets functions that print help and version information 317 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 318 This routine enables a "higher-level" package that uses PETSc to print its messages first. 319 320 Input Parameter: 321 + help - the help function (may be NULL) 322 - version - the version function (may be NULL) 323 324 Level: developer 325 326 @*/ 327 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 328 { 329 PetscFunctionBegin; 330 PetscExternalHelpFunction = help; 331 PetscExternalVersionFunction = version; 332 PetscFunctionReturn(0); 333 } 334 335 #if defined(PETSC_USE_LOG) 336 PETSC_INTERN PetscBool PetscObjectsLog; 337 #endif 338 339 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(void) 340 { 341 char string[64],mname[PETSC_MAX_PATH_LEN],*f; 342 MPI_Comm comm = PETSC_COMM_WORLD; 343 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag; 344 PetscErrorCode ierr; 345 PetscReal si; 346 PetscInt intensity; 347 int i; 348 PetscMPIInt rank; 349 char version[256],helpoptions[256]; 350 #if !defined(PETSC_HAVE_THREADSAFETY) 351 PetscReal logthreshold; 352 #endif 353 #if defined(PETSC_USE_LOG) 354 PetscViewerFormat format; 355 PetscBool flg4 = PETSC_FALSE; 356 #endif 357 #if defined(PETSC_HAVE_CUDA) 358 PetscBool initCuda = PETSC_TRUE; 359 #endif 360 361 PetscFunctionBegin; 362 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 363 364 #if !defined(PETSC_HAVE_THREADSAFETY) 365 /* 366 Setup the memory management; support for tracing malloc() usage 367 */ 368 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);CHKERRQ(ierr); 369 logthreshold = 0.0; 370 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);CHKERRQ(ierr); 371 if (flg1) flg3 = PETSC_TRUE; 372 #if defined(PETSC_USE_DEBUG) 373 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);CHKERRQ(ierr); 374 if ((!flg2 || flg1) && !petscsetmallocvisited) { 375 if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) { 376 /* turn off default -malloc if valgrind is being used */ 377 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 378 } 379 } 380 #else 381 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);CHKERRQ(ierr); 382 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);CHKERRQ(ierr); 383 if (flg1 || flg2 || flg3) {ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr);} 384 #endif 385 if (flg3) { 386 ierr = PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);CHKERRQ(ierr); 387 } 388 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 389 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 390 flg1 = PETSC_FALSE; 391 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);CHKERRQ(ierr); 392 if (flg1) { 393 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 394 ierr = PetscMallocDebug(PETSC_TRUE);CHKERRQ(ierr); 395 } 396 flg1 = PETSC_FALSE; 397 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);CHKERRQ(ierr); 398 #if defined(PETSC_USE_DEBUG) 399 if (flg1 && !PETSC_RUNNING_ON_VALGRIND) { 400 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 401 ierr = PetscMallocSetDumpLog();CHKERRQ(ierr); 402 ierr = PetscMallocDebug(PETSC_TRUE);CHKERRQ(ierr); 403 } 404 #endif 405 flg1 = PETSC_FALSE; 406 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 407 /* ignore this option if malloc is already set */ 408 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 409 410 flg1 = PETSC_FALSE; 411 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 412 if (!flg1) { 413 flg1 = PETSC_FALSE; 414 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 415 } 416 if (flg1) { 417 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 418 } 419 #endif 420 421 #if defined(PETSC_USE_LOG) 422 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 423 #endif 424 425 /* 426 Set the display variable for graphics 427 */ 428 ierr = PetscSetDisplay();CHKERRQ(ierr); 429 430 /* 431 Print the PETSc version information 432 */ 433 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 434 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 435 ierr = PetscOptionsHasHelp(NULL,&flg3);CHKERRQ(ierr); 436 if (flg1 || flg2 || flg3) { 437 438 /* 439 Print "higher-level" package version message 440 */ 441 if (PetscExternalVersionFunction) { 442 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 443 } 444 445 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 446 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");CHKERRQ(ierr); 447 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 448 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 449 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 450 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 451 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 452 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 453 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");CHKERRQ(ierr); 454 } 455 456 /* 457 Print "higher-level" package help message 458 */ 459 if (flg3) { 460 if (PetscExternalHelpFunction) { 461 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 462 } 463 } 464 465 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 466 if (flg1) { 467 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 468 if (flg2) { 469 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 470 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 471 ierr = MPI_Finalize();CHKERRQ(ierr); 472 exit(0); 473 } 474 } 475 476 /* 477 Setup the error handling 478 */ 479 flg1 = PETSC_FALSE; 480 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 481 if (flg1) { 482 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 483 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr); 484 } 485 flg1 = PETSC_FALSE; 486 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 487 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);CHKERRQ(ierr);} 488 flg1 = PETSC_FALSE; 489 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 490 if (flg1) { 491 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 492 } 493 flg1 = PETSC_FALSE; 494 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 495 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 496 flg1 = PETSC_FALSE; 497 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);CHKERRQ(ierr); 498 if (flg1) {ierr = PetscSetFPTrap(PETSC_FP_TRAP_ON);CHKERRQ(ierr);} 499 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 500 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 501 502 /* 503 Setup debugger information 504 */ 505 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 506 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);CHKERRQ(ierr); 507 if (flg1) { 508 MPI_Errhandler err_handler; 509 510 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 511 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 512 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 513 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);CHKERRQ(ierr); 514 } 515 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);CHKERRQ(ierr); 516 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 517 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);CHKERRQ(ierr); 518 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);CHKERRQ(ierr); 519 if (flg1 || flg2) { 520 PetscMPIInt size; 521 PetscInt lsize,*nodes; 522 MPI_Errhandler err_handler; 523 /* 524 we have to make sure that all processors have opened 525 connections to all other processors, otherwise once the 526 debugger has stated it is likely to receive a SIGUSR1 527 and kill the program. 528 */ 529 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 530 if (size > 2) { 531 PetscMPIInt dummy = 0; 532 MPI_Status status; 533 for (i=0; i<size; i++) { 534 if (rank != i) { 535 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 536 } 537 } 538 for (i=0; i<size; i++) { 539 if (rank != i) { 540 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 541 } 542 } 543 } 544 /* check if this processor node should be in debugger */ 545 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 546 lsize = size; 547 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 548 if (flag) { 549 for (i=0; i<lsize; i++) { 550 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 551 } 552 } 553 if (!flag) { 554 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 555 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr); 556 if (flg1) { 557 ierr = PetscAttachDebugger();CHKERRQ(ierr); 558 } else { 559 ierr = PetscStopForDebugger();CHKERRQ(ierr); 560 } 561 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 562 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 563 } 564 ierr = PetscFree(nodes);CHKERRQ(ierr); 565 } 566 567 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);CHKERRQ(ierr); 568 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 569 570 /* 571 Setup profiling and logging 572 */ 573 #if defined(PETSC_USE_INFO) 574 { 575 char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0; 576 ierr = PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);CHKERRQ(ierr); 577 if (flg1 && logname[0]) { 578 ierr = PetscInfoAllow(PETSC_TRUE,logname);CHKERRQ(ierr); 579 } else if (flg1) { 580 ierr = PetscInfoAllow(PETSC_TRUE,NULL);CHKERRQ(ierr); 581 } 582 } 583 #endif 584 #if defined(PETSC_USE_LOG) 585 mname[0] = 0; 586 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 587 if (flg1) { 588 if (mname[0]) { 589 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 590 } else { 591 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 592 } 593 } 594 595 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr); 596 597 #if defined(PETSC_HAVE_MPE) 598 flg1 = PETSC_FALSE; 599 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 600 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 601 #endif 602 flg1 = PETSC_FALSE; 603 flg3 = PETSC_FALSE; 604 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 605 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 606 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 607 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 608 609 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);CHKERRQ(ierr); 610 if (flg1) { 611 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 612 FILE *file; 613 if (mname[0]) { 614 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 615 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 616 file = fopen(fname,"w"); 617 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 618 } else file = PETSC_STDOUT; 619 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 620 } 621 622 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 623 if (flg4) { 624 if (format == PETSC_VIEWER_ASCII_XML) { 625 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 626 } else { 627 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 628 } 629 PetscLogMemory = PETSC_FALSE; 630 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr); 631 if (PetscLogMemory) { 632 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 633 } 634 } 635 if (flg4 && format == PETSC_VIEWER_ASCII_XML) { 636 PetscReal threshold = PetscRealConstant(0.01); 637 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr); 638 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);} 639 } 640 #endif 641 642 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 643 644 #if defined(PETSC_HAVE_CUDA) 645 ierr = PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");CHKERRQ(ierr); 646 ierr = PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCuda,&initCuda,NULL);CHKERRQ(ierr); 647 ierr = PetscOptionsEnd();CHKERRQ(ierr); 648 if (initCuda) {ierr = PetscCUDAInitialize(PETSC_COMM_WORLD);CHKERRQ(ierr);} 649 #endif 650 651 /* 652 Print basic help message 653 */ 654 ierr = PetscOptionsHasHelp(NULL,&flg1);CHKERRQ(ierr); 655 if (flg1) { 656 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 657 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 658 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 659 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 660 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 661 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 662 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 663 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 664 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 665 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 666 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 667 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 668 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 669 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 670 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 671 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 672 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 673 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 674 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 675 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 676 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 677 ierr = (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");CHKERRQ(ierr); 678 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");CHKERRQ(ierr); 679 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 680 ierr = (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");CHKERRQ(ierr); 681 ierr = (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");CHKERRQ(ierr); 682 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr); 683 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 684 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 685 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 686 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 687 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 688 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 689 #if defined(PETSC_USE_LOG) 690 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 691 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 692 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 693 #if defined(PETSC_HAVE_MPE) 694 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 695 #endif 696 ierr = (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");CHKERRQ(ierr); 697 #endif 698 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 699 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 700 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 701 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 702 } 703 704 #if defined(PETSC_HAVE_POPEN) 705 { 706 char machine[128]; 707 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);CHKERRQ(ierr); 708 if (flg1) { 709 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 710 } 711 } 712 #endif 713 714 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 715 if (flg1) { 716 ierr = PetscSleep(si);CHKERRQ(ierr); 717 } 718 719 ierr = PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 720 if (flg1) { 721 ierr = PetscStrstr(mname,"null",&f);CHKERRQ(ierr); 722 if (f) { 723 ierr = PetscInfoDeactivateClass(0);CHKERRQ(ierr); 724 } 725 } 726 727 #if defined(PETSC_HAVE_VIENNACL) 728 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 729 if (!flg3) { 730 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 731 } 732 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 733 PetscViennaCLSynchronize = flg3; 734 ierr = PetscViennaCLInit();CHKERRQ(ierr); 735 #endif 736 737 /* 738 Creates the logging data structures; this is enabled even if logging is not turned on 739 This is the last thing we do before returning to the user code to prevent having the 740 logging numbers contaminated by any startup time associated with MPI and the GPUs 741 */ 742 #if defined(PETSC_USE_LOG) 743 ierr = PetscLogInitialize();CHKERRQ(ierr); 744 #endif 745 746 PetscFunctionReturn(0); 747 } 748