1 /* 2 3 This file defines part of the initialization of PETSc 4 5 This file uses regular malloc and free because it cannot known 6 what malloc is being used until it has already processed the input. 7 */ 8 9 #include <petscsys.h> /*I "petscsys.h" I*/ 10 #include <petsc/private/petscimpl.h> 11 #include <petscvalgrind.h> 12 #include <petscviewer.h> 13 14 #if defined(PETSC_HAVE_SYS_SYSINFO_H) 15 #include <sys/sysinfo.h> 16 #endif 17 #if defined(PETSC_HAVE_UNISTD_H) 18 #include <unistd.h> 19 #endif 20 #if defined(PETSC_HAVE_CUDA) 21 #include <cuda_runtime.h> 22 #endif 23 24 #if defined(PETSC_HAVE_VIENNACL) 25 PETSC_EXTERN PetscErrorCode PetscViennaCLInit(); 26 #endif 27 28 /* ------------------------Nasty global variables -------------------------------*/ 29 /* 30 Indicates if PETSc started up MPI, or it was 31 already started before PETSc was initialized. 32 */ 33 PetscBool PetscBeganMPI = PETSC_FALSE; 34 PetscBool PetscInitializeCalled = PETSC_FALSE; 35 PetscBool PetscFinalizeCalled = PETSC_FALSE; 36 PetscBool PetscCUDAInitialized = PETSC_FALSE; 37 38 PetscMPIInt PetscGlobalRank = -1; 39 PetscMPIInt PetscGlobalSize = -1; 40 41 #if defined(PETSC_HAVE_COMPLEX) 42 #if defined(PETSC_COMPLEX_INSTANTIATE) 43 template <> class std::complex<double>; /* instantiate complex template class */ 44 #endif 45 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 46 MPI_Datatype MPIU_C_DOUBLE_COMPLEX; 47 MPI_Datatype MPIU_C_COMPLEX; 48 #endif 49 50 /*MC 51 PETSC_i - the imaginary number i 52 53 Synopsis: 54 #include <petscsys.h> 55 PetscComplex PETSC_i; 56 57 Level: beginner 58 59 Note: 60 Complex numbers are automatically available if PETSc located a working complex implementation 61 62 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex() 63 M*/ 64 PetscComplex PETSC_i; 65 #endif 66 #if defined(PETSC_USE_REAL___FLOAT128) 67 MPI_Datatype MPIU___FLOAT128 = 0; 68 #if defined(PETSC_HAVE_COMPLEX) 69 MPI_Datatype MPIU___COMPLEX128 = 0; 70 #endif 71 #elif defined(PETSC_USE_REAL___FP16) 72 MPI_Datatype MPIU___FP16 = 0; 73 #endif 74 MPI_Datatype MPIU_2SCALAR = 0; 75 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 76 MPI_Datatype MPIU_2INT = 0; 77 #endif 78 MPI_Datatype MPIU_BOOL; 79 MPI_Datatype MPIU_ENUM; 80 81 /* 82 Function that is called to display all error messages 83 */ 84 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 85 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 86 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 87 /* 88 This is needed to turn on/off GPU synchronization 89 */ 90 PetscBool PetscViennaCLSynchronize = PETSC_FALSE; 91 PetscBool PetscCUDASynchronize = PETSC_FALSE; 92 93 /* ------------------------------------------------------------------------------*/ 94 /* 95 Optional file where all PETSc output from various prints is saved 96 */ 97 FILE *petsc_history = NULL; 98 99 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 100 { 101 PetscErrorCode ierr; 102 PetscMPIInt rank,size; 103 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 104 char version[256]; 105 106 PetscFunctionBegin; 107 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 108 if (!rank) { 109 char arch[10]; 110 int err; 111 112 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 113 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 114 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 115 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 116 if (filename) { 117 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 118 } else { 119 ierr = PetscGetHomeDirectory(pfile,240);CHKERRQ(ierr); 120 ierr = PetscStrcat(pfile,"/.petschistory");CHKERRQ(ierr); 121 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 122 } 123 124 *fd = fopen(fname,"a"); 125 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 126 127 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 128 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 129 ierr = PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr); 130 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 131 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 132 133 err = fflush(*fd); 134 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 135 } 136 PetscFunctionReturn(0); 137 } 138 139 PetscErrorCode PetscCloseHistoryFile(FILE **fd) 140 { 141 PetscErrorCode ierr; 142 PetscMPIInt rank; 143 char date[64]; 144 int err; 145 146 PetscFunctionBegin; 147 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 148 if (!rank) { 149 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 150 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 151 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 153 err = fflush(*fd); 154 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 155 err = fclose(*fd); 156 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 157 } 158 PetscFunctionReturn(0); 159 } 160 161 /* ------------------------------------------------------------------------------*/ 162 163 /* 164 This is ugly and probably belongs somewhere else, but I want to 165 be able to put a true MPI abort error handler with command line args. 166 167 This is so MPI errors in the debugger will leave all the stack 168 frames. The default MP_Abort() cleans up and exits thus providing no useful information 169 in the debugger hence we call abort() instead of MPI_Abort(). 170 */ 171 172 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag) 173 { 174 PetscFunctionBegin; 175 (*PetscErrorPrintf)("MPI error %d\n",*flag); 176 abort(); 177 } 178 179 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag) 180 { 181 PetscErrorCode ierr; 182 183 PetscFunctionBegin; 184 (*PetscErrorPrintf)("MPI error %d\n",*flag); 185 ierr = PetscAttachDebugger(); 186 if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */ 187 } 188 189 /*@C 190 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 191 wishes a clean exit somewhere deep in the program. 192 193 Collective on PETSC_COMM_WORLD 194 195 Options Database Keys are the same as for PetscFinalize() 196 197 Level: advanced 198 199 Note: 200 See PetscInitialize() for more general runtime options. 201 202 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 203 @*/ 204 PetscErrorCode PetscEnd(void) 205 { 206 PetscFunctionBegin; 207 PetscFinalize(); 208 exit(0); 209 return 0; 210 } 211 212 PetscBool PetscOptionsPublish = PETSC_FALSE; 213 extern PetscErrorCode PetscSetUseTrMalloc_Private(void); 214 extern PetscErrorCode PetscSetUseHBWMalloc_Private(void); 215 extern PetscBool petscsetmallocvisited; 216 static char emacsmachinename[256]; 217 218 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0; 219 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0; 220 221 /*@C 222 PetscSetHelpVersionFunctions - Sets functions that print help and version information 223 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 224 This routine enables a "higher-level" package that uses PETSc to print its messages first. 225 226 Input Parameter: 227 + help - the help function (may be NULL) 228 - version - the version function (may be NULL) 229 230 Level: developer 231 232 Concepts: package help message 233 234 @*/ 235 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 236 { 237 PetscFunctionBegin; 238 PetscExternalHelpFunction = help; 239 PetscExternalVersionFunction = version; 240 PetscFunctionReturn(0); 241 } 242 243 #if defined(PETSC_USE_LOG) 244 extern PetscBool PetscObjectsLog; 245 #endif 246 247 PetscErrorCode PetscOptionsCheckInitial_Private(void) 248 { 249 char string[64],mname[PETSC_MAX_PATH_LEN],*f; 250 MPI_Comm comm = PETSC_COMM_WORLD; 251 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag; 252 PetscErrorCode ierr; 253 PetscReal si; 254 PetscInt intensity; 255 int i; 256 PetscMPIInt rank; 257 char version[256],helpoptions[256]; 258 #if !defined(PETSC_HAVE_THREADSAFETY) 259 PetscReal logthreshold; 260 #endif 261 #if defined(PETSC_USE_LOG) 262 PetscViewerFormat format; 263 PetscBool flg4 = PETSC_FALSE; 264 #endif 265 266 PetscFunctionBegin; 267 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 268 269 #if !defined(PETSC_HAVE_THREADSAFETY) 270 /* 271 Setup the memory management; support for tracing malloc() usage 272 */ 273 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);CHKERRQ(ierr); 274 logthreshold = 0.0; 275 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);CHKERRQ(ierr); 276 if (flg1) flg3 = PETSC_TRUE; 277 #if defined(PETSC_USE_DEBUG) 278 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);CHKERRQ(ierr); 279 if ((!flg2 || flg1) && !petscsetmallocvisited) { 280 if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) { 281 /* turn off default -malloc if valgrind is being used */ 282 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 283 } 284 } 285 #else 286 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);CHKERRQ(ierr); 287 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);CHKERRQ(ierr); 288 if (flg1 || flg2 || flg3) {ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr);} 289 #endif 290 if (flg3) { 291 ierr = PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);CHKERRQ(ierr); 292 } 293 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr); 294 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);} 295 flg1 = PETSC_FALSE; 296 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);CHKERRQ(ierr); 297 if (flg1) { 298 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 299 ierr = PetscMallocDebug(PETSC_TRUE);CHKERRQ(ierr); 300 } 301 flg1 = PETSC_FALSE; 302 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);CHKERRQ(ierr); 303 #if defined(PETSC_USE_DEBUG) 304 if (flg1 && !PETSC_RUNNING_ON_VALGRIND) { 305 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 306 ierr = PetscMallocSetDumpLog();CHKERRQ(ierr); 307 ierr = PetscMallocDebug(PETSC_TRUE);CHKERRQ(ierr); 308 } 309 #endif 310 flg1 = PETSC_FALSE; 311 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr); 312 /* ignore this option if malloc is already set */ 313 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);} 314 315 flg1 = PETSC_FALSE; 316 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr); 317 if (!flg1) { 318 flg1 = PETSC_FALSE; 319 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr); 320 } 321 if (flg1) { 322 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 323 } 324 #endif 325 326 #if defined(PETSC_USE_LOG) 327 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr); 328 #endif 329 330 /* 331 Set the display variable for graphics 332 */ 333 ierr = PetscSetDisplay();CHKERRQ(ierr); 334 335 /* 336 Print the PETSc version information 337 */ 338 ierr = PetscOptionsHasName(NULL,NULL,"-v",&flg1);CHKERRQ(ierr); 339 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg2);CHKERRQ(ierr); 340 ierr = PetscOptionsHasName(NULL,NULL,"-help",&flg3);CHKERRQ(ierr); 341 if (flg1 || flg2 || flg3) { 342 343 /* 344 Print "higher-level" package version message 345 */ 346 if (PetscExternalVersionFunction) { 347 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 348 } 349 350 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 351 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");CHKERRQ(ierr); 352 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 353 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 354 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 355 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 356 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 357 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 358 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");CHKERRQ(ierr); 359 } 360 361 /* 362 Print "higher-level" package help message 363 */ 364 if (flg3) { 365 if (PetscExternalHelpFunction) { 366 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 367 } 368 } 369 370 ierr = PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);CHKERRQ(ierr); 371 if (flg1) { 372 ierr = PetscStrcmp(helpoptions,"intro",&flg2);CHKERRQ(ierr); 373 if (flg2) { 374 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr); 375 ierr = PetscFreeMPIResources();CHKERRQ(ierr); 376 ierr = MPI_Finalize();CHKERRQ(ierr); 377 exit(0); 378 } 379 } 380 381 /* 382 Setup the error handling 383 */ 384 flg1 = PETSC_FALSE; 385 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr); 386 if (flg1) { 387 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr); 388 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr); 389 } 390 flg1 = PETSC_FALSE; 391 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr); 392 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);CHKERRQ(ierr);} 393 flg1 = PETSC_FALSE; 394 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr); 395 if (flg1) { 396 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 397 } 398 flg1 = PETSC_FALSE; 399 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr); 400 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);} 401 flg1 = PETSC_FALSE; 402 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);CHKERRQ(ierr); 403 if (flg1) {ierr = PetscSetFPTrap(PETSC_FP_TRAP_ON);CHKERRQ(ierr);} 404 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr); 405 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);} 406 407 /* 408 Setup debugger information 409 */ 410 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 411 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);CHKERRQ(ierr); 412 if (flg1) { 413 MPI_Errhandler err_handler; 414 415 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 416 ierr = MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 417 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 418 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);CHKERRQ(ierr); 419 } 420 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);CHKERRQ(ierr); 421 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 422 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);CHKERRQ(ierr); 423 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);CHKERRQ(ierr); 424 if (flg1 || flg2) { 425 PetscMPIInt size; 426 PetscInt lsize,*nodes; 427 MPI_Errhandler err_handler; 428 /* 429 we have to make sure that all processors have opened 430 connections to all other processors, otherwise once the 431 debugger has stated it is likely to receive a SIGUSR1 432 and kill the program. 433 */ 434 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 435 if (size > 2) { 436 PetscMPIInt dummy = 0; 437 MPI_Status status; 438 for (i=0; i<size; i++) { 439 if (rank != i) { 440 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr); 441 } 442 } 443 for (i=0; i<size; i++) { 444 if (rank != i) { 445 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr); 446 } 447 } 448 } 449 /* check if this processor node should be in debugger */ 450 ierr = PetscMalloc1(size,&nodes);CHKERRQ(ierr); 451 lsize = size; 452 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 453 if (flag) { 454 for (i=0; i<lsize; i++) { 455 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 456 } 457 } 458 if (!flag) { 459 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 460 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr); 461 if (flg1) { 462 ierr = PetscAttachDebugger();CHKERRQ(ierr); 463 } else { 464 ierr = PetscStopForDebugger();CHKERRQ(ierr); 465 } 466 ierr = MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 467 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr); 468 } 469 ierr = PetscFree(nodes);CHKERRQ(ierr); 470 } 471 472 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);CHKERRQ(ierr); 473 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 474 475 /* 476 Setup profiling and logging 477 */ 478 #if defined(PETSC_USE_INFO) 479 { 480 char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0; 481 ierr = PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);CHKERRQ(ierr); 482 if (flg1 && logname[0]) { 483 ierr = PetscInfoAllow(PETSC_TRUE,logname);CHKERRQ(ierr); 484 } else if (flg1) { 485 ierr = PetscInfoAllow(PETSC_TRUE,NULL);CHKERRQ(ierr); 486 } 487 } 488 #endif 489 #if defined(PETSC_USE_LOG) 490 mname[0] = 0; 491 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 492 if (flg1) { 493 if (mname[0]) { 494 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 495 } else { 496 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr); 497 } 498 } 499 #if defined(PETSC_HAVE_MPE) 500 flg1 = PETSC_FALSE; 501 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 502 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);} 503 #endif 504 flg1 = PETSC_FALSE; 505 flg3 = PETSC_FALSE; 506 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr); 507 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 508 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 509 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);} 510 511 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);CHKERRQ(ierr); 512 if (flg1) { 513 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 514 FILE *file; 515 if (mname[0]) { 516 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank); 517 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 518 file = fopen(fname,"w"); 519 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 520 } else file = PETSC_STDOUT; 521 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 522 } 523 524 ierr = PetscOptionsGetViewer(comm,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr); 525 if (flg4) { 526 if (format == PETSC_VIEWER_ASCII_XML){ 527 ierr = PetscLogNestedBegin();CHKERRQ(ierr); 528 } else { 529 ierr = PetscLogDefaultBegin();CHKERRQ(ierr); 530 } 531 } 532 #endif 533 534 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr); 535 536 537 #if defined(PETSC_HAVE_CUDA) 538 ierr = PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);CHKERRQ(ierr); 539 if (flg1) { 540 struct cudaDeviceProp prop; 541 int devCount; 542 PetscInt device; 543 cudaError_t err = cudaSuccess; 544 545 err = cudaGetDeviceCount(&devCount); 546 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 547 for (device = 0; device < devCount; ++device) { 548 err = cudaGetDeviceProperties(&prop, (int)device); 549 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err)); 550 ierr = PetscPrintf(comm, "CUDA device %D: %s\n", device, prop.name);CHKERRQ(ierr); 551 } 552 } 553 if (!PetscCUDAInitialized) { 554 PetscMPIInt size; 555 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 556 if (size>1) { 557 int devCount; 558 PetscInt device; 559 PetscMPIInt rank; 560 cudaError_t err = cudaSuccess; 561 562 /* check to see if we force multiple ranks to hit the same GPU */ 563 ierr = PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);CHKERRQ(ierr); 564 if (flg1) { 565 err = cudaSetDevice((int)device); 566 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 567 } else { 568 /* we're not using the same GPU on multiple MPI threads. So try to allocated different GPUs to different processes */ 569 570 /* First get the device count */ 571 err = cudaGetDeviceCount(&devCount); 572 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err)); 573 574 /* next determine the rank and then set the device via a mod */ 575 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 576 device = rank % devCount; 577 err = cudaSetDevice((int)device); 578 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 579 } 580 581 /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess 582 multiple devices may try to set the flags on the same device. So long as one of them succeeds, things 583 are ok. */ 584 err = cudaSetDeviceFlags(cudaDeviceMapHost); 585 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 586 } else { 587 PetscInt device; 588 cudaError_t err = cudaSuccess; 589 590 /* the code below works for serial GPU simulations */ 591 ierr = PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);CHKERRQ(ierr); 592 if (flg1) { 593 err = cudaSetDevice((int)device); 594 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err)); 595 } 596 597 /* set the device flags so that it can map host memory ... here, we error check. */ 598 err = cudaSetDeviceFlags(cudaDeviceMapHost); 599 if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err)); 600 } 601 602 PetscCUDAInitialized = PETSC_TRUE; 603 } 604 #endif 605 606 607 /* 608 Print basic help message 609 */ 610 ierr = PetscOptionsHasName(NULL,NULL,"-help",&flg1);CHKERRQ(ierr); 611 if (flg1) { 612 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 613 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 614 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 615 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 616 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 617 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 618 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 619 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 620 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 621 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 622 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 623 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 624 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 625 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 626 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 627 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr); 628 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 629 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 630 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 631 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 632 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 633 ierr = (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");CHKERRQ(ierr); 634 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");CHKERRQ(ierr); 635 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 636 ierr = (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");CHKERRQ(ierr); 637 ierr = (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");CHKERRQ(ierr); 638 ierr = (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");CHKERRQ(ierr); 639 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 640 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 641 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 642 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 643 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 644 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr); 645 #if defined(PETSC_USE_LOG) 646 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 647 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr); 648 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 649 #if defined(PETSC_HAVE_MPE) 650 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr); 651 #endif 652 ierr = (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");CHKERRQ(ierr); 653 #endif 654 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 655 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 656 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 657 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 658 } 659 660 #if defined(PETSC_HAVE_POPEN) 661 { 662 char machine[128]; 663 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);CHKERRQ(ierr); 664 if (flg1) { 665 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr); 666 } 667 } 668 #endif 669 670 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 671 if (flg1) { 672 ierr = PetscSleep(si);CHKERRQ(ierr); 673 } 674 675 ierr = PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 676 if (flg1) { 677 ierr = PetscStrstr(mname,"null",&f);CHKERRQ(ierr); 678 if (f) { 679 ierr = PetscInfoDeactivateClass(0);CHKERRQ(ierr); 680 } 681 } 682 683 #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_VECCUDA) 684 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr); 685 if (!flg3) { 686 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr); 687 } 688 #endif 689 #if defined(PETSC_HAVE_VIENNACL) 690 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr); 691 PetscViennaCLSynchronize = flg3; 692 #endif 693 #if defined(PETSC_HAVE_VECCUDA) 694 ierr = PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);CHKERRQ(ierr); 695 PetscCUDASynchronize = flg3; 696 #endif 697 698 #if defined(PETSC_HAVE_VIENNACL) 699 ierr = PetscViennaCLInit();CHKERRQ(ierr); 700 #endif 701 702 PetscFunctionReturn(0); 703 } 704