1 /* $Id: petsclog.h,v 1.155 2001/09/06 14:51:20 bsmith Exp $ */ 2 3 /* 4 Defines profile/logging in PETSc. 5 */ 6 7 #if !defined(__PetscLog_H) 8 #define __PetscLog_H 9 #include "petsc.h" 10 PETSC_EXTERN_CXX_BEGIN 11 /* 12 Each PETSc object class has it's own cookie (internal integer in the 13 data structure used for error checking). These are all defined by an offset 14 from the lowest one, PETSC_COOKIE. 15 */ 16 #define PETSC_COOKIE 1211211 17 extern int PETSC_LARGEST_COOKIE; 18 #define PETSC_EVENT 1311311 19 extern int PETSC_LARGEST_EVENT; 20 21 /* Events for the Petsc standard library */ 22 extern int PETSC_Barrier; 23 24 /* Global flop counter */ 25 extern PetscLogDouble _TotalFlops; 26 27 /* General logging of information; different from event logging */ 28 EXTERN int PetscLogInfo(void*,const char[],...) PETSC_PRINTF_FORMAT_CHECK(2,3); 29 EXTERN int PetscLogInfoDeactivateClass(int); 30 EXTERN int PetscLogInfoActivateClass(int); 31 extern PetscTruth PetscLogPrintInfo; /* if true, indicates PetscLogInfo() is turned on */ 32 33 #if defined(PETSC_USE_LOG) /* --- Logging is turned on --------------------------------*/ 34 35 /* 36 Flop counting: We count each arithmetic operation (e.g., addition, multiplication) separately. 37 38 For the complex numbers version, note that 39 1 complex addition = 2 flops 40 1 complex multiplication = 6 flops, 41 where we define 1 flop as that for a double precision scalar. We roughly approximate 42 flop counting for complex numbers by multiplying the total flops by 4; this corresponds 43 to the assumption that we're counting mostly additions and multiplications -- and 44 roughly the same number of each. More accurate counting could be done by distinguishing 45 among the various arithmetic operations. 46 */ 47 48 #if defined(PETSC_USE_COMPLEX) 49 #define PetscLogFlops(n) (_TotalFlops += (4*n),0) 50 #else 51 #define PetscLogFlops(n) (_TotalFlops += (n),0) 52 #endif 53 54 #if defined (PETSC_HAVE_MPE) 55 #include "mpe.h" 56 EXTERN int PetscLogMPEBegin(void); 57 EXTERN int PetscLogMPEDump(const char[]); 58 extern PetscTruth UseMPE; 59 #define PETSC_LOG_EVENT_MPE_BEGIN(e) \ 60 if(UseMPE && _stageLog->stageInfo[_stageLog->curStage].eventLog->eventInfo[e].active) \ 61 MPE_Log_event(_stageLog->eventLog->eventInfo[e].mpe_id_begin,0,""); 62 63 #define PETSC_LOG_EVENT_MPE_END(e) \ 64 if(UseMPE && _stageLog->stageInfo[_stageLog->curStage].eventLog->eventInfo[e].active) \ 65 MPE_Log_event(_stageLog->eventLog->eventInfo[e].mpe_id_end,0,""); 66 67 #else 68 #define PETSC_LOG_EVENT_MPE_BEGIN(e) 69 #define PETSC_LOG_EVENT_MPE_END(e) 70 #endif 71 72 EXTERN int (*_PetscLogPLB)(int,int,PetscObject,PetscObject,PetscObject,PetscObject); 73 EXTERN int (*_PetscLogPLE)(int,int,PetscObject,PetscObject,PetscObject,PetscObject); 74 EXTERN int (*_PetscLogPHC)(PetscObject); 75 EXTERN int (*_PetscLogPHD)(PetscObject); 76 77 #define PetscLogObjectParent(p,c) \ 78 if (c) {\ 79 PetscValidHeader((PetscObject)(c),2);\ 80 PetscValidHeader((PetscObject)(p),1);\ 81 ((PetscObject)(c))->parent = (PetscObject)(p);\ 82 ((PetscObject)(c))->parentid = ((PetscObject)p)->id;\ 83 } 84 #define PetscLogObjectParents(p,n,d) {int _i; for (_i=0; _i<n; _i++) PetscLogObjectParent(p,(d)[_i]);} 85 #define PetscLogObjectCreate(h) {if (_PetscLogPHC) (*_PetscLogPHC)((PetscObject)h);} 86 #define PetscLogObjectDestroy(h) {if (_PetscLogPHD) (*_PetscLogPHD)((PetscObject)h);} 87 #define PetscLogObjectMemory(p,m) {PetscValidHeader((PetscObject)p,1);((PetscObject)(p))->mem += (m);} 88 /* Initialization functions */ 89 EXTERN int PetscLogBegin(void); 90 EXTERN int PetscLogAllBegin(void); 91 EXTERN int PetscLogTraceBegin(FILE *); 92 EXTERN int PetscLogActions(PetscTruth); 93 EXTERN int PetscLogObjects(PetscTruth); 94 /* General functions */ 95 EXTERN int PetscLogGetRGBColor(const char*[]); 96 EXTERN int PetscLogDestroy(void); 97 EXTERN int PetscLogSet(int (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject), 98 int (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject)); 99 EXTERN int PetscLogObjectState(PetscObject, const char[], ...) PETSC_PRINTF_FORMAT_CHECK(2,3); 100 /* Output functions */ 101 EXTERN int PetscLogPrintSummary(MPI_Comm, const char[]); 102 EXTERN int PetscLogDump(const char[]); 103 /* Counter functions */ 104 EXTERN int PetscGetFlops(PetscLogDouble *); 105 /* Stage functions */ 106 EXTERN int PetscLogStageRegister(int *, const char[]); 107 EXTERN int PetscLogStagePush(int); 108 EXTERN int PetscLogStagePop(void); 109 EXTERN int PetscLogStageSetActive(int, PetscTruth); 110 EXTERN int PetscLogStageGetActive(int, PetscTruth *); 111 EXTERN int PetscLogStageSetVisible(int, PetscTruth); 112 EXTERN int PetscLogStageGetVisible(int, PetscTruth *); 113 EXTERN int PetscLogStageGetId(const char [], int *); 114 /* Event functions */ 115 EXTERN int PetscLogEventRegister(int *, const char[], int); 116 EXTERN int PetscLogEventActivate(int); 117 EXTERN int PetscLogEventDeactivate(int); 118 EXTERN int PetscLogEventSetActiveAll(int, PetscTruth); 119 EXTERN int PetscLogEventActivateClass(int); 120 EXTERN int PetscLogEventDeactivateClass(int); 121 /* Class functions */ 122 EXTERN int PetscLogClassRegister(int *, const char []); 123 124 /* Global counters */ 125 extern PetscLogDouble irecv_ct, isend_ct, recv_ct, send_ct; 126 extern PetscLogDouble irecv_len, isend_len, recv_len, send_len; 127 extern PetscLogDouble allreduce_ct; 128 extern PetscLogDouble wait_ct, wait_any_ct, wait_all_ct, sum_of_waits_ct; 129 extern int PETSC_DUMMY, PETSC_DUMMY_SIZE; 130 131 /* We must make these structures available if we are to access the event 132 activation flags in the PetscLogEventBegin/End() macros. If we forced a 133 function call each time, we could leave these structures in plog.h 134 */ 135 /* Default log */ 136 typedef struct _StageLog *StageLog; 137 extern StageLog _stageLog; 138 139 /* A simple stack (should replace) */ 140 typedef struct _IntStack *IntStack; 141 142 /* The structures for logging performance */ 143 typedef struct _EventPerfInfo { 144 int id; /* The integer identifying this section */ 145 PetscTruth active; /* The flag to activate logging */ 146 PetscTruth visible; /* The flag to print info in summary */ 147 int depth; /* The nesting depth of the event call */ 148 int count; /* The number of times this section was executed */ 149 PetscLogDouble flops; /* The flops used in this section */ 150 PetscLogDouble time; /* The time taken for this section */ 151 PetscLogDouble numMessages; /* The number of messages in this section */ 152 PetscLogDouble messageLength; /* The total message lengths in this section */ 153 PetscLogDouble numReductions; /* The number of reductions in this section */ 154 } EventPerfInfo; 155 156 typedef struct _ClassPerfInfo { 157 int id; /* The integer identifying this class */ 158 int creations; /* The number of objects of this class created */ 159 int destructions; /* The number of objects of this class destroyed */ 160 PetscLogDouble mem; /* The total memory allocated by objects of this class */ 161 PetscLogDouble descMem; /* The total memory allocated by descendents of these objects */ 162 } ClassPerfInfo; 163 164 /* The structures for logging registration */ 165 typedef struct _ClassRegInfo { 166 char *name; /* The class name */ 167 int cookie; /* The integer identifying this class */ 168 } ClassRegInfo; 169 170 typedef struct _EventRegInfo { 171 char *name; /* The name of this event */ 172 int cookie; /* The class id for this event (should maybe give class ID instead) */ 173 #if defined (PETSC_HAVE_MPE) 174 int mpe_id_begin; /* MPE IDs that define the event */ 175 int mpe_id_end; 176 #endif 177 } EventRegInfo; 178 179 /* The structure for logging events */ 180 typedef int PetscEvent; 181 182 typedef struct _EventRegLog *EventRegLog; 183 struct _EventRegLog { 184 int numEvents; /* The number of registered events */ 185 int maxEvents; /* The maximum number of events */ 186 EventRegInfo *eventInfo; /* The registration information for each event */ 187 }; 188 189 typedef struct _EventPerfLog *EventPerfLog; 190 struct _EventPerfLog { 191 int numEvents; /* The number of logging events */ 192 int maxEvents; /* The maximum number of events */ 193 EventPerfInfo *eventInfo; /* The performance information for each event */ 194 }; 195 196 /* The structure for logging class information */ 197 typedef struct _ClassRegLog *ClassRegLog; 198 struct _ClassRegLog { 199 int numClasses; /* The number of classes registered */ 200 int maxClasses; /* The maximum number of classes */ 201 ClassRegInfo *classInfo; /* The structure for class information (cookies are monotonicly increasing) */ 202 }; 203 204 typedef struct _ClassPerfLog *ClassPerfLog; 205 struct _ClassPerfLog { 206 int numClasses; /* The number of logging classes */ 207 int maxClasses; /* The maximum number of classes */ 208 ClassPerfInfo *classInfo; /* The structure for class information (cookies are monotonicly increasing) */ 209 }; 210 211 /* The structures for logging in stages */ 212 typedef struct _StageInfo { 213 char *name; /* The stage name */ 214 PetscTruth used; /* The stage was pushed on this processor */ 215 EventPerfInfo perfInfo; /* The stage performance information */ 216 EventPerfLog eventLog; /* The event information for this stage */ 217 ClassPerfLog classLog; /* The class information for this stage */ 218 } StageInfo; 219 220 struct _StageLog { 221 /* Size information */ 222 int numStages; /* The number of registered stages */ 223 int maxStages; /* The maximum number of stages */ 224 /* Runtime information */ 225 IntStack stack; /* The stack for active stages */ 226 int curStage; /* The current stage (only used in macros so we don't call StackTop) */ 227 /* Stage specific information */ 228 StageInfo *stageInfo; /* The information for each stage */ 229 EventRegLog eventLog; /* The registered events */ 230 ClassRegLog classLog; /* The registered classes */ 231 }; 232 233 #define PetscLogEventBarrierBegin(e,o1,o2,o3,o4,cm) 0; \ 234 {\ 235 int _2_ierr;\ 236 if (_PetscLogPLB && \ 237 _stageLog->stageInfo[_stageLog->curStage].perfInfo.active && \ 238 _stageLog->stageInfo[_stageLog->curStage].eventLog->eventInfo[e].active) {\ 239 _2_ierr = PetscLogEventBegin((e),o1,o2,o3,o4);CHKERRQ(_2_ierr);\ 240 _2_ierr = MPI_Barrier(cm);CHKERRQ(_2_ierr);\ 241 _2_ierr = PetscLogEventEnd((e),o1,o2,o3,o4);CHKERRQ(_2_ierr);\ 242 }\ 243 _2_ierr = PetscLogEventBegin((e)+1,o1,o2,o3,o4);CHKERRQ(_2_ierr);\ 244 } 245 246 #define PetscLogEventBegin(e,o1,o2,o3,o4) 0; \ 247 {\ 248 if (_PetscLogPLB && \ 249 _stageLog->stageInfo[_stageLog->curStage].perfInfo.active && \ 250 _stageLog->stageInfo[_stageLog->curStage].eventLog->eventInfo[e].active) {\ 251 (*_PetscLogPLB)((e),0,(PetscObject)(o1),(PetscObject)(o2),(PetscObject)(o3),(PetscObject)(o4));\ 252 }\ 253 PETSC_LOG_EVENT_MPE_BEGIN(e); \ 254 } 255 256 #define PetscLogEventBarrierEnd(e,o1,o2,o3,o4,cm) PetscLogEventEnd(e+1,o1,o2,o3,o4) 257 258 #define PetscLogEventEnd(e,o1,o2,o3,o4) 0; \ 259 {\ 260 if (_PetscLogPLE && \ 261 _stageLog->stageInfo[_stageLog->curStage].perfInfo.active && \ 262 _stageLog->stageInfo[_stageLog->curStage].eventLog->eventInfo[e].active) {\ 263 (*_PetscLogPLE)((e),0,(PetscObject)(o1),(PetscObject)(o2),(PetscObject)(o3),(PetscObject)(o4));\ 264 }\ 265 PETSC_LOG_EVENT_MPE_END(e); \ 266 } 267 268 /* Creation and destruction functions */ 269 EXTERN int StageLogCreate(StageLog *); 270 EXTERN int StageLogDestroy(StageLog); 271 /* Registration functions */ 272 EXTERN int StageLogRegister(StageLog, const char [], int *); 273 /* Runtime functions */ 274 EXTERN int PetscLogGetStageLog(StageLog *); 275 EXTERN int StageLogPush(StageLog, int); 276 EXTERN int StageLogPop(StageLog); 277 EXTERN int StageLogGetCurrent(StageLog, int *); 278 EXTERN int StageLogSetActive(StageLog, int, PetscTruth); 279 EXTERN int StageLogGetActive(StageLog, int, PetscTruth *); 280 EXTERN int StageLogSetVisible(StageLog, int, PetscTruth); 281 EXTERN int StageLogGetVisible(StageLog, int, PetscTruth *); 282 EXTERN int StageLogGetStage(StageLog, const char [], int *); 283 284 /* 285 This does not work for MPI-Uni because our src/mpiuni/mpi.h file 286 uses macros to defined the MPI operations. 287 288 It does not work correctly from HP-UX because it processes the 289 macros in a way that sometimes it double counts, hence 290 PETSC_HAVE_BROKEN_RECURSIVE_MACRO 291 292 It does not work with Windows NT because winmpich lacks MPI_Type_size() 293 */ 294 #if !defined(_petsc_mpi_uni) && !defined(PETSC_HAVE_BROKEN_RECURSIVE_MACRO) && !defined (PETSC_HAVE_MPI_MISSING_TYPESIZE) 295 /* 296 Logging of MPI activities 297 */ 298 299 #define TypeSize(buff,count,type) \ 300 (\ 301 MPI_Type_size(type,&PETSC_DUMMY_SIZE),buff += ((PetscLogDouble) ((count)*PETSC_DUMMY_SIZE))\ 302 ) 303 304 #define MPI_Irecv(buf,count, datatype,source,tag,comm,request) \ 305 (\ 306 PETSC_DUMMY = MPI_Irecv(buf,count, datatype,source,tag,comm,request),\ 307 irecv_ct++,TypeSize(irecv_len,count,datatype),PETSC_DUMMY\ 308 ) 309 310 #define MPI_Isend(buf,count, datatype,dest,tag,comm,request) \ 311 (\ 312 PETSC_DUMMY = MPI_Isend(buf,count, datatype,dest,tag,comm,request),\ 313 isend_ct++, TypeSize(isend_len,count,datatype),PETSC_DUMMY\ 314 ) 315 316 #define MPI_Startall_irecv(count,number,requests) \ 317 (\ 318 PETSC_DUMMY = MPI_Startall(number,requests),\ 319 irecv_ct += (PetscLogDouble)(number),irecv_len += ((PetscLogDouble) ((count)*sizeof(PetscScalar))),PETSC_DUMMY\ 320 ) 321 322 #define MPI_Startall_isend(count,number,requests) \ 323 (\ 324 PETSC_DUMMY = MPI_Startall(number,requests),\ 325 isend_ct += (PetscLogDouble)(number),isend_len += ((PetscLogDouble) ((count)*sizeof(PetscScalar))),PETSC_DUMMY\ 326 ) 327 328 #define MPI_Start_isend(count, requests) \ 329 (\ 330 PETSC_DUMMY = MPI_Start(requests),\ 331 isend_ct++,isend_len += ((PetscLogDouble) ((count)*sizeof(PetscScalar))),PETSC_DUMMY\ 332 ) 333 334 #define MPI_Recv(buf,count, datatype,source,tag,comm,status) \ 335 (\ 336 PETSC_DUMMY = MPI_Recv(buf,count, datatype,source,tag,comm,status),\ 337 recv_ct++,TypeSize(recv_len,count,datatype),PETSC_DUMMY\ 338 ) 339 340 #define MPI_Send(buf,count, datatype,dest,tag,comm) \ 341 (\ 342 PETSC_DUMMY = MPI_Send(buf,count, datatype,dest,tag,comm),\ 343 send_ct++, TypeSize(send_len,count,datatype),PETSC_DUMMY\ 344 ) 345 346 #define MPI_Wait(request,status) \ 347 (\ 348 wait_ct++,sum_of_waits_ct++,\ 349 MPI_Wait(request,status)\ 350 ) 351 352 #define MPI_Waitany(a,b,c,d) \ 353 (\ 354 wait_any_ct++,sum_of_waits_ct++,\ 355 MPI_Waitany(a,b,c,d)\ 356 ) 357 358 #define MPI_Waitall(count,array_of_requests,array_of_statuses) \ 359 (\ 360 wait_all_ct++,sum_of_waits_ct += (PetscLogDouble) (count),\ 361 MPI_Waitall(count,array_of_requests,array_of_statuses)\ 362 ) 363 364 #define MPI_Allreduce(sendbuf, recvbuf,count,datatype,op,comm) \ 365 (\ 366 allreduce_ct++,MPI_Allreduce(sendbuf,recvbuf,count,datatype,op,comm)\ 367 ) 368 369 #else 370 371 #define MPI_Startall_irecv(count,number,requests) \ 372 (\ 373 MPI_Startall(number,requests)\ 374 ) 375 376 #define MPI_Startall_isend(count,number,requests) \ 377 (\ 378 MPI_Startall(number,requests)\ 379 ) 380 381 #define MPI_Start_isend(count, requests) \ 382 (\ 383 MPI_Start(requests)\ 384 ) 385 386 #endif /* !_petsc_mpi_uni && ! PETSC_HAVE_BROKEN_RECURSIVE_MACRO */ 387 388 #else /* ---Logging is turned off --------------------------------------------*/ 389 390 #define PetscLogFlops(n) 0 391 392 /* 393 With logging turned off, then MPE has to be turned off 394 */ 395 #define PetscLogMPEBegin() 0 396 #define PetscLogMPEDump(a) 0 397 398 #define PetscLogEventActivate(a) 0 399 #define PetscLogEventDeactivate(a) 0 400 401 #define PetscLogEventActivateClass(a) 0 402 #define PetscLogEventDeactivateClass(a) 0 403 404 #define _PetscLogPLB 0 405 #define _PetscLogPLE 0 406 #define _PetscLogPHC 0 407 #define _PetscLogPHD 0 408 #define PetscGetFlops(a) (*(a) = 0.0,0) 409 #define PetscLogEventBegin(e,o1,o2,o3,o4) 0 410 #define PetscLogEventEnd(e,o1,o2,o3,o4) 0 411 #define PetscLogEventBarrierBegin(e,o1,o2,o3,o4,cm) 0 412 #define PetscLogEventBarrierEnd(e,o1,o2,o3,o4,cm) 0 413 #define PetscLogObjectParent(p,c) 414 #define PetscLogObjectParents(p,n,c) 415 #define PetscLogObjectCreate(h) 416 #define PetscLogObjectDestroy(h) 417 #define PetscLogObjectMemory(p,m) 418 #define PetscLogDestroy() 0 419 #define PetscLogStagePush(a) 0 420 #define PetscLogStagePop() 0 421 #define PetscLogStageRegister(a,b) 0 422 #define PetscLogStagePrint(a,flg) 0 423 #define PetscLogPrintSummary(comm,file) 0 424 #define PetscLogBegin() 0 425 #define PetscLogTraceBegin(file) 0 426 #define PetscLogSet(lb,le) 0 427 #define PetscLogAllBegin() 0 428 #define PetscLogDump(c) 0 429 #define PetscLogEventRegister(a,b,c) 0 430 #define PetscLogObjects(a) 0 431 #define PetscLogActions(a) 0 432 EXTERN int PetscLogObjectState(PetscObject,const char[],...) PETSC_PRINTF_FORMAT_CHECK(2,3); 433 434 /* If PETSC_USE_LOG is NOT defined, these still need to be! */ 435 #define MPI_Startall_irecv(count,number,requests) MPI_Startall(number,requests) 436 #define MPI_Startall_isend(count,number,requests) MPI_Startall(number,requests) 437 #define MPI_Start_isend(count,requests) MPI_Start(requests) 438 439 /* Creation and destruction functions */ 440 #define StageLogCreate(stageLog) 0 441 #define StageLogDestroy(stageLog) 0 442 /* Registration functions */ 443 #define StageLogRegister(stageLog, name, stage) 0 444 /* Runtime functions */ 445 #define PetscLogGetStageLog(stageLog) 0 446 #define StageLogPush(stageLog, stage) 0 447 #define StageLogPop(stageLog) 0 448 #define StageLogGetCurrent(stageLog, stage) 0 449 #define StageLogSetActive(stageLog, stage, active) 0 450 #define StageLogGetActive(stageLog, stage, active) 0 451 #define StageLogSetVisible(stageLog, stage, visible) 0 452 #define StageLogGetVisible(stageLog, stage, visible) 0 453 #define StageLogGetStage(stageLog, name, stage) 0 454 455 #endif /* PETSC_USE_LOG */ 456 457 extern PetscTruth PetscPreLoadingUsed; /* true if we are or have done preloading */ 458 extern PetscTruth PetscPreLoadingOn; /* true if we are currently in a preloading calculation */ 459 460 #define PreLoadBegin(flag,name) \ 461 {\ 462 PetscTruth PreLoading = flag;\ 463 int PreLoadMax,PreLoadIt,_stageNum,_3_ierr;\ 464 _3_ierr = PetscOptionsGetLogical(PETSC_NULL,"-preload",&PreLoading,PETSC_NULL);CHKERRQ(_3_ierr);\ 465 PreLoadMax = (int)(PreLoading);\ 466 PetscPreLoadingUsed = PreLoading ? PETSC_TRUE : PetscPreLoadingUsed;\ 467 for (PreLoadIt=0; PreLoadIt<=PreLoadMax; PreLoadIt++) {\ 468 PetscPreLoadingOn = PreLoading;\ 469 _3_ierr = PetscBarrier(PETSC_NULL);CHKERRQ(_3_ierr);\ 470 if (PreLoadIt>0) {\ 471 _3_ierr = PetscLogStageGetId(name,&_stageNum);CHKERRQ(_3_ierr);\ 472 } else {\ 473 _3_ierr = PetscLogStageRegister(&_stageNum,name);CHKERRQ(_3_ierr);\ 474 }\ 475 _3_ierr = PetscLogStageSetActive(_stageNum,(PetscTruth)(!PreLoadMax || PreLoadIt));\ 476 _3_ierr = PetscLogStagePush(_stageNum);CHKERRQ(_3_ierr); 477 478 #define PreLoadEnd() \ 479 _3_ierr = PetscLogStagePop();CHKERRQ(_3_ierr);\ 480 PreLoading = PETSC_FALSE;\ 481 }\ 482 } 483 484 #define PreLoadStage(name) \ 485 _3_ierr = PetscLogStagePop();CHKERRQ(_3_ierr);\ 486 if (PreLoadIt>0) {\ 487 _3_ierr = PetscLogStageGetId(name,&_stageNum);CHKERRQ(_3_ierr);\ 488 } else {\ 489 _3_ierr = PetscLogStageRegister(&_stageNum,name);CHKERRQ(_3_ierr);\ 490 }\ 491 _3_ierr = PetscLogStageSetActive(_stageNum,(PetscTruth)(!PreLoadMax || PreLoadIt));\ 492 _3_ierr = PetscLogStagePush(_stageNum);CHKERRQ(_3_ierr); 493 494 PETSC_EXTERN_CXX_END 495 #endif 496