1 /* 2 Defines the vector component of PETSc. Vectors generally represent 3 degrees of freedom for finite element/finite difference functions 4 on a grid. They have more mathematical structure then simple arrays. 5 */ 6 7 #ifndef __PETSCVEC_H 8 #define __PETSCVEC_H 9 #include "petscis.h" 10 #include "petscsys.h" 11 PETSC_EXTERN_CXX_BEGIN 12 13 /*S 14 PetscMap - Abstract PETSc object that defines the layout of vector and 15 matrices across processors 16 17 Level: advanced 18 19 Notes: 20 Does not play a role in the PETSc design, can be ignored 21 22 Concepts: parallel decomposition 23 24 .seealso: PetscMapCreateMPI() 25 S*/ 26 typedef struct _p_PetscMap* PetscMap; 27 28 #define MAP_SEQ "seq" 29 #define MAP_MPI "mpi" 30 #define PetscMapType char* 31 32 /* Logging support */ 33 extern int MAP_COOKIE; 34 35 EXTERN int PetscMapCreate(MPI_Comm,PetscMap*); 36 EXTERN int PetscMapCreateMPI(MPI_Comm,int,int,PetscMap*); 37 EXTERN int PetscMapSetFromOptions(PetscMap); 38 EXTERN int PetscMapPrintHelp(PetscMap); 39 EXTERN int PetscMapDestroy(PetscMap); 40 41 EXTERN int PetscMapSetLocalSize(PetscMap,int); 42 EXTERN int PetscMapGetLocalSize(PetscMap,int *); 43 EXTERN int PetscMapSetSize(PetscMap,int); 44 EXTERN int PetscMapGetSize(PetscMap,int *); 45 EXTERN int PetscMapGetLocalRange(PetscMap,int *,int *); 46 EXTERN int PetscMapGetGlobalRange(PetscMap,int *[]); 47 48 /* Dynamic creation and loading functions */ 49 extern PetscFList PetscMapList; 50 extern PetscTruth PetscMapRegisterAllCalled; 51 EXTERN int PetscMapSetType(PetscMap, const PetscMapType); 52 EXTERN int PetscMapGetType(PetscMap, PetscMapType *); 53 EXTERN int PetscMapRegister(const char[],const char[],const char[],int(*)(PetscMap)); 54 EXTERN int PetscMapRegisterAll(const char []); 55 EXTERN int PetscMapRegisterDestroy(void); 56 #if defined(PETSC_USE_DYNAMIC_LIBRARIES) 57 #define PetscMapRegisterDynamic(a,b,c,d) PetscMapRegister(a,b,c,0) 58 #else 59 #define PetscMapRegisterDynamic(a,b,c,d) PetscMapRegister(a,b,c,d) 60 #endif 61 62 /*S 63 Vec - Abstract PETSc vector object 64 65 Level: beginner 66 67 Concepts: field variables, unknowns, arrays 68 69 .seealso: VecCreate(), VecType, VecSetType() 70 S*/ 71 typedef struct _p_Vec* Vec; 72 73 /*S 74 VecScatter - Object used to manage communication of data 75 between vectors in parallel. Manages both scatters and gathers 76 77 Level: beginner 78 79 Concepts: scatter 80 81 .seealso: VecScatterCreate(), VecScatterBegin(), VecScatterEnd() 82 S*/ 83 typedef struct _p_VecScatter* VecScatter; 84 85 /*E 86 VecType - String with the name of a PETSc vector or the creation function 87 with an optional dynamic library name, for example 88 http://www.mcs.anl.gov/petsc/lib.a:myveccreate() 89 90 Level: beginner 91 92 .seealso: VecSetType(), Vec 93 E*/ 94 #define VECSEQ "seq" 95 #define VECMPI "mpi" 96 #define VECFETI "feti" 97 #define VECSHARED "shared" 98 #define VECESI "esi" 99 #define VECPETSCESI "petscesi" 100 #define VecType char* 101 102 /* Logging support */ 103 #define VEC_FILE_COOKIE 1211214 104 extern int VEC_COOKIE; 105 extern int VEC_SCATTER_COOKIE; 106 extern int VEC_View, VEC_Max, VEC_Min, VEC_DotBarrier, VEC_Dot, VEC_MDotBarrier, VEC_MDot, VEC_TDot, VEC_MTDot, VEC_NormBarrier; 107 extern int VEC_Norm, VEC_Normalize, VEC_Scale, VEC_Copy, VEC_Set, VEC_AXPY, VEC_AYPX, VEC_WAXPY, VEC_MAXPY, VEC_Swap, VEC_AssemblyBegin; 108 extern int VEC_AssemblyEnd, VEC_PointwiseMult, VEC_SetValues, VEC_Load, VEC_ScatterBarrier, VEC_ScatterBegin, VEC_ScatterEnd; 109 extern int VEC_SetRandom, VEC_ReduceArithmetic, VEC_ReduceBarrier, VEC_ReduceCommunication; 110 111 EXTERN int VecInitializePackage(char *); 112 113 EXTERN int VecCreate(MPI_Comm,Vec *); 114 EXTERN int VecCreateSeq(MPI_Comm,int,Vec*); 115 EXTERN int VecCreateMPI(MPI_Comm,int,int,Vec*); 116 EXTERN int VecCreateSeqWithArray(MPI_Comm,int,const PetscScalar[],Vec*); 117 EXTERN int VecCreateMPIWithArray(MPI_Comm,int,int,const PetscScalar[],Vec*); 118 EXTERN int VecCreateShared(MPI_Comm,int,int,Vec*); 119 EXTERN int VecSetFromOptions(Vec); 120 EXTERN int VecPrintHelp(Vec); 121 EXTERN int VecDestroy(Vec); 122 123 EXTERN int VecSetSizes(Vec,int,int); 124 125 EXTERN int VecDot(Vec,Vec,PetscScalar*); 126 EXTERN int VecTDot(Vec,Vec,PetscScalar*); 127 EXTERN int VecMDot(int,Vec,const Vec[],PetscScalar*); 128 EXTERN int VecMTDot(int,Vec,const Vec[],PetscScalar*); 129 130 /*E 131 NormType - determines what type of norm to compute 132 133 Level: beginner 134 135 .seealso: VecNorm(), VecNormBegin(), VecNormEnd(), MatNorm() 136 E*/ 137 typedef enum {NORM_1=1,NORM_2=2,NORM_FROBENIUS=3,NORM_INFINITY=4,NORM_1_AND_2=5} NormType; 138 #define NORM_MAX NORM_INFINITY 139 140 /*MC 141 NORM_1 - the one norm, ||v|| = sum_i | v_i |. ||A|| = max_j || v_*j ||, maximum column sum 142 143 Level: beginner 144 145 .seealso: NormType, MatNorm(), VecNorm(), VecNormBegin(), VecNormEnd(), NORM_2, NORM_FROBENIUS, 146 NORM_INFINITY, NORM_1_AND_2 147 148 M*/ 149 150 /*MC 151 NORM_2 - the two norm, ||v|| = sqrt(sum_i (v_i)^2) (vectors only) 152 153 Level: beginner 154 155 .seealso: NormType, MatNorm(), VecNorm(), VecNormBegin(), VecNormEnd(), NORM_1, NORM_FROBENIUS, 156 NORM_INFINITY, NORM_1_AND_2 157 158 M*/ 159 160 /*MC 161 NORM_FROBENIUS - ||A|| = sqrt(sum_ij (A_ij)^2), same as NORM_2 for vectors 162 163 Level: beginner 164 165 .seealso: NormType, MatNorm(), VecNorm(), VecNormBegin(), VecNormEnd(), NORM_1, NORM_2, 166 NORM_INFINITY, NORM_1_AND_2 167 168 M*/ 169 170 /*MC 171 NORM_INFINITY - ||v|| = max_i |v_i|. ||A|| = max_i || v_i* ||, maximum row sum 172 173 Level: beginner 174 175 .seealso: NormType, MatNorm(), VecNorm(), VecNormBegin(), VecNormEnd(), NORM_1, NORM_2, 176 NORM_FROBINIUS, NORM_1_AND_2 177 178 M*/ 179 180 /*MC 181 NORM_1_AND_2 - computes both the 1 and 2 norm of a vector 182 183 Level: beginner 184 185 .seealso: NormType, MatNorm(), VecNorm(), VecNormBegin(), VecNormEnd(), NORM_1, NORM_2, 186 NORM_FROBINIUS, NORM_INFINITY 187 188 M*/ 189 190 /*MC 191 NORM_MAX - see NORM_INFINITY 192 193 Level: beginner 194 195 M*/ 196 197 EXTERN int VecNorm(Vec,NormType,PetscReal *); 198 EXTERN int VecNormComposedDataID(NormType,int*); 199 EXTERN int VecNormalize(Vec,PetscReal *); 200 EXTERN int VecSum(Vec,PetscScalar*); 201 EXTERN int VecMax(Vec,int*,PetscReal *); 202 EXTERN int VecMin(Vec,int*,PetscReal *); 203 EXTERN int VecScale(const PetscScalar *a,Vec v); 204 EXTERN int VecCopy(Vec,Vec); 205 EXTERN int VecSetRandom(PetscRandom,Vec); 206 EXTERN int VecSet(const PetscScalar*,Vec); 207 EXTERN int VecSwap(Vec,Vec); 208 EXTERN int VecAXPY(const PetscScalar*,Vec,Vec); 209 EXTERN int VecAXPBY(const PetscScalar*,const PetscScalar *,Vec,Vec); 210 EXTERN int VecMAXPY(int,const PetscScalar*,Vec,Vec*); 211 EXTERN int VecAYPX(const PetscScalar*,Vec,Vec); 212 EXTERN int VecWAXPY(const PetscScalar*,Vec,Vec,Vec); 213 EXTERN int VecPointwiseMult(Vec,Vec,Vec); 214 EXTERN int VecPointwiseDivide(Vec,Vec,Vec); 215 EXTERN int VecMaxPointwiseDivide(Vec,Vec,PetscReal*); 216 EXTERN int VecShift(const PetscScalar*,Vec); 217 EXTERN int VecReciprocal(Vec); 218 EXTERN int VecPermute(Vec, IS, PetscTruth); 219 EXTERN int VecSqrt(Vec); 220 EXTERN int VecAbs(Vec); 221 EXTERN int VecDuplicate(Vec,Vec*); 222 EXTERN int VecDuplicateVecs(Vec,int,Vec*[]); 223 EXTERN int VecDestroyVecs(const Vec[],int); 224 EXTERN int VecGetPetscMap(Vec,PetscMap*); 225 226 EXTERN int VecStrideNormAll(Vec,NormType,PetscReal*); 227 EXTERN int VecStrideMaxAll(Vec,int *,PetscReal *); 228 EXTERN int VecStrideMinAll(Vec,int *,PetscReal *); 229 EXTERN int VecStrideScaleAll(Vec,PetscScalar*); 230 231 EXTERN int VecStrideNorm(Vec,int,NormType,PetscReal*); 232 EXTERN int VecStrideMax(Vec,int,int *,PetscReal *); 233 EXTERN int VecStrideMin(Vec,int,int *,PetscReal *); 234 EXTERN int VecStrideScale(Vec,int,PetscScalar*); 235 EXTERN int VecStrideGather(Vec,int,Vec,InsertMode); 236 EXTERN int VecStrideScatter(Vec,int,Vec,InsertMode); 237 EXTERN int VecStrideGatherAll(Vec,Vec*,InsertMode); 238 EXTERN int VecStrideScatterAll(Vec*,Vec,InsertMode); 239 240 EXTERN int VecSetValues(Vec,int,const int[],const PetscScalar[],InsertMode); 241 EXTERN int VecAssemblyBegin(Vec); 242 EXTERN int VecAssemblyEnd(Vec); 243 EXTERN int VecStashSetInitialSize(Vec,int,int); 244 EXTERN int VecStashView(Vec,PetscViewer); 245 EXTERN int VecStashGetInfo(Vec,int*,int*,int*,int*); 246 247 extern int VecSetValue_Row; 248 extern PetscScalar VecSetValue_Value; 249 /*MC 250 VecSetValue - Set a single entry into a vector. 251 252 Synopsis: 253 int VecSetValue(Vec v,int row,PetscScalar value, InsertMode mode); 254 255 Not Collective 256 257 Input Parameters: 258 + v - the vector 259 . row - the row location of the entry 260 . value - the value to insert 261 - mode - either INSERT_VALUES or ADD_VALUES 262 263 Notes: 264 For efficiency one should use VecSetValues() and set several or 265 many values simultaneously if possible. 266 267 These values may be cached, so VecAssemblyBegin() and VecAssemblyEnd() 268 MUST be called after all calls to VecSetValues() have been completed. 269 270 VecSetValues() uses 0-based indices in Fortran as well as in C. 271 272 Level: beginner 273 274 .seealso: VecSetValues(), VecAssemblyBegin(), VecAssemblyEnd(), VecSetValuesBlockedLocal(), VecSetValueLocal() 275 M*/ 276 #define VecSetValue(v,i,va,mode) (VecSetValue_Row = i, VecSetValue_Value = va, VecSetValues(v,1,&VecSetValue_Row,&VecSetValue_Value,mode)) 277 278 /*MC 279 VecSetValueLocal - Set a single entry into a vector using the local numbering 280 281 Synopsis: 282 int VecSetValueLocal(Vec v,int row,PetscScalar value, InsertMode mode); 283 284 Not Collective 285 286 Input Parameters: 287 + v - the vector 288 . row - the row location of the entry 289 . value - the value to insert 290 - mode - either INSERT_VALUES or ADD_VALUES 291 292 Notes: 293 For efficiency one should use VecSetValues() and set several or 294 many values simultaneously if possible. 295 296 These values may be cached, so VecAssemblyBegin() and VecAssemblyEnd() 297 MUST be called after all calls to VecSetValues() have been completed. 298 299 VecSetValues() uses 0-based indices in Fortran as well as in C. 300 301 Level: beginner 302 303 .seealso: VecSetValues(), VecAssemblyBegin(), VecAssemblyEnd(), VecSetValuesBlockedLocal(), VecSetValue() 304 M*/ 305 #define VecSetValueLocal(v,i,va,mode) (VecSetValue_Row = i,VecSetValue_Value = va,VecSetValuesLocal(v,1,&VecSetValue_Row,&VecSetValue_Value,mode)) 306 307 EXTERN int VecSetBlockSize(Vec,int); 308 EXTERN int VecGetBlockSize(Vec,int*); 309 EXTERN int VecSetValuesBlocked(Vec,int,const int[],const PetscScalar[],InsertMode); 310 311 /* Dynamic creation and loading functions */ 312 extern PetscFList VecList; 313 extern PetscTruth VecRegisterAllCalled; 314 EXTERN int VecSetType(Vec, const VecType); 315 EXTERN int VecGetType(Vec, VecType *); 316 EXTERN int VecRegister(const char[],const char[],const char[],int(*)(Vec)); 317 EXTERN int VecRegisterAll(const char []); 318 EXTERN int VecRegisterDestroy(void); 319 320 /*MC 321 VecRegisterDynamic - Adds a new vector component implementation 322 323 Synopsis: 324 VecRegisterDynamic(char *name, char *path, char *func_name, int (*create_func)(Vec)) 325 326 Not Collective 327 328 Input Parameters: 329 + name - The name of a new user-defined creation routine 330 . path - The path (either absolute or relative) of the library containing this routine 331 . func_name - The name of routine to create method context 332 - create_func - The creation routine itself 333 334 Notes: 335 VecRegisterDynamic() may be called multiple times to add several user-defined vectors 336 337 If dynamic libraries are used, then the fourth input argument (routine_create) is ignored. 338 339 Sample usage: 340 .vb 341 VecRegisterDynamic("my_vec","/home/username/my_lib/lib/libO/solaris/libmy.a", "MyVectorCreate", MyVectorCreate); 342 .ve 343 344 Then, your vector type can be chosen with the procedural interface via 345 .vb 346 VecCreate(MPI_Comm, Vec *); 347 VecSetType(Vec,"my_vector_name"); 348 .ve 349 or at runtime via the option 350 .vb 351 -vec_type my_vector_name 352 .ve 353 354 Notes: $PETSC_ARCH and $BOPT occuring in pathname will be replaced with appropriate values. 355 If your function is not being put into a shared library then use VecRegister() instead 356 357 Level: advanced 358 359 .keywords: Vec, register 360 .seealso: VecRegisterAll(), VecRegisterDestroy(), VecRegister() 361 M*/ 362 #if defined(PETSC_USE_DYNAMIC_LIBRARIES) 363 #define VecRegisterDynamic(a,b,c,d) VecRegister(a,b,c,0) 364 #else 365 #define VecRegisterDynamic(a,b,c,d) VecRegister(a,b,c,d) 366 #endif 367 368 369 EXTERN int VecScatterCreate(Vec,IS,Vec,IS,VecScatter *); 370 EXTERN int VecScatterPostRecvs(Vec,Vec,InsertMode,ScatterMode,VecScatter); 371 EXTERN int VecScatterBegin(Vec,Vec,InsertMode,ScatterMode,VecScatter); 372 EXTERN int VecScatterEnd(Vec,Vec,InsertMode,ScatterMode,VecScatter); 373 EXTERN int VecScatterDestroy(VecScatter); 374 EXTERN int VecScatterCopy(VecScatter,VecScatter *); 375 EXTERN int VecScatterView(VecScatter,PetscViewer); 376 EXTERN int VecScatterRemap(VecScatter,int *,int*); 377 378 typedef enum {PIPELINE_DOWN=0,PIPELINE_UP=1} PipelineDirection; 379 typedef enum {PIPELINE_NONE=1,PIPELINE_SEQUENTIAL=2, 380 PIPELINE_REDBLACK=3,PIPELINE_MULTICOLOR=4} PipelineType; 381 382 typedef struct _p_VecPipeline* VecPipeline; 383 384 EXTERN int VecPipelineCreate(MPI_Comm,Vec,IS,Vec,IS,VecPipeline *); 385 EXTERN int VecPipelineSetType(VecPipeline,PipelineType,PetscObject); 386 EXTERN int VecPipelineSetup(VecPipeline); 387 EXTERN int VecPipelineBegin(Vec,Vec,InsertMode,ScatterMode,PipelineDirection,VecPipeline); 388 EXTERN int VecPipelineEnd(Vec,Vec,InsertMode,ScatterMode,PipelineDirection,VecPipeline); 389 EXTERN int VecPipelineView(VecPipeline,PetscViewer); 390 EXTERN int VecPipelineDestroy(VecPipeline); 391 392 EXTERN int VecGetArray_Private(Vec,PetscScalar*[]); 393 EXTERN int VecRestoreArray_Private(Vec,PetscScalar*[]); 394 EXTERN int VecGetArray4d(Vec,int,int,int,int,int,int,int,int,PetscScalar****[]); 395 EXTERN int VecRestoreArray4d(Vec,int,int,int,int,int,int,int,int,PetscScalar****[]); 396 EXTERN int VecGetArray3d(Vec,int,int,int,int,int,int,PetscScalar***[]); 397 EXTERN int VecRestoreArray3d(Vec,int,int,int,int,int,int,PetscScalar***[]); 398 EXTERN int VecGetArray2d(Vec,int,int,int,int,PetscScalar**[]); 399 EXTERN int VecRestoreArray2d(Vec,int,int,int,int,PetscScalar**[]); 400 EXTERN int VecGetArray1d(Vec,int,int,PetscScalar *[]); 401 EXTERN int VecRestoreArray1d(Vec,int,int,PetscScalar *[]); 402 403 EXTERN int VecPlaceArray(Vec,const PetscScalar[]); 404 EXTERN int VecResetArray(Vec); 405 EXTERN int VecReplaceArray(Vec,const PetscScalar[]); 406 EXTERN int VecGetArrays(const Vec[],int,PetscScalar**[]); 407 EXTERN int VecRestoreArrays(const Vec[],int,PetscScalar**[]); 408 409 EXTERN int VecValid(Vec,PetscTruth*); 410 EXTERN int VecView(Vec,PetscViewer); 411 EXTERN int VecViewFromOptions(Vec, char *); 412 EXTERN int VecEqual(Vec,Vec,PetscTruth*); 413 EXTERN int VecLoad(PetscViewer,const VecType,Vec*); 414 EXTERN int VecLoadIntoVector(PetscViewer,Vec); 415 416 EXTERN int VecGetSize(Vec,int*); 417 EXTERN int VecGetLocalSize(Vec,int*); 418 EXTERN int VecGetOwnershipRange(Vec,int*,int*); 419 420 EXTERN int VecSetLocalToGlobalMapping(Vec,ISLocalToGlobalMapping); 421 EXTERN int VecSetValuesLocal(Vec,int,const int[],const PetscScalar[],InsertMode); 422 EXTERN int VecSetLocalToGlobalMappingBlock(Vec,ISLocalToGlobalMapping); 423 EXTERN int VecSetValuesBlockedLocal(Vec,int,const int[],const PetscScalar[],InsertMode); 424 425 EXTERN int VecDotBegin(Vec,Vec,PetscScalar *); 426 EXTERN int VecDotEnd(Vec,Vec,PetscScalar *); 427 EXTERN int VecTDotBegin(Vec,Vec,PetscScalar *); 428 EXTERN int VecTDotEnd(Vec,Vec,PetscScalar *); 429 EXTERN int VecNormBegin(Vec,NormType,PetscReal *); 430 EXTERN int VecNormEnd(Vec,NormType,PetscReal *); 431 432 typedef enum {VEC_IGNORE_OFF_PROC_ENTRIES,VEC_TREAT_OFF_PROC_ENTRIES} VecOption; 433 EXTERN int VecSetOption(Vec,VecOption); 434 435 /* 436 Expose VecGetArray()/VecRestoreArray() to users. Allows this to work without any function 437 call overhead on any 'native' Vecs. 438 */ 439 #include "vecimpl.h" 440 441 EXTERN int VecContourScale(Vec,PetscReal,PetscReal); 442 443 /* 444 These numbers need to match the entries in 445 the function table in vecimpl.h 446 */ 447 typedef enum { VECOP_VIEW = 32, 448 VECOP_LOADINTOVECTOR = 38 449 } VecOperation; 450 EXTERN int VecSetOperation(Vec,VecOperation,void(*)(void)); 451 452 /* 453 Routines for dealing with ghosted vectors: 454 vectors with ghost elements at the end of the array. 455 */ 456 EXTERN int VecCreateGhost(MPI_Comm,int,int,int,const int[],Vec*); 457 EXTERN int VecCreateGhostWithArray(MPI_Comm,int,int,int,const int[],const PetscScalar[],Vec*); 458 EXTERN int VecCreateGhostBlock(MPI_Comm,int,int,int,int,const int[],Vec*); 459 EXTERN int VecCreateGhostBlockWithArray(MPI_Comm,int,int,int,int,const int[],const PetscScalar[],Vec*); 460 EXTERN int VecGhostGetLocalForm(Vec,Vec*); 461 EXTERN int VecGhostRestoreLocalForm(Vec,Vec*); 462 EXTERN int VecGhostUpdateBegin(Vec,InsertMode,ScatterMode); 463 EXTERN int VecGhostUpdateEnd(Vec,InsertMode,ScatterMode); 464 465 EXTERN int VecConjugate(Vec); 466 467 EXTERN int VecScatterCreateToAll(Vec,VecScatter*,Vec*); 468 EXTERN int VecScatterCreateToZero(Vec,VecScatter*,Vec*); 469 470 EXTERN int VecESISetType(Vec,const char*); 471 EXTERN int VecESISetFromOptions(Vec); 472 473 EXTERN int PetscViewerMathematicaGetVector(PetscViewer, Vec); 474 EXTERN int PetscViewerMathematicaPutVector(PetscViewer, Vec); 475 476 /*S 477 Vecs - Collection of vectors where the data for the vectors is stored in 478 one continquous memory 479 480 Level: advanced 481 482 Notes: 483 Temporary construct for handling multiply right hand side solves 484 485 This is faked by storing a single vector that has enough array space for 486 n vectors 487 488 Concepts: parallel decomposition 489 490 S*/ 491 struct _p_Vecs {int n; Vec v;}; 492 typedef struct _p_Vecs* Vecs; 493 #define VecsDestroy(x) (VecDestroy((x)->v) || PetscFree(x)) 494 #define VecsCreateSeq(comm,p,m,x) (PetscNew(struct _p_Vecs,x) || VecCreateSeq(comm,p*m,&(*(x))->v) || (-1 == ((*(x))->n = (m)))) 495 #define VecsCreateSeqWithArray(comm,p,m,a,x) (PetscNew(struct _p_Vecs,x) || VecCreateSeqWithArray(comm,p*m,a,&(*(x))->v) || (-1 == ((*(x))->n = (m)))) 496 #define VecsDuplicate(x,y) (PetscNew(struct _p_Vecs,y) || VecDuplicate(x->v,&(*(y))->v) || (-1 == ((*(y))->n = (x)->n))) 497 498 499 PETSC_EXTERN_CXX_END 500 #endif 501