1 /* 2 This file contains Fortran stubs for PetscInitialize and Finalize. 3 */ 4 5 /* 6 This is to prevent the Cray T3D version of MPI (University of Edinburgh) 7 from stupidly redefining MPI_INIT(). They put this in to detect errors 8 in C code,but here I do want to be calling the Fortran version from a 9 C subroutine. 10 */ 11 #define T3DMPI_FORTRAN 12 #define T3EMPI_FORTRAN 13 14 #define PETSC_DESIRE_COMPLEX 15 #include <petsc-private/fortranimpl.h> 16 17 #if defined(PETSC_HAVE_CUSP) 18 #include <cublas.h> 19 #endif 20 #include <petscthreadcomm.h> 21 22 #if defined(PETSC_HAVE_FORTRAN_CAPS) 23 #define petscinitialize_ PETSCINITIALIZE 24 #define petscfinalize_ PETSCFINALIZE 25 #define petscend_ PETSCEND 26 #define iargc_ IARGC 27 #define getarg_ GETARG 28 #define mpi_init_ MPI_INIT 29 #define petscgetcommoncomm_ PETSCGETCOMMONCOMM 30 #elif !defined(PETSC_HAVE_FORTRAN_UNDERSCORE) 31 #define petscinitialize_ petscinitialize 32 #define petscfinalize_ petscfinalize 33 #define petscend_ petscend 34 #define mpi_init_ mpi_init 35 #define iargc_ iargc 36 #define getarg_ getarg 37 #define petscgetcommoncomm_ petscgetcommoncomm 38 #endif 39 40 #if defined(PETSC_HAVE_NAGF90) 41 #undef iargc_ 42 #undef getarg_ 43 #define iargc_ f90_unix_MP_iargc 44 #define getarg_ f90_unix_MP_getarg 45 #endif 46 #if defined(PETSC_USE_NARGS) /* Digital Fortran */ 47 #undef iargc_ 48 #undef getarg_ 49 #define iargc_ NARGS 50 #define getarg_ GETARG 51 #elif defined(PETSC_HAVE_PXFGETARG_NEW) /* cray x1 */ 52 #undef iargc_ 53 #undef getarg_ 54 #define iargc_ ipxfargc_ 55 #define getarg_ pxfgetarg_ 56 #endif 57 #if defined(PETSC_HAVE_FORTRAN_IARGC_UNDERSCORE) /* HPUX + no underscore */ 58 #undef iargc_ 59 #undef getarg_ 60 #define iargc_ iargc_ 61 #define getarg_ getarg_ 62 #endif 63 #if defined(PETSC_HAVE_GFORTRAN_IARGC) /* gfortran from gcc4 */ 64 #undef iargc_ 65 #undef getarg_ 66 #define iargc_ _gfortran_iargc 67 #define getarg_ _gfortran_getarg_i4 68 #elif defined(PETSC_HAVE_BGL_IARGC) /* bgl g77 has different external & internal name mangling */ 69 #undef iargc_ 70 #undef getarg_ 71 #define iargc iargc_ 72 #define getarg getarg_ 73 #endif 74 75 /* 76 The extra _ is because the f2c compiler puts an 77 extra _ at the end if the original routine name 78 contained any _. 79 */ 80 #if defined(PETSC_HAVE_FORTRAN_UNDERSCORE_UNDERSCORE) 81 #undef mpi_init_ 82 #define mpi_init_ mpi_init__ 83 #endif 84 85 PETSC_EXTERN void PETSC_STDCALL mpi_init_(int*); 86 PETSC_EXTERN void PETSC_STDCALL petscgetcommoncomm_(PetscMPIInt*); 87 88 /* 89 Different Fortran compilers handle command lines in different ways 90 */ 91 #if defined(PETSC_USE_NARGS) 92 PETSC_EXTERN short __stdcall NARGS(); 93 PETSC_EXTERN void __stdcall GETARG(short*,char*,int,short *); 94 95 #elif defined(PETSC_HAVE_FORTRAN_STDCALL) 96 PETSC_EXTERN int PETSC_STDCALL IARGC(); 97 PETSC_EXTERN void PETSC_STDCALL GETARG(int *,char *,int); 98 99 #elif defined(PETSC_HAVE_PXFGETARG_NEW) 100 PETSC_EXTERN int iargc_(); 101 PETSC_EXTERN void getarg_(int*,char*,int*,int*,int); 102 103 #else 104 PETSC_EXTERN int iargc_(); 105 PETSC_EXTERN void getarg_(int*,char*,int); 106 /* 107 The Cray T3D/T3E use the PXFGETARG() function 108 */ 109 #if defined(PETSC_HAVE_PXFGETARG) 110 PETSC_EXTERN void PXFGETARG(int*,_fcd,int*,int*); 111 #endif 112 #endif 113 114 #if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128) 115 extern MPI_Op MPIU_SUM; 116 117 PETSC_EXTERN void MPIAPI PetscSum_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 118 119 #endif 120 #if defined(PETSC_USE_REAL___FLOAT128) 121 122 PETSC_EXTERN void MPIAPI PetscSum_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 123 PETSC_EXTERN void MPIAPI PetscMax_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 124 PETSC_EXTERN void MPIAPI PetscMin_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 125 #endif 126 127 extern MPI_Op PetscMaxSum_Op; 128 129 PETSC_EXTERN void MPIAPI PetscMaxSum_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 130 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelCounter(MPI_Comm,PetscMPIInt,void*,void*); 131 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelComm_Inner(MPI_Comm,PetscMPIInt,void*,void*); 132 PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelComm_Outer(MPI_Comm,PetscMPIInt,void*,void*); 133 134 extern PetscErrorCode PetscOptionsCheckInitial_Private(void); 135 extern PetscErrorCode PetscOptionsCheckInitial_Components(void); 136 extern PetscErrorCode PetscInitialize_DynamicLibraries(void); 137 #if defined(PETSC_USE_LOG) 138 extern PetscErrorCode PetscLogBegin_Private(void); 139 #endif 140 extern PetscErrorCode PetscMallocAlign(size_t,int,const char[],const char[],void**); 141 extern PetscErrorCode PetscFreeAlign(void*,int,const char[],const char[]); 142 extern int PetscGlobalArgc; 143 extern char **PetscGlobalArgs; 144 145 /* 146 Reads in Fortran command line argments and sends them to 147 all processors and adds them to Options database. 148 */ 149 150 PetscErrorCode PETScParseFortranArgs_Private(int *argc,char ***argv) 151 { 152 #if defined(PETSC_USE_NARGS) 153 short i,flg; 154 #else 155 int i; 156 #endif 157 PetscErrorCode ierr; 158 int warg = 256; 159 PetscMPIInt rank; 160 char *p; 161 162 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 163 if (!rank) { 164 #if defined(PETSC_HAVE_IARG_COUNT_PROGNAME) 165 *argc = iargc_(); 166 #else 167 /* most compilers do not count the program name for argv[0] */ 168 *argc = 1 + iargc_(); 169 #endif 170 } 171 ierr = MPI_Bcast(argc,1,MPI_INT,0,PETSC_COMM_WORLD);CHKERRQ(ierr); 172 173 /* PetscTrMalloc() not yet set, so don't use PetscMalloc() */ 174 ierr = PetscMallocAlign((*argc+1)*(warg*sizeof(char)+sizeof(char*)),0,0,0,(void**)argv);CHKERRQ(ierr); 175 (*argv)[0] = (char*)(*argv + *argc + 1); 176 177 if (!rank) { 178 ierr = PetscMemzero((*argv)[0],(*argc)*warg*sizeof(char));CHKERRQ(ierr); 179 for (i=0; i<*argc; i++) { 180 (*argv)[i+1] = (*argv)[i] + warg; 181 #if defined(PETSC_HAVE_PXFGETARG_NEW) 182 {char *tmp = (*argv)[i]; 183 int ilen; 184 getarg_(&i,tmp,&ilen,&ierr,warg);CHKERRQ(ierr); 185 tmp[ilen] = 0;} 186 #elif defined(PETSC_USE_NARGS) 187 GETARG(&i,(*argv)[i],warg,&flg); 188 #else 189 /* 190 Because the stupid #defines above define all kinds of things to getarg_ we cannot do this test 191 #elif defined(PETSC_HAVE_GETARG) 192 getarg_(&i,(*argv)[i],warg); 193 #else 194 SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot get Fortran command line arguments"); 195 */ 196 getarg_(&i,(*argv)[i],warg); 197 #endif 198 /* zero out garbage at end of each argument */ 199 p = (*argv)[i] + warg-1; 200 while (p > (*argv)[i]) { 201 if (*p == ' ') *p = 0; 202 p--; 203 } 204 } 205 } 206 ierr = MPI_Bcast((*argv)[0],*argc*warg,MPI_CHAR,0,PETSC_COMM_WORLD);CHKERRQ(ierr); 207 if (rank) { 208 for (i=0; i<*argc; i++) (*argv)[i+1] = (*argv)[i] + warg; 209 } 210 return 0; 211 } 212 213 /* -----------------------------------------------------------------------------------------------*/ 214 215 extern MPI_Op PetscADMax_Op; 216 extern MPI_Op PetscADMin_Op; 217 PETSC_EXTERN void MPIAPI PetscADMax_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 218 PETSC_EXTERN void MPIAPI PetscADMin_Local(void*,void*,PetscMPIInt*,MPI_Datatype*); 219 220 #if defined(PETSC_HAVE_SAWS) 221 #include <petscviewersaws.h> 222 extern PetscErrorCode PetscInitializeSAWs(const char[]); 223 #endif 224 225 /* 226 petscinitialize - Version called from Fortran. 227 228 Notes: 229 Since this is called from Fortran it does not return error codes 230 231 */ 232 PETSC_EXTERN void PETSC_STDCALL petscinitialize_(CHAR filename PETSC_MIXED_LEN(len),PetscErrorCode *ierr PETSC_END_LEN(len)) 233 { 234 #if defined (PETSC_USE_NARGS) 235 short flg,i; 236 #else 237 int i; 238 #if !defined(PETSC_HAVE_PXFGETARG_NEW) && !defined (PETSC_HAVE_PXFGETARG_NEW) 239 int j; 240 #endif 241 #endif 242 int flag; 243 PetscMPIInt size; 244 char *t1,name[256],hostname[64]; 245 PetscMPIInt f_petsc_comm_world; 246 247 *ierr = PetscMemzero(name,256); if (*ierr) return; 248 if (PetscInitializeCalled) {*ierr = 0; return;} 249 250 /* this must be initialized in a routine, not as a constant declaration*/ 251 PETSC_STDOUT = stdout; 252 PETSC_STDERR = stderr; 253 254 *ierr = PetscOptionsCreate(); 255 if (*ierr) return; 256 i = 0; 257 #if defined (PETSC_HAVE_PXFGETARG_NEW) 258 { int ilen,sierr; 259 getarg_(&i,name,&ilen,&sierr,256); 260 if (sierr) PetscStrncpy(name,"Unknown Name",256); 261 else name[ilen] = 0; 262 } 263 #elif defined(PETSC_USE_NARGS) 264 GETARG(&i,name,256,&flg); 265 #else 266 getarg_(&i,name,256); 267 /* Eliminate spaces at the end of the string */ 268 for (j=254; j>=0; j--) { 269 if (name[j] != ' ') { 270 name[j+1] = 0; 271 break; 272 } 273 } 274 if (j<0) PetscStrncpy(name,"Unknown Name",256); 275 #endif 276 *ierr = PetscSetProgramName(name); 277 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize: Calling PetscSetProgramName()\n");return;} 278 279 /* check if PETSC_COMM_WORLD is initialized by the user in fortran */ 280 petscgetcommoncomm_(&f_petsc_comm_world); 281 MPI_Initialized(&flag); 282 if (!flag) { 283 PetscMPIInt mierr; 284 285 if (f_petsc_comm_world) {(*PetscErrorPrintf)("You cannot set PETSC_COMM_WORLD if you have not initialized MPI first\n");return;} 286 /* MPI requires calling Fortran mpi_init() if main program is Fortran */ 287 #if defined(PETSC_HAVE_MPIUNI) && defined(MPIUNI_AVOID_MPI_NAMESPACE) 288 mierr = MPI_Init((int*)0, (char***)0); 289 #else 290 mpi_init_(&mierr); 291 #endif 292 if (mierr) { 293 *ierr = mierr; 294 (*PetscErrorPrintf)("PetscInitialize: Calling Fortran MPI_Init()\n"); 295 return; 296 } 297 PetscBeganMPI = PETSC_TRUE; 298 } 299 if (f_petsc_comm_world) PETSC_COMM_WORLD = MPI_Comm_f2c(*(MPI_Fint*)&f_petsc_comm_world); /* User called MPI_INITIALIZE() and changed PETSC_COMM_WORLD */ 300 else PETSC_COMM_WORLD = MPI_COMM_WORLD; 301 PetscInitializeCalled = PETSC_TRUE; 302 303 *ierr = PetscErrorPrintfInitialize(); 304 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize: Calling PetscErrorPrintfInitialize()\n");return;} 305 *ierr = MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank); 306 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize: Setting PetscGlobalRank\n");return;} 307 *ierr = MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize); 308 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize: Setting PetscGlobalSize\n");return;} 309 310 MPIU_BOOL = MPI_INT; 311 MPIU_ENUM = MPI_INT; 312 313 #if defined(PETSC_HAVE_COMPLEX) 314 /* 315 Initialized the global variable; this is because with 316 shared libraries the constructors for global variables 317 are not called; at least on IRIX. 318 */ 319 { 320 #if defined(PETSC_CLANGUAGE_CXX) 321 PetscComplex ic(0.0,1.0); 322 PETSC_i = ic; 323 #else 324 PETSC_i = _Complex_I; 325 #endif 326 } 327 328 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 329 *ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU_C_DOUBLE_COMPLEX); 330 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 331 *ierr = MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX); 332 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 333 *ierr = MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX); 334 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 335 *ierr = MPI_Type_commit(&MPIU_C_COMPLEX); 336 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 337 *ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM); 338 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 339 #endif 340 341 #endif 342 343 #if defined(PETSC_USE_REAL___FLOAT128) 344 *ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128); 345 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 346 *ierr = MPI_Type_commit(&MPIU___FLOAT128); 347 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 348 #if defined(PETSC_HAVE_COMPLEX) 349 *ierr = MPI_Type_contiguous(4,MPI_DOUBLE,&MPIU___COMPLEX128); 350 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 351 *ierr = MPI_Type_commit(&MPIU___COMPLEX128); 352 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 353 #endif 354 *ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM); 355 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 356 *ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX); 357 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 358 *ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN); 359 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 360 #endif 361 362 /* 363 Create the PETSc MPI reduction operator that sums of the first 364 half of the entries and maxes the second half. 365 */ 366 *ierr = MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op); 367 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 368 369 *ierr = MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR); 370 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 371 *ierr = MPI_Type_commit(&MPIU_2SCALAR); 372 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 373 #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT) 374 *ierr = MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT); 375 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 376 *ierr = MPI_Type_commit(&MPIU_2INT); 377 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI types\n");return;} 378 #endif 379 *ierr = MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op); 380 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 381 *ierr = MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op); 382 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI ops\n");return;} 383 *ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0); 384 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI keyvals\n");return;} 385 *ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm_Outer,&Petsc_InnerComm_keyval,(void*)0); 386 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI keyvals\n");return;} 387 *ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm_Inner,&Petsc_OuterComm_keyval,(void*)0); 388 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating MPI keyvals\n");return;} 389 390 /* 391 PetscInitializeFortran() is called twice. Here it initializes 392 PETSC_NULL_CHARACTER_Fortran. Below it initializes the PETSC_VIEWERs. 393 The PETSC_VIEWERs have not been created yet, so they must be initialized 394 below. 395 */ 396 PetscInitializeFortran(); 397 PETScParseFortranArgs_Private(&PetscGlobalArgc,&PetscGlobalArgs); 398 FIXCHAR(filename,len,t1); 399 *ierr = PetscOptionsInsert(&PetscGlobalArgc,&PetscGlobalArgs,t1); 400 FREECHAR(filename,t1); 401 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Creating options database\n");return;} 402 *ierr = PetscOptionsCheckInitial_Private(); 403 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Checking initial options\n");return;} 404 *ierr = PetscCitationsInitialize(); 405 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:PetscCitationsInitialize()\n");return;} 406 #if defined(PETSC_HAVE_SAWS) 407 *ierr = PetscInitializeSAWs(NULL); 408 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Initializing SAWs\n");return;} 409 #endif 410 #if defined(PETSC_USE_LOG) 411 *ierr = PetscLogBegin_Private(); 412 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize: intializing logging\n");return;} 413 #endif 414 *ierr = PetscInitialize_DynamicLibraries(); 415 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Initializing dynamic libraries\n");return;} 416 417 *ierr = PetscInitializeFortran(); 418 if (*ierr) { (*PetscErrorPrintf)("PetscInitialize:Setting up common block\n");return;} 419 420 *ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size); 421 if (*ierr) { (*PetscErrorPrintf)("PetscInitialize:Getting MPI_Comm_size()\n");return;} 422 *ierr = PetscInfo1(0,"(Fortran):PETSc successfully started: procs %d\n",size); 423 if (*ierr) { (*PetscErrorPrintf)("PetscInitialize:Calling PetscInfo()\n");return;} 424 *ierr = PetscGetHostName(hostname,64); 425 if (*ierr) { (*PetscErrorPrintf)("PetscInitialize:Getting hostname\n");return;} 426 *ierr = PetscInfo1(0,"Running on machine: %s\n",hostname); 427 if (*ierr) { (*PetscErrorPrintf)("PetscInitialize:Calling PetscInfo()\n");return;} 428 *ierr = PetscOptionsCheckInitial_Components(); 429 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Checking initial options\n");return;} 430 431 *ierr = PetscThreadCommInitializePackage(); 432 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:Calling PetscThreadCommInitialize()\n");return;} 433 434 PetscThreadLocalRegister((PetscThreadKey*)&petscstack); /* Creates pthread_key */ 435 #if defined(PETSC_USE_DEBUG) 436 *ierr = PetscStackCreate(); 437 if (*ierr) {(*PetscErrorPrintf)("PetscInitialize:PetscStackCreate()\n");return;} 438 #endif 439 440 #if defined(PETSC_HAVE_CUDA) 441 flg = PETSC_TRUE; 442 *ierr = PetscOptionsGetBool(NULL,"-cublas",&flg,NULL); 443 if (flg) cublasInit(); 444 #endif 445 } 446 447 PETSC_EXTERN void PETSC_STDCALL petscfinalize_(PetscErrorCode *ierr) 448 { 449 #if defined(PETSC_HAVE_SUNMATHPRO) 450 extern void standard_arithmetic(); 451 standard_arithmetic(); 452 #endif 453 /* was malloced with PetscMallocAlign() so free the same way */ 454 *ierr = PetscFreeAlign(PetscGlobalArgs,0,0,0);if (*ierr) {(*PetscErrorPrintf)("PetscFinalize:Freeing args\n");return;} 455 456 *ierr = PetscFinalize(); 457 } 458 459 void PETSC_STDCALL petscend_(PetscErrorCode *ierr) 460 { 461 #if defined(PETSC_HAVE_SUNMATHPRO) 462 extern void standard_arithmetic(); 463 standard_arithmetic(); 464 #endif 465 466 *ierr = PetscEnd(); 467 } 468 469