1 #include <petsc/private/dmpleximpl.h> /*I "petscdmplex.h" I*/ 2 #include <petscsf.h> 3 4 #include <petscblaslapack.h> 5 #include <petsc/private/hashsetij.h> 6 #include <petsc/private/petscfeimpl.h> 7 #include <petsc/private/petscfvimpl.h> 8 9 PetscBool Clementcite = PETSC_FALSE; 10 const char ClementCitation[] = "@article{clement1975approximation,\n" 11 " title = {Approximation by finite element functions using local regularization},\n" 12 " author = {Philippe Cl{\\'e}ment},\n" 13 " journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n" 14 " volume = {9},\n" 15 " number = {R2},\n" 16 " pages = {77--84},\n" 17 " year = {1975}\n}\n"; 18 19 static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy) 20 { 21 PetscBool isPlex; 22 23 PetscFunctionBegin; 24 PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex)); 25 if (isPlex) { 26 *plex = dm; 27 PetscCall(PetscObjectReference((PetscObject)dm)); 28 } else { 29 PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex)); 30 if (!*plex) { 31 PetscCall(DMConvert(dm, DMPLEX, plex)); 32 PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex)); 33 } else { 34 PetscCall(PetscObjectReference((PetscObject)*plex)); 35 } 36 if (copy) { 37 DMSubDomainHookLink link; 38 39 PetscCall(DMCopyDS(dm, PETSC_DETERMINE, PETSC_DETERMINE, *plex)); 40 PetscCall(DMCopyAuxiliaryVec(dm, *plex)); 41 /* Run the subdomain hook (this will copy the DMSNES/DMTS) */ 42 for (link = dm->subdomainhook; link; link = link->next) { 43 if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx)); 44 } 45 } 46 } 47 PetscFunctionReturn(PETSC_SUCCESS); 48 } 49 50 static PetscErrorCode PetscContainerCtxDestroy_PetscFEGeom(void **ctx) 51 { 52 PetscFEGeom *geom = (PetscFEGeom *)*ctx; 53 54 PetscFunctionBegin; 55 PetscCall(PetscFEGeomDestroy(&geom)); 56 PetscFunctionReturn(PETSC_SUCCESS); 57 } 58 59 static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom) 60 { 61 char composeStr[33] = {0}; 62 PetscObjectId id; 63 PetscContainer container; 64 65 PetscFunctionBegin; 66 PetscCall(PetscObjectGetId((PetscObject)quad, &id)); 67 PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id)); 68 PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container)); 69 if (container) { 70 PetscCall(PetscContainerGetPointer(container, (void **)geom)); 71 } else { 72 PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, mode, geom)); 73 PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container)); 74 PetscCall(PetscContainerSetPointer(container, (void *)*geom)); 75 PetscCall(PetscContainerSetCtxDestroy(container, PetscContainerCtxDestroy_PetscFEGeom)); 76 PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container)); 77 PetscCall(PetscContainerDestroy(&container)); 78 } 79 PetscFunctionReturn(PETSC_SUCCESS); 80 } 81 82 static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom) 83 { 84 PetscFunctionBegin; 85 *geom = NULL; 86 PetscFunctionReturn(PETSC_SUCCESS); 87 } 88 89 /*@ 90 DMPlexGetScale - Get the scale for the specified fundamental unit 91 92 Not Collective 93 94 Input Parameters: 95 + dm - the `DM` 96 - unit - The SI unit 97 98 Output Parameter: 99 . scale - The value used to scale all quantities with this unit 100 101 Level: advanced 102 103 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetScale()`, `PetscUnit` 104 @*/ 105 PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale) 106 { 107 DM_Plex *mesh = (DM_Plex *)dm->data; 108 109 PetscFunctionBegin; 110 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 111 PetscAssertPointer(scale, 3); 112 *scale = mesh->scale[unit]; 113 PetscFunctionReturn(PETSC_SUCCESS); 114 } 115 116 /*@ 117 DMPlexSetScale - Set the scale for the specified fundamental unit 118 119 Not Collective 120 121 Input Parameters: 122 + dm - the `DM` 123 . unit - The SI unit 124 - scale - The value used to scale all quantities with this unit 125 126 Level: advanced 127 128 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetScale()`, `PetscUnit` 129 @*/ 130 PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale) 131 { 132 DM_Plex *mesh = (DM_Plex *)dm->data; 133 134 PetscFunctionBegin; 135 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 136 mesh->scale[unit] = scale; 137 PetscFunctionReturn(PETSC_SUCCESS); 138 } 139 140 PetscErrorCode DMPlexGetUseCeed_Plex(DM dm, PetscBool *useCeed) 141 { 142 DM_Plex *mesh = (DM_Plex *)dm->data; 143 144 PetscFunctionBegin; 145 *useCeed = mesh->useCeed; 146 PetscFunctionReturn(PETSC_SUCCESS); 147 } 148 PetscErrorCode DMPlexSetUseCeed_Plex(DM dm, PetscBool useCeed) 149 { 150 DM_Plex *mesh = (DM_Plex *)dm->data; 151 152 PetscFunctionBegin; 153 mesh->useCeed = useCeed; 154 PetscFunctionReturn(PETSC_SUCCESS); 155 } 156 157 /*@ 158 DMPlexGetUseCeed - Get flag for using the LibCEED backend 159 160 Not collective 161 162 Input Parameter: 163 . dm - The `DM` 164 165 Output Parameter: 166 . useCeed - The flag 167 168 Level: intermediate 169 170 .seealso: `DMPlexSetUseCeed()` 171 @*/ 172 PetscErrorCode DMPlexGetUseCeed(DM dm, PetscBool *useCeed) 173 { 174 PetscFunctionBegin; 175 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 176 PetscAssertPointer(useCeed, 2); 177 *useCeed = PETSC_FALSE; 178 PetscTryMethod(dm, "DMPlexGetUseCeed_C", (DM, PetscBool *), (dm, useCeed)); 179 PetscFunctionReturn(PETSC_SUCCESS); 180 } 181 182 /*@ 183 DMPlexSetUseCeed - Set flag for using the LibCEED backend 184 185 Not collective 186 187 Input Parameters: 188 + dm - The `DM` 189 - useCeed - The flag 190 191 Level: intermediate 192 193 .seealso: `DMPlexGetUseCeed()` 194 @*/ 195 PetscErrorCode DMPlexSetUseCeed(DM dm, PetscBool useCeed) 196 { 197 PetscFunctionBegin; 198 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 199 PetscValidLogicalCollectiveBool(dm, useCeed, 2); 200 PetscUseMethod(dm, "DMPlexSetUseCeed_C", (DM, PetscBool), (dm, useCeed)); 201 PetscFunctionReturn(PETSC_SUCCESS); 202 } 203 204 /*@ 205 DMPlexGetUseMatClosurePermutation - Get flag for using a closure permutation for matrix insertion 206 207 Not collective 208 209 Input Parameter: 210 . dm - The `DM` 211 212 Output Parameter: 213 . useClPerm - The flag 214 215 Level: intermediate 216 217 .seealso: `DMPlexSetUseMatClosurePermutation()` 218 @*/ 219 PetscErrorCode DMPlexGetUseMatClosurePermutation(DM dm, PetscBool *useClPerm) 220 { 221 DM_Plex *mesh = (DM_Plex *)dm->data; 222 223 PetscFunctionBegin; 224 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 225 PetscAssertPointer(useClPerm, 2); 226 *useClPerm = mesh->useMatClPerm; 227 PetscFunctionReturn(PETSC_SUCCESS); 228 } 229 230 /*@ 231 DMPlexSetUseMatClosurePermutation - Set flag for using a closure permutation for matrix insertion 232 233 Not collective 234 235 Input Parameters: 236 + dm - The `DM` 237 - useClPerm - The flag 238 239 Level: intermediate 240 241 .seealso: `DMPlexGetUseMatClosurePermutation()` 242 @*/ 243 PetscErrorCode DMPlexSetUseMatClosurePermutation(DM dm, PetscBool useClPerm) 244 { 245 DM_Plex *mesh = (DM_Plex *)dm->data; 246 247 PetscFunctionBegin; 248 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 249 PetscValidLogicalCollectiveBool(dm, useClPerm, 2); 250 mesh->useMatClPerm = useClPerm; 251 PetscFunctionReturn(PETSC_SUCCESS); 252 } 253 254 static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx) 255 { 256 const PetscInt eps[3][3][3] = { 257 {{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, 258 {{0, 0, -1}, {0, 0, 0}, {1, 0, 0} }, 259 {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0} } 260 }; 261 PetscInt *ctxInt = (PetscInt *)ctx; 262 PetscInt dim2 = ctxInt[0]; 263 PetscInt d = ctxInt[1]; 264 PetscInt i, j, k = dim > 2 ? d - dim : d; 265 266 PetscFunctionBegin; 267 PetscCheck(dim == dim2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2); 268 for (i = 0; i < dim; i++) mode[i] = 0.; 269 if (d < dim) { 270 mode[d] = 1.; /* Translation along axis d */ 271 } else { 272 for (i = 0; i < dim; i++) { 273 for (j = 0; j < dim; j++) { mode[j] += eps[i][j][k] * X[i]; /* Rotation about axis d */ } 274 } 275 } 276 PetscFunctionReturn(PETSC_SUCCESS); 277 } 278 279 /*@ 280 DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation 281 282 Collective 283 284 Input Parameters: 285 + dm - the `DM` 286 - field - The field number for the rigid body space, or 0 for the default 287 288 Output Parameter: 289 . sp - the null space 290 291 Level: advanced 292 293 Note: 294 This is necessary to provide a suitable coarse space for algebraic multigrid 295 296 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()`, `PCGAMG` 297 @*/ 298 PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp) 299 { 300 PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *); 301 MPI_Comm comm; 302 Vec mode[6]; 303 PetscSection section, globalSection; 304 PetscInt dim, dimEmbed, Nf, n, m, mmin, d, i, j; 305 void **ctxs; 306 307 PetscFunctionBegin; 308 PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); 309 PetscCall(DMGetDimension(dm, &dim)); 310 PetscCall(DMGetCoordinateDim(dm, &dimEmbed)); 311 PetscCall(DMGetNumFields(dm, &Nf)); 312 PetscCheck(!Nf || !(field < 0 || field >= Nf), comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf); 313 if (dim == 1 && Nf < 2) { 314 PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp)); 315 PetscFunctionReturn(PETSC_SUCCESS); 316 } 317 PetscCall(DMGetLocalSection(dm, §ion)); 318 PetscCall(DMGetGlobalSection(dm, &globalSection)); 319 PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n)); 320 PetscCall(PetscCalloc2(Nf, &func, Nf, &ctxs)); 321 m = (dim * (dim + 1)) / 2; 322 PetscCall(VecCreate(comm, &mode[0])); 323 PetscCall(VecSetType(mode[0], dm->vectype)); 324 PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE)); 325 PetscCall(VecSetUp(mode[0])); 326 PetscCall(VecGetSize(mode[0], &n)); 327 mmin = PetscMin(m, n); 328 func[field] = DMPlexProjectRigidBody_Private; 329 for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i])); 330 for (d = 0; d < m; d++) { 331 PetscInt ctx[2]; 332 333 ctxs[field] = (void *)(&ctx[0]); 334 ctx[0] = dimEmbed; 335 ctx[1] = d; 336 PetscCall(DMProjectFunction(dm, 0.0, func, ctxs, INSERT_VALUES, mode[d])); 337 } 338 /* Orthonormalize system */ 339 for (i = 0; i < mmin; ++i) { 340 PetscScalar dots[6]; 341 342 PetscCall(VecNormalize(mode[i], NULL)); 343 PetscCall(VecMDot(mode[i], mmin - i - 1, mode + i + 1, dots + i + 1)); 344 for (j = i + 1; j < mmin; ++j) { 345 dots[j] *= -1.0; 346 PetscCall(VecAXPY(mode[j], dots[j], mode[i])); 347 } 348 } 349 PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp)); 350 for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i])); 351 PetscCall(PetscFree2(func, ctxs)); 352 PetscFunctionReturn(PETSC_SUCCESS); 353 } 354 355 /*@ 356 DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation 357 358 Collective 359 360 Input Parameters: 361 + dm - the `DM` 362 . nb - The number of bodies 363 . label - The `DMLabel` marking each domain 364 . nids - The number of ids per body 365 - ids - An array of the label ids in sequence for each domain 366 367 Output Parameter: 368 . sp - the null space 369 370 Level: advanced 371 372 Note: 373 This is necessary to provide a suitable coarse space for algebraic multigrid 374 375 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()` 376 @*/ 377 PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp) 378 { 379 MPI_Comm comm; 380 PetscSection section, globalSection; 381 Vec *mode; 382 PetscScalar *dots; 383 PetscInt dim, dimEmbed, n, m, b, d, i, j, off; 384 385 PetscFunctionBegin; 386 PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); 387 PetscCall(DMGetDimension(dm, &dim)); 388 PetscCall(DMGetCoordinateDim(dm, &dimEmbed)); 389 PetscCall(DMGetLocalSection(dm, §ion)); 390 PetscCall(DMGetGlobalSection(dm, &globalSection)); 391 PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n)); 392 m = nb * (dim * (dim + 1)) / 2; 393 PetscCall(PetscMalloc2(m, &mode, m, &dots)); 394 PetscCall(VecCreate(comm, &mode[0])); 395 PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE)); 396 PetscCall(VecSetUp(mode[0])); 397 for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i])); 398 for (b = 0, off = 0; b < nb; ++b) { 399 for (d = 0; d < m / nb; ++d) { 400 PetscInt ctx[2]; 401 PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private; 402 void *voidctx = (void *)(&ctx[0]); 403 404 ctx[0] = dimEmbed; 405 ctx[1] = d; 406 PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d])); 407 off += nids[b]; 408 } 409 } 410 /* Orthonormalize system */ 411 for (i = 0; i < m; ++i) { 412 PetscScalar dots[6]; 413 414 PetscCall(VecNormalize(mode[i], NULL)); 415 PetscCall(VecMDot(mode[i], m - i - 1, mode + i + 1, dots + i + 1)); 416 for (j = i + 1; j < m; ++j) { 417 dots[j] *= -1.0; 418 PetscCall(VecAXPY(mode[j], dots[j], mode[i])); 419 } 420 } 421 PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp)); 422 for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i])); 423 PetscCall(PetscFree2(mode, dots)); 424 PetscFunctionReturn(PETSC_SUCCESS); 425 } 426 427 /*@ 428 DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs 429 are computed by associating the basis function with one of the mesh points in its transitively-closed support, and 430 evaluating the dual space basis of that point. 431 432 Input Parameters: 433 + dm - the `DMPLEX` object 434 - height - the maximum projection height >= 0 435 436 Level: advanced 437 438 Notes: 439 A basis function is associated with the point in its transitively-closed support whose mesh 440 height is highest (w.r.t. DAG height), but not greater than the maximum projection height, 441 which is set with this function. By default, the maximum projection height is zero, which 442 means that only mesh cells are used to project basis functions. A height of one, for 443 example, evaluates a cell-interior basis functions using its cells dual space basis, but all 444 other basis functions with the dual space basis of a face. 445 446 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()` 447 @*/ 448 PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height) 449 { 450 DM_Plex *plex = (DM_Plex *)dm->data; 451 452 PetscFunctionBegin; 453 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 454 plex->maxProjectionHeight = height; 455 PetscFunctionReturn(PETSC_SUCCESS); 456 } 457 458 /*@ 459 DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in 460 DMPlexProjectXXXLocal() functions. 461 462 Input Parameter: 463 . dm - the `DMPLEX` object 464 465 Output Parameter: 466 . height - the maximum projection height 467 468 Level: intermediate 469 470 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()` 471 @*/ 472 PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height) 473 { 474 DM_Plex *plex = (DM_Plex *)dm->data; 475 476 PetscFunctionBegin; 477 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 478 *height = plex->maxProjectionHeight; 479 PetscFunctionReturn(PETSC_SUCCESS); 480 } 481 482 typedef struct { 483 PetscReal alpha; /* The first Euler angle, and in 2D the only one */ 484 PetscReal beta; /* The second Euler angle */ 485 PetscReal gamma; /* The third Euler angle */ 486 PetscInt dim; /* The dimension of R */ 487 PetscScalar *R; /* The rotation matrix, transforming a vector in the local basis to the global basis */ 488 PetscScalar *RT; /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */ 489 } RotCtx; 490 491 /* 492 Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that 493 we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows: 494 $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis. 495 $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis. 496 $ The XYZ system rotates a third time about the z axis by gamma. 497 */ 498 static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx) 499 { 500 RotCtx *rc = (RotCtx *)ctx; 501 PetscInt dim = rc->dim; 502 PetscReal c1, s1, c2, s2, c3, s3; 503 504 PetscFunctionBegin; 505 PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT)); 506 switch (dim) { 507 case 2: 508 c1 = PetscCosReal(rc->alpha); 509 s1 = PetscSinReal(rc->alpha); 510 rc->R[0] = c1; 511 rc->R[1] = s1; 512 rc->R[2] = -s1; 513 rc->R[3] = c1; 514 PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim))); 515 DMPlex_Transpose2D_Internal(rc->RT); 516 break; 517 case 3: 518 c1 = PetscCosReal(rc->alpha); 519 s1 = PetscSinReal(rc->alpha); 520 c2 = PetscCosReal(rc->beta); 521 s2 = PetscSinReal(rc->beta); 522 c3 = PetscCosReal(rc->gamma); 523 s3 = PetscSinReal(rc->gamma); 524 rc->R[0] = c1 * c3 - c2 * s1 * s3; 525 rc->R[1] = c3 * s1 + c1 * c2 * s3; 526 rc->R[2] = s2 * s3; 527 rc->R[3] = -c1 * s3 - c2 * c3 * s1; 528 rc->R[4] = c1 * c2 * c3 - s1 * s3; 529 rc->R[5] = c3 * s2; 530 rc->R[6] = s1 * s2; 531 rc->R[7] = -c1 * s2; 532 rc->R[8] = c2; 533 PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim))); 534 DMPlex_Transpose3D_Internal(rc->RT); 535 break; 536 default: 537 SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim); 538 } 539 PetscFunctionReturn(PETSC_SUCCESS); 540 } 541 542 static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx) 543 { 544 RotCtx *rc = (RotCtx *)ctx; 545 546 PetscFunctionBegin; 547 PetscCall(PetscFree2(rc->R, rc->RT)); 548 PetscCall(PetscFree(rc)); 549 PetscFunctionReturn(PETSC_SUCCESS); 550 } 551 552 static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx) 553 { 554 RotCtx *rc = (RotCtx *)ctx; 555 556 PetscFunctionBeginHot; 557 PetscAssertPointer(ctx, 5); 558 if (l2g) { 559 *A = rc->R; 560 } else { 561 *A = rc->RT; 562 } 563 PetscFunctionReturn(PETSC_SUCCESS); 564 } 565 566 PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx) 567 { 568 PetscFunctionBegin; 569 #if defined(PETSC_USE_COMPLEX) 570 switch (dim) { 571 case 2: { 572 PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0, 0.0}; 573 574 PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx)); 575 z[0] = PetscRealPart(zt[0]); 576 z[1] = PetscRealPart(zt[1]); 577 } break; 578 case 3: { 579 PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0, 0.0, 0.0}; 580 581 PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx)); 582 z[0] = PetscRealPart(zt[0]); 583 z[1] = PetscRealPart(zt[1]); 584 z[2] = PetscRealPart(zt[2]); 585 } break; 586 } 587 #else 588 PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx)); 589 #endif 590 PetscFunctionReturn(PETSC_SUCCESS); 591 } 592 593 PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx) 594 { 595 const PetscScalar *A; 596 597 PetscFunctionBeginHot; 598 PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx)); 599 switch (dim) { 600 case 2: 601 DMPlex_Mult2D_Internal(A, 1, y, z); 602 break; 603 case 3: 604 DMPlex_Mult3D_Internal(A, 1, y, z); 605 break; 606 } 607 PetscFunctionReturn(PETSC_SUCCESS); 608 } 609 610 static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a) 611 { 612 PetscSection ts; 613 const PetscScalar *ta, *tva; 614 PetscInt dof; 615 616 PetscFunctionBeginHot; 617 PetscCall(DMGetLocalSection(tdm, &ts)); 618 PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof)); 619 PetscCall(VecGetArrayRead(tv, &ta)); 620 PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva)); 621 if (l2g) { 622 switch (dof) { 623 case 4: 624 DMPlex_Mult2D_Internal(tva, 1, a, a); 625 break; 626 case 9: 627 DMPlex_Mult3D_Internal(tva, 1, a, a); 628 break; 629 } 630 } else { 631 switch (dof) { 632 case 4: 633 DMPlex_MultTranspose2D_Internal(tva, 1, a, a); 634 break; 635 case 9: 636 DMPlex_MultTranspose3D_Internal(tva, 1, a, a); 637 break; 638 } 639 } 640 PetscCall(VecRestoreArrayRead(tv, &ta)); 641 PetscFunctionReturn(PETSC_SUCCESS); 642 } 643 644 static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a) 645 { 646 PetscSection s, ts; 647 const PetscScalar *ta, *tvaf, *tvag; 648 PetscInt fdof, gdof, fpdof, gpdof; 649 650 PetscFunctionBeginHot; 651 PetscCall(DMGetLocalSection(dm, &s)); 652 PetscCall(DMGetLocalSection(tdm, &ts)); 653 PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof)); 654 PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof)); 655 PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof)); 656 PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof)); 657 PetscCall(VecGetArrayRead(tv, &ta)); 658 PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf)); 659 PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag)); 660 if (l2g) { 661 switch (fdof) { 662 case 4: 663 DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a); 664 break; 665 case 9: 666 DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a); 667 break; 668 } 669 switch (gdof) { 670 case 4: 671 DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a); 672 break; 673 case 9: 674 DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a); 675 break; 676 } 677 } else { 678 switch (fdof) { 679 case 4: 680 DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a); 681 break; 682 case 9: 683 DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a); 684 break; 685 } 686 switch (gdof) { 687 case 4: 688 DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a); 689 break; 690 case 9: 691 DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a); 692 break; 693 } 694 } 695 PetscCall(VecRestoreArrayRead(tv, &ta)); 696 PetscFunctionReturn(PETSC_SUCCESS); 697 } 698 699 PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a) 700 { 701 PetscSection s; 702 PetscSection clSection; 703 IS clPoints; 704 const PetscInt *clp; 705 PetscInt *points = NULL; 706 PetscInt Nf, f, Np, cp, dof, d = 0; 707 708 PetscFunctionBegin; 709 PetscCall(DMGetLocalSection(dm, &s)); 710 PetscCall(PetscSectionGetNumFields(s, &Nf)); 711 PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp)); 712 for (f = 0; f < Nf; ++f) { 713 for (cp = 0; cp < Np * 2; cp += 2) { 714 PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof)); 715 if (!dof) continue; 716 if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d])); 717 d += dof; 718 } 719 } 720 PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp)); 721 PetscFunctionReturn(PETSC_SUCCESS); 722 } 723 724 PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a) 725 { 726 PetscSection s; 727 PetscSection clSection; 728 IS clPoints; 729 const PetscInt *clp; 730 PetscInt *points = NULL; 731 PetscInt Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0; 732 733 PetscFunctionBegin; 734 PetscCall(DMGetLocalSection(dm, &s)); 735 PetscCall(PetscSectionGetNumFields(s, &Nf)); 736 PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp)); 737 for (f = 0, r = 0; f < Nf; ++f) { 738 for (cpf = 0; cpf < Np * 2; cpf += 2) { 739 PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof)); 740 for (g = 0, c = 0; g < Nf; ++g) { 741 for (cpg = 0; cpg < Np * 2; cpg += 2) { 742 PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof)); 743 PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r * lda + c])); 744 c += gdof; 745 } 746 } 747 PetscCheck(c == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda); 748 r += fdof; 749 } 750 } 751 PetscCheck(r == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda); 752 PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp)); 753 PetscFunctionReturn(PETSC_SUCCESS); 754 } 755 756 static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g) 757 { 758 DM tdm; 759 Vec tv; 760 PetscSection ts, s; 761 const PetscScalar *ta; 762 PetscScalar *a, *va; 763 PetscInt pStart, pEnd, p, Nf, f; 764 765 PetscFunctionBegin; 766 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 767 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 768 PetscCall(DMGetLocalSection(tdm, &ts)); 769 PetscCall(DMGetLocalSection(dm, &s)); 770 PetscCall(PetscSectionGetChart(s, &pStart, &pEnd)); 771 PetscCall(PetscSectionGetNumFields(s, &Nf)); 772 PetscCall(VecGetArray(lv, &a)); 773 PetscCall(VecGetArrayRead(tv, &ta)); 774 for (p = pStart; p < pEnd; ++p) { 775 for (f = 0; f < Nf; ++f) { 776 PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va)); 777 PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va)); 778 } 779 } 780 PetscCall(VecRestoreArray(lv, &a)); 781 PetscCall(VecRestoreArrayRead(tv, &ta)); 782 PetscFunctionReturn(PETSC_SUCCESS); 783 } 784 785 /*@ 786 DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis 787 788 Input Parameters: 789 + dm - The `DM` 790 - lv - A local vector with values in the global basis 791 792 Output Parameter: 793 . lv - A local vector with values in the local basis 794 795 Level: developer 796 797 Note: 798 This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal. 799 800 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLocalToGlobalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()` 801 @*/ 802 PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv) 803 { 804 PetscFunctionBegin; 805 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 806 PetscValidHeaderSpecific(lv, VEC_CLASSID, 2); 807 PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE)); 808 PetscFunctionReturn(PETSC_SUCCESS); 809 } 810 811 /*@ 812 DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis 813 814 Input Parameters: 815 + dm - The `DM` 816 - lv - A local vector with values in the local basis 817 818 Output Parameter: 819 . lv - A local vector with values in the global basis 820 821 Level: developer 822 823 Note: 824 This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal. 825 826 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()` 827 @*/ 828 PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv) 829 { 830 PetscFunctionBegin; 831 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 832 PetscValidHeaderSpecific(lv, VEC_CLASSID, 2); 833 PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE)); 834 PetscFunctionReturn(PETSC_SUCCESS); 835 } 836 837 /*@ 838 DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions 839 and global solutions, to a local basis, appropriate for discretization integrals and assembly. 840 841 Input Parameters: 842 + dm - The `DM` 843 . alpha - The first Euler angle, and in 2D the only one 844 . beta - The second Euler angle 845 - gamma - The third Euler angle 846 847 Level: developer 848 849 Note: 850 Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that 851 we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows 852 .vb 853 The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis. 854 The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis. 855 The XYZ system rotates a third time about the z axis by gamma. 856 .ve 857 858 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMPlexLocalToGlobalBasis()` 859 @*/ 860 PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma) 861 { 862 RotCtx *rc; 863 PetscInt cdim; 864 865 PetscFunctionBegin; 866 PetscCall(DMGetCoordinateDim(dm, &cdim)); 867 PetscCall(PetscMalloc1(1, &rc)); 868 dm->transformCtx = rc; 869 dm->transformSetUp = DMPlexBasisTransformSetUp_Rotation_Internal; 870 dm->transformDestroy = DMPlexBasisTransformDestroy_Rotation_Internal; 871 dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal; 872 rc->dim = cdim; 873 rc->alpha = alpha; 874 rc->beta = beta; 875 rc->gamma = gamma; 876 PetscCall((*dm->transformSetUp)(dm, dm->transformCtx)); 877 PetscCall(DMConstructBasisTransform_Internal(dm)); 878 PetscFunctionReturn(PETSC_SUCCESS); 879 } 880 881 /*@C 882 DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates 883 884 Input Parameters: 885 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 886 . time - The time 887 . field - The field to constrain 888 . Nc - The number of constrained field components, or 0 for all components 889 . comps - An array of constrained component numbers, or `NULL` for all components 890 . label - The `DMLabel` defining constrained points 891 . numids - The number of `DMLabel` ids for constrained points 892 . ids - An array of ids for constrained points 893 . func - A pointwise function giving boundary values 894 - ctx - An optional user context for bcFunc 895 896 Output Parameter: 897 . locX - A local vector to receives the boundary values 898 899 Level: developer 900 901 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabel`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()` 902 @*/ 903 PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX) 904 { 905 PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx); 906 void **ctxs; 907 PetscInt numFields; 908 909 PetscFunctionBegin; 910 PetscCall(DMGetNumFields(dm, &numFields)); 911 PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs)); 912 funcs[field] = func; 913 ctxs[field] = ctx; 914 PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX)); 915 PetscCall(PetscFree2(funcs, ctxs)); 916 PetscFunctionReturn(PETSC_SUCCESS); 917 } 918 919 /*@C 920 DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data 921 922 Input Parameters: 923 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 924 . time - The time 925 . locU - A local vector with the input solution values 926 . field - The field to constrain 927 . Nc - The number of constrained field components, or 0 for all components 928 . comps - An array of constrained component numbers, or `NULL` for all components 929 . label - The `DMLabel` defining constrained points 930 . numids - The number of `DMLabel` ids for constrained points 931 . ids - An array of ids for constrained points 932 . func - A pointwise function giving boundary values 933 - ctx - An optional user context for bcFunc 934 935 Output Parameter: 936 . locX - A local vector to receives the boundary values 937 938 Level: developer 939 940 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()` 941 @*/ 942 PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX) 943 { 944 void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]); 945 void **ctxs; 946 PetscInt numFields; 947 948 PetscFunctionBegin; 949 PetscCall(DMGetNumFields(dm, &numFields)); 950 PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs)); 951 funcs[field] = func; 952 ctxs[field] = ctx; 953 PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX)); 954 PetscCall(PetscFree2(funcs, ctxs)); 955 PetscFunctionReturn(PETSC_SUCCESS); 956 } 957 958 /*@C 959 DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coordinates and boundary field data 960 961 Collective 962 963 Input Parameters: 964 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 965 . time - The time 966 . locU - A local vector with the input solution values 967 . field - The field to constrain 968 . Nc - The number of constrained field components, or 0 for all components 969 . comps - An array of constrained component numbers, or `NULL` for all components 970 . label - The `DMLabel` defining constrained points 971 . numids - The number of `DMLabel` ids for constrained points 972 . ids - An array of ids for constrained points 973 . func - A pointwise function giving boundary values, the calling sequence is given in `DMProjectBdFieldLabelLocal()` 974 - ctx - An optional user context for `func` 975 976 Output Parameter: 977 . locX - A local vector to receive the boundary values 978 979 Level: developer 980 981 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectBdFieldLabelLocal()`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()` 982 @*/ 983 PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX) 984 { 985 void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]); 986 void **ctxs; 987 PetscInt numFields; 988 989 PetscFunctionBegin; 990 PetscCall(DMGetNumFields(dm, &numFields)); 991 PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs)); 992 funcs[field] = func; 993 ctxs[field] = ctx; 994 PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX)); 995 PetscCall(PetscFree2(funcs, ctxs)); 996 PetscFunctionReturn(PETSC_SUCCESS); 997 } 998 999 /*@C 1000 DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector 1001 1002 Input Parameters: 1003 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 1004 . time - The time 1005 . faceGeometry - A vector with the FVM face geometry information 1006 . cellGeometry - A vector with the FVM cell geometry information 1007 . Grad - A vector with the FVM cell gradient information 1008 . field - The field to constrain 1009 . Nc - The number of constrained field components, or 0 for all components 1010 . comps - An array of constrained component numbers, or `NULL` for all components 1011 . label - The `DMLabel` defining constrained points 1012 . numids - The number of `DMLabel` ids for constrained points 1013 . ids - An array of ids for constrained points 1014 . func - A pointwise function giving boundary values 1015 - ctx - An optional user context for bcFunc 1016 1017 Output Parameter: 1018 . locX - A local vector to receives the boundary values 1019 1020 Level: developer 1021 1022 Note: 1023 This implementation currently ignores the numcomps/comps argument from `DMAddBoundary()` 1024 1025 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()` 1026 @*/ 1027 PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *), void *ctx, Vec locX) 1028 { 1029 PetscDS prob; 1030 PetscSF sf; 1031 DM dmFace, dmCell, dmGrad; 1032 const PetscScalar *facegeom, *cellgeom = NULL, *grad; 1033 const PetscInt *leaves; 1034 PetscScalar *x, *fx; 1035 PetscInt dim, nleaves, loc, fStart, fEnd, pdim, i; 1036 PetscErrorCode ierru = PETSC_SUCCESS; 1037 1038 PetscFunctionBegin; 1039 PetscCall(DMGetPointSF(dm, &sf)); 1040 PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL)); 1041 nleaves = PetscMax(0, nleaves); 1042 PetscCall(DMGetDimension(dm, &dim)); 1043 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 1044 PetscCall(DMGetDS(dm, &prob)); 1045 PetscCall(VecGetDM(faceGeometry, &dmFace)); 1046 PetscCall(VecGetArrayRead(faceGeometry, &facegeom)); 1047 if (cellGeometry) { 1048 PetscCall(VecGetDM(cellGeometry, &dmCell)); 1049 PetscCall(VecGetArrayRead(cellGeometry, &cellgeom)); 1050 } 1051 if (Grad) { 1052 PetscFV fv; 1053 1054 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fv)); 1055 PetscCall(VecGetDM(Grad, &dmGrad)); 1056 PetscCall(VecGetArrayRead(Grad, &grad)); 1057 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 1058 PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx)); 1059 } 1060 PetscCall(VecGetArray(locX, &x)); 1061 for (i = 0; i < numids; ++i) { 1062 IS faceIS; 1063 const PetscInt *faces; 1064 PetscInt numFaces, f; 1065 1066 PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS)); 1067 if (!faceIS) continue; /* No points with that id on this process */ 1068 PetscCall(ISGetLocalSize(faceIS, &numFaces)); 1069 PetscCall(ISGetIndices(faceIS, &faces)); 1070 for (f = 0; f < numFaces; ++f) { 1071 const PetscInt face = faces[f], *cells; 1072 PetscFVFaceGeom *fg; 1073 1074 if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */ 1075 PetscCall(PetscFindInt(face, nleaves, (PetscInt *)leaves, &loc)); 1076 if (loc >= 0) continue; 1077 PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg)); 1078 PetscCall(DMPlexGetSupport(dm, face, &cells)); 1079 if (Grad) { 1080 PetscFVCellGeom *cg; 1081 PetscScalar *cx, *cgrad; 1082 PetscScalar *xG; 1083 PetscReal dx[3]; 1084 PetscInt d; 1085 1086 PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg)); 1087 PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx)); 1088 PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad)); 1089 PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG)); 1090 DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx); 1091 for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d * dim], dx); 1092 PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx)); 1093 } else { 1094 PetscScalar *xI; 1095 PetscScalar *xG; 1096 1097 PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI)); 1098 PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG)); 1099 ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx); 1100 if (ierru) { 1101 PetscCall(ISRestoreIndices(faceIS, &faces)); 1102 PetscCall(ISDestroy(&faceIS)); 1103 goto cleanup; 1104 } 1105 } 1106 } 1107 PetscCall(ISRestoreIndices(faceIS, &faces)); 1108 PetscCall(ISDestroy(&faceIS)); 1109 } 1110 cleanup: 1111 PetscCall(VecRestoreArray(locX, &x)); 1112 if (Grad) { 1113 PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx)); 1114 PetscCall(VecRestoreArrayRead(Grad, &grad)); 1115 } 1116 if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom)); 1117 PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom)); 1118 PetscCall(ierru); 1119 PetscFunctionReturn(PETSC_SUCCESS); 1120 } 1121 1122 static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx) 1123 { 1124 PetscInt c; 1125 for (c = 0; c < Nc; ++c) u[c] = 0.0; 1126 return PETSC_SUCCESS; 1127 } 1128 1129 PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1130 { 1131 PetscObject isZero; 1132 PetscDS prob; 1133 PetscInt numBd, b; 1134 1135 PetscFunctionBegin; 1136 PetscCall(DMGetDS(dm, &prob)); 1137 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 1138 PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero)); 1139 PetscCall(PetscDSUpdateBoundaryLabels(prob, dm)); 1140 for (b = 0; b < numBd; ++b) { 1141 PetscWeakForm wf; 1142 DMBoundaryConditionType type; 1143 const char *name; 1144 DMLabel label; 1145 PetscInt field, Nc; 1146 const PetscInt *comps; 1147 PetscObject obj; 1148 PetscClassId id; 1149 void (*bvfunc)(void); 1150 PetscInt numids; 1151 const PetscInt *ids; 1152 void *ctx; 1153 1154 PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx)); 1155 if (insertEssential != (type & DM_BC_ESSENTIAL)) continue; 1156 PetscCall(DMGetField(dm, field, NULL, &obj)); 1157 PetscCall(PetscObjectGetClassId(obj, &id)); 1158 if (id == PETSCFE_CLASSID) { 1159 switch (type) { 1160 /* for FEM, there is no insertion to be done for non-essential boundary conditions */ 1161 case DM_BC_ESSENTIAL: { 1162 PetscSimplePointFn *func = (PetscSimplePointFn *)bvfunc; 1163 1164 if (isZero) func = zero; 1165 PetscCall(DMPlexLabelAddCells(dm, label)); 1166 PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX)); 1167 PetscCall(DMPlexLabelClearCells(dm, label)); 1168 } break; 1169 case DM_BC_ESSENTIAL_FIELD: { 1170 PetscPointFunc func = (PetscPointFunc)bvfunc; 1171 1172 PetscCall(DMPlexLabelAddCells(dm, label)); 1173 PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX)); 1174 PetscCall(DMPlexLabelClearCells(dm, label)); 1175 } break; 1176 default: 1177 break; 1178 } 1179 } else if (id == PETSCFV_CLASSID) { 1180 { 1181 PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *) = (PetscErrorCode (*)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *))bvfunc; 1182 1183 if (!faceGeomFVM) continue; 1184 PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX)); 1185 } 1186 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1187 } 1188 PetscFunctionReturn(PETSC_SUCCESS); 1189 } 1190 1191 PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1192 { 1193 PetscObject isZero; 1194 PetscDS prob; 1195 PetscInt numBd, b; 1196 1197 PetscFunctionBegin; 1198 if (!locX) PetscFunctionReturn(PETSC_SUCCESS); 1199 PetscCall(DMGetDS(dm, &prob)); 1200 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 1201 PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero)); 1202 for (b = 0; b < numBd; ++b) { 1203 PetscWeakForm wf; 1204 DMBoundaryConditionType type; 1205 const char *name; 1206 DMLabel label; 1207 PetscInt field, Nc; 1208 const PetscInt *comps; 1209 PetscObject obj; 1210 PetscClassId id; 1211 PetscInt numids; 1212 const PetscInt *ids; 1213 void (*bvfunc)(void); 1214 void *ctx; 1215 1216 PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx)); 1217 if (insertEssential != (type & DM_BC_ESSENTIAL)) continue; 1218 PetscCall(DMGetField(dm, field, NULL, &obj)); 1219 PetscCall(PetscObjectGetClassId(obj, &id)); 1220 if (id == PETSCFE_CLASSID) { 1221 switch (type) { 1222 /* for FEM, there is no insertion to be done for non-essential boundary conditions */ 1223 case DM_BC_ESSENTIAL: { 1224 PetscSimplePointFn *func_t = (PetscSimplePointFn *)bvfunc; 1225 1226 if (isZero) func_t = zero; 1227 PetscCall(DMPlexLabelAddCells(dm, label)); 1228 PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX)); 1229 PetscCall(DMPlexLabelClearCells(dm, label)); 1230 } break; 1231 case DM_BC_ESSENTIAL_FIELD: { 1232 PetscPointFunc func_t = (PetscPointFunc)bvfunc; 1233 1234 PetscCall(DMPlexLabelAddCells(dm, label)); 1235 PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX)); 1236 PetscCall(DMPlexLabelClearCells(dm, label)); 1237 } break; 1238 default: 1239 break; 1240 } 1241 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1242 } 1243 PetscFunctionReturn(PETSC_SUCCESS); 1244 } 1245 1246 /*@ 1247 DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector 1248 1249 Not Collective 1250 1251 Input Parameters: 1252 + dm - The `DM` 1253 . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions 1254 . time - The time 1255 . faceGeomFVM - Face geometry data for FV discretizations 1256 . cellGeomFVM - Cell geometry data for FV discretizations 1257 - gradFVM - Gradient reconstruction data for FV discretizations 1258 1259 Output Parameter: 1260 . locX - Solution updated with boundary values 1261 1262 Level: intermediate 1263 1264 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`, `DMAddBoundary()` 1265 @*/ 1266 PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1267 { 1268 PetscFunctionBegin; 1269 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1270 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 1271 if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5); 1272 if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6); 1273 if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7); 1274 PetscTryMethod(dm, "DMPlexInsertBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX, time, faceGeomFVM, cellGeomFVM, gradFVM)); 1275 PetscFunctionReturn(PETSC_SUCCESS); 1276 } 1277 1278 /*@ 1279 DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector 1280 1281 Input Parameters: 1282 + dm - The `DM` 1283 . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions 1284 . time - The time 1285 . faceGeomFVM - Face geometry data for FV discretizations 1286 . cellGeomFVM - Cell geometry data for FV discretizations 1287 - gradFVM - Gradient reconstruction data for FV discretizations 1288 1289 Output Parameter: 1290 . locX_t - Solution updated with boundary values 1291 1292 Level: developer 1293 1294 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()` 1295 @*/ 1296 PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1297 { 1298 PetscFunctionBegin; 1299 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1300 if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3); 1301 if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5); 1302 if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6); 1303 if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7); 1304 PetscTryMethod(dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX_t, time, faceGeomFVM, cellGeomFVM, gradFVM)); 1305 PetscFunctionReturn(PETSC_SUCCESS); 1306 } 1307 1308 // Handle non-essential (e.g. outflow) boundary values 1309 PetscErrorCode DMPlexInsertBoundaryValuesFVM(DM dm, PetscFV fv, Vec locX, PetscReal time, Vec *locGradient) 1310 { 1311 DM dmGrad; 1312 Vec cellGeometryFVM, faceGeometryFVM, locGrad = NULL; 1313 1314 PetscFunctionBegin; 1315 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1316 PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2); 1317 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 1318 if (locGradient) { 1319 PetscAssertPointer(locGradient, 5); 1320 *locGradient = NULL; 1321 } 1322 PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL)); 1323 /* Reconstruct and limit cell gradients */ 1324 PetscCall(DMPlexGetGradientDM(dm, fv, &dmGrad)); 1325 if (dmGrad) { 1326 Vec grad; 1327 PetscInt fStart, fEnd; 1328 1329 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 1330 PetscCall(DMGetGlobalVector(dmGrad, &grad)); 1331 PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad)); 1332 /* Communicate gradient values */ 1333 PetscCall(DMGetLocalVector(dmGrad, &locGrad)); 1334 PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad)); 1335 PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad)); 1336 PetscCall(DMRestoreGlobalVector(dmGrad, &grad)); 1337 } 1338 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad)); 1339 if (locGradient) *locGradient = locGrad; 1340 else if (locGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad)); 1341 PetscFunctionReturn(PETSC_SUCCESS); 1342 } 1343 1344 PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff) 1345 { 1346 Vec localX; 1347 1348 PetscFunctionBegin; 1349 PetscCall(DMGetLocalVector(dm, &localX)); 1350 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL)); 1351 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1352 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1353 PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff)); 1354 PetscCall(DMRestoreLocalVector(dm, &localX)); 1355 PetscFunctionReturn(PETSC_SUCCESS); 1356 } 1357 1358 /*@C 1359 DMPlexComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h. 1360 1361 Collective 1362 1363 Input Parameters: 1364 + dm - The `DM` 1365 . time - The time 1366 . funcs - The functions to evaluate for each field component 1367 . ctxs - Optional array of contexts to pass to each function, or `NULL`. 1368 - localX - The coefficient vector u_h, a local vector 1369 1370 Output Parameter: 1371 . diff - The diff ||u - u_h||_2 1372 1373 Level: developer 1374 1375 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1376 @*/ 1377 PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff) 1378 { 1379 const PetscInt debug = ((DM_Plex *)dm->data)->printL2; 1380 DM tdm; 1381 Vec tv; 1382 PetscSection section; 1383 PetscQuadrature quad; 1384 PetscFEGeom fegeom; 1385 PetscScalar *funcVal, *interpolant; 1386 PetscReal *coords, *gcoords; 1387 PetscReal localDiff = 0.0; 1388 const PetscReal *quadWeights; 1389 PetscInt dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset; 1390 PetscBool transform; 1391 1392 PetscFunctionBegin; 1393 PetscCall(DMGetDimension(dm, &dim)); 1394 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1395 fegeom.dimEmbed = coordDim; 1396 PetscCall(DMGetLocalSection(dm, §ion)); 1397 PetscCall(PetscSectionGetNumFields(section, &numFields)); 1398 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 1399 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 1400 PetscCall(DMHasBasisTransform(dm, &transform)); 1401 PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!"); 1402 for (field = 0; field < numFields; ++field) { 1403 PetscObject obj; 1404 PetscClassId id; 1405 PetscInt Nc; 1406 1407 PetscCall(DMGetField(dm, field, NULL, &obj)); 1408 PetscCall(PetscObjectGetClassId(obj, &id)); 1409 if (id == PETSCFE_CLASSID) { 1410 PetscFE fe = (PetscFE)obj; 1411 1412 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1413 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1414 } else if (id == PETSCFV_CLASSID) { 1415 PetscFV fv = (PetscFV)obj; 1416 1417 PetscCall(PetscFVGetQuadrature(fv, &quad)); 1418 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 1419 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1420 numComponents += Nc; 1421 } 1422 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights)); 1423 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1424 PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * (Nq + 1), &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ)); 1425 PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight)); 1426 PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd)); 1427 for (c = cStart; c < cEnd; ++c) { 1428 PetscScalar *x = NULL; 1429 PetscReal elemDiff = 0.0; 1430 PetscInt qc = 0; 1431 1432 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1433 PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x)); 1434 1435 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1436 PetscObject obj; 1437 PetscClassId id; 1438 void *const ctx = ctxs ? ctxs[field] : NULL; 1439 PetscInt Nb, Nc, q, fc; 1440 1441 PetscCall(DMGetField(dm, field, NULL, &obj)); 1442 PetscCall(PetscObjectGetClassId(obj, &id)); 1443 if (id == PETSCFE_CLASSID) { 1444 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1445 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1446 } else if (id == PETSCFV_CLASSID) { 1447 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1448 Nb = 1; 1449 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1450 if (debug) { 1451 char title[1024]; 1452 PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field)); 1453 PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset])); 1454 } 1455 for (q = 0; q < Nq; ++q) { 1456 PetscFEGeom qgeom; 1457 PetscErrorCode ierr; 1458 1459 qgeom.dimEmbed = fegeom.dimEmbed; 1460 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1461 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1462 qgeom.detJ = &fegeom.detJ[q]; 1463 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1464 if (transform) { 1465 gcoords = &coords[coordDim * Nq]; 1466 PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx)); 1467 } else { 1468 gcoords = &coords[coordDim * q]; 1469 } 1470 PetscCall(PetscArrayzero(funcVal, Nc)); 1471 ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx); 1472 if (ierr) { 1473 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1474 PetscCall(DMRestoreLocalVector(dm, &localX)); 1475 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1476 } 1477 if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx)); 1478 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant)); 1479 else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant)); 1480 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1481 for (fc = 0; fc < Nc; ++fc) { 1482 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1483 if (debug) 1484 PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim * q] : 0), (double)(coordDim > 1 ? coords[coordDim * q + 1] : 0), (double)(coordDim > 2 ? coords[coordDim * q + 2] : 0), 1485 (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc]))); 1486 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1487 } 1488 } 1489 fieldOffset += Nb; 1490 qc += Nc; 1491 } 1492 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1493 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff)); 1494 localDiff += elemDiff; 1495 } 1496 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1497 PetscCallMPI(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 1498 *diff = PetscSqrtReal(*diff); 1499 PetscFunctionReturn(PETSC_SUCCESS); 1500 } 1501 1502 PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff) 1503 { 1504 const PetscInt debug = ((DM_Plex *)dm->data)->printL2; 1505 DM tdm; 1506 PetscSection section; 1507 PetscQuadrature quad; 1508 Vec localX, tv; 1509 PetscScalar *funcVal, *interpolant; 1510 const PetscReal *quadWeights; 1511 PetscFEGeom fegeom; 1512 PetscReal *coords, *gcoords; 1513 PetscReal localDiff = 0.0; 1514 PetscInt dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset; 1515 PetscBool transform; 1516 1517 PetscFunctionBegin; 1518 PetscCall(DMGetDimension(dm, &dim)); 1519 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1520 fegeom.dimEmbed = coordDim; 1521 PetscCall(DMGetLocalSection(dm, §ion)); 1522 PetscCall(PetscSectionGetNumFields(section, &numFields)); 1523 PetscCall(DMGetLocalVector(dm, &localX)); 1524 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1525 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1526 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 1527 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 1528 PetscCall(DMHasBasisTransform(dm, &transform)); 1529 for (field = 0; field < numFields; ++field) { 1530 PetscFE fe; 1531 PetscInt Nc; 1532 1533 PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe)); 1534 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1535 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1536 numComponents += Nc; 1537 } 1538 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights)); 1539 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1540 /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */ 1541 PetscCall(PetscMalloc6(numComponents, &funcVal, coordDim * (Nq + 1), &coords, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ, numComponents * coordDim, &interpolant, Nq, &fegeom.detJ)); 1542 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1543 for (c = cStart; c < cEnd; ++c) { 1544 PetscScalar *x = NULL; 1545 PetscReal elemDiff = 0.0; 1546 PetscInt qc = 0; 1547 1548 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1549 PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x)); 1550 1551 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1552 PetscFE fe; 1553 void *const ctx = ctxs ? ctxs[field] : NULL; 1554 PetscInt Nb, Nc, q, fc; 1555 1556 PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe)); 1557 PetscCall(PetscFEGetDimension(fe, &Nb)); 1558 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1559 if (debug) { 1560 char title[1024]; 1561 PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field)); 1562 PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset])); 1563 } 1564 for (q = 0; q < Nq; ++q) { 1565 PetscFEGeom qgeom; 1566 PetscErrorCode ierr; 1567 1568 qgeom.dimEmbed = fegeom.dimEmbed; 1569 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1570 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1571 qgeom.detJ = &fegeom.detJ[q]; 1572 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1573 if (transform) { 1574 gcoords = &coords[coordDim * Nq]; 1575 PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx)); 1576 } else { 1577 gcoords = &coords[coordDim * q]; 1578 } 1579 PetscCall(PetscArrayzero(funcVal, Nc)); 1580 ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx); 1581 if (ierr) { 1582 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1583 PetscCall(DMRestoreLocalVector(dm, &localX)); 1584 PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ)); 1585 } 1586 if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx)); 1587 PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant)); 1588 /* Overwrite with the dot product if the normal is given */ 1589 if (n) { 1590 for (fc = 0; fc < Nc; ++fc) { 1591 PetscScalar sum = 0.0; 1592 PetscInt d; 1593 for (d = 0; d < dim; ++d) sum += interpolant[fc * dim + d] * n[d]; 1594 interpolant[fc] = sum; 1595 } 1596 } 1597 for (fc = 0; fc < Nc; ++fc) { 1598 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1599 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]))); 1600 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1601 } 1602 } 1603 fieldOffset += Nb; 1604 qc += Nc; 1605 } 1606 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1607 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff)); 1608 localDiff += elemDiff; 1609 } 1610 PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ)); 1611 PetscCall(DMRestoreLocalVector(dm, &localX)); 1612 PetscCallMPI(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 1613 *diff = PetscSqrtReal(*diff); 1614 PetscFunctionReturn(PETSC_SUCCESS); 1615 } 1616 1617 PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff) 1618 { 1619 const PetscInt debug = ((DM_Plex *)dm->data)->printL2; 1620 DM tdm; 1621 DMLabel depthLabel; 1622 PetscSection section; 1623 Vec localX, tv; 1624 PetscReal *localDiff; 1625 PetscInt dim, depth, dE, Nf, f, Nds, s; 1626 PetscBool transform; 1627 1628 PetscFunctionBegin; 1629 PetscCall(DMGetDimension(dm, &dim)); 1630 PetscCall(DMGetCoordinateDim(dm, &dE)); 1631 PetscCall(DMGetLocalSection(dm, §ion)); 1632 PetscCall(DMGetLocalVector(dm, &localX)); 1633 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 1634 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 1635 PetscCall(DMHasBasisTransform(dm, &transform)); 1636 PetscCall(DMGetNumFields(dm, &Nf)); 1637 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 1638 PetscCall(DMLabelGetNumValues(depthLabel, &depth)); 1639 1640 PetscCall(VecSet(localX, 0.0)); 1641 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1642 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1643 PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX)); 1644 PetscCall(DMGetNumDS(dm, &Nds)); 1645 PetscCall(PetscCalloc1(Nf, &localDiff)); 1646 for (s = 0; s < Nds; ++s) { 1647 PetscDS ds; 1648 DMLabel label; 1649 IS fieldIS, pointIS; 1650 const PetscInt *fields, *points = NULL; 1651 PetscQuadrature quad; 1652 const PetscReal *quadPoints, *quadWeights; 1653 PetscFEGeom fegeom; 1654 PetscReal *coords, *gcoords; 1655 PetscScalar *funcVal, *interpolant; 1656 PetscBool isCohesive; 1657 PetscInt qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf; 1658 1659 PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds, NULL)); 1660 PetscCall(ISGetIndices(fieldIS, &fields)); 1661 PetscCall(PetscDSIsCohesive(ds, &isCohesive)); 1662 PetscCall(PetscDSGetNumFields(ds, &dsNf)); 1663 PetscCall(PetscDSGetTotalComponents(ds, &totNc)); 1664 PetscCall(PetscDSGetQuadrature(ds, &quad)); 1665 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 1666 PetscCheck(!(qNc != 1) || !(qNc != totNc), PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc); 1667 PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE * (Nq + 1), &coords, Nq, &fegeom.detJ, dE * dE * Nq, &fegeom.J, dE * dE * Nq, &fegeom.invJ)); 1668 if (!label) { 1669 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1670 } else { 1671 PetscCall(DMLabelGetStratumIS(label, 1, &pointIS)); 1672 PetscCall(ISGetLocalSize(pointIS, &cEnd)); 1673 PetscCall(ISGetIndices(pointIS, &points)); 1674 } 1675 for (c = cStart; c < cEnd; ++c) { 1676 const PetscInt cell = points ? points[c] : c; 1677 PetscScalar *x = NULL; 1678 const PetscInt *cone; 1679 PetscInt qc = 0, fOff = 0, dep; 1680 1681 PetscCall(DMLabelGetValue(depthLabel, cell, &dep)); 1682 if (dep != depth - 1) continue; 1683 if (isCohesive) { 1684 PetscCall(DMPlexGetCone(dm, cell, &cone)); 1685 PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1686 } else { 1687 PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1688 } 1689 PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, cell, 0, NULL, &x)); 1690 for (f = 0; f < dsNf; ++f) { 1691 PetscObject obj; 1692 PetscClassId id; 1693 void *const ctx = ctxs ? ctxs[fields[f]] : NULL; 1694 PetscInt Nb, Nc, q, fc; 1695 PetscReal elemDiff = 0.0; 1696 PetscBool cohesive; 1697 1698 PetscCall(PetscDSGetCohesive(ds, f, &cohesive)); 1699 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 1700 PetscCall(PetscObjectGetClassId(obj, &id)); 1701 if (id == PETSCFE_CLASSID) { 1702 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1703 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1704 } else if (id == PETSCFV_CLASSID) { 1705 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1706 Nb = 1; 1707 } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]); 1708 if (isCohesive && !cohesive) { 1709 fOff += Nb * 2; 1710 qc += Nc; 1711 continue; 1712 } 1713 if (debug) { 1714 char title[1024]; 1715 PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f])); 1716 PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff])); 1717 } 1718 for (q = 0; q < Nq; ++q) { 1719 PetscFEGeom qgeom; 1720 PetscErrorCode ierr; 1721 1722 qgeom.dimEmbed = fegeom.dimEmbed; 1723 qgeom.J = &fegeom.J[q * dE * dE]; 1724 qgeom.invJ = &fegeom.invJ[q * dE * dE]; 1725 qgeom.detJ = &fegeom.detJ[q]; 1726 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q); 1727 if (transform) { 1728 gcoords = &coords[dE * Nq]; 1729 PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE * q], PETSC_TRUE, dE, &coords[dE * q], gcoords, dm->transformCtx)); 1730 } else { 1731 gcoords = &coords[dE * q]; 1732 } 1733 for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.; 1734 ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx); 1735 if (ierr) { 1736 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x)); 1737 PetscCall(DMRestoreLocalVector(dm, &localX)); 1738 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1739 } 1740 if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx)); 1741 /* Call once for each face, except for lagrange field */ 1742 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fOff], &qgeom, q, interpolant)); 1743 else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fOff], q, interpolant)); 1744 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]); 1745 for (fc = 0; fc < Nc; ++fc) { 1746 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1747 if (debug) 1748 PetscCall(PetscPrintf(PETSC_COMM_SELF, " cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE * q] : 0), (double)(dE > 1 ? coords[dE * q + 1] : 0), (double)(dE > 2 ? coords[dE * q + 2] : 0), 1749 (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]))); 1750 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1751 } 1752 } 1753 fOff += Nb; 1754 qc += Nc; 1755 localDiff[fields[f]] += elemDiff; 1756 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]])); 1757 } 1758 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x)); 1759 } 1760 if (label) { 1761 PetscCall(ISRestoreIndices(pointIS, &points)); 1762 PetscCall(ISDestroy(&pointIS)); 1763 } 1764 PetscCall(ISRestoreIndices(fieldIS, &fields)); 1765 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1766 } 1767 PetscCall(DMRestoreLocalVector(dm, &localX)); 1768 PetscCallMPI(MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 1769 PetscCall(PetscFree(localDiff)); 1770 for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]); 1771 PetscFunctionReturn(PETSC_SUCCESS); 1772 } 1773 1774 /*@C 1775 DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec. 1776 1777 Collective 1778 1779 Input Parameters: 1780 + dm - The `DM` 1781 . time - The time 1782 . funcs - The functions to evaluate for each field component: `NULL` means that component does not contribute to error calculation 1783 . ctxs - Optional array of contexts to pass to each function, or `NULL`. 1784 - X - The coefficient vector u_h 1785 1786 Output Parameter: 1787 . D - A `Vec` which holds the difference ||u - u_h||_2 for each cell 1788 1789 Level: developer 1790 1791 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1792 @*/ 1793 PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D) 1794 { 1795 PetscSection section; 1796 PetscQuadrature quad; 1797 Vec localX; 1798 PetscFEGeom fegeom; 1799 PetscScalar *funcVal, *interpolant; 1800 PetscReal *coords; 1801 const PetscReal *quadPoints, *quadWeights; 1802 PetscInt dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset; 1803 1804 PetscFunctionBegin; 1805 PetscCall(VecSet(D, 0.0)); 1806 PetscCall(DMGetDimension(dm, &dim)); 1807 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1808 PetscCall(DMGetLocalSection(dm, §ion)); 1809 PetscCall(PetscSectionGetNumFields(section, &numFields)); 1810 PetscCall(DMGetLocalVector(dm, &localX)); 1811 PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX)); 1812 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1813 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1814 for (field = 0; field < numFields; ++field) { 1815 PetscObject obj; 1816 PetscClassId id; 1817 PetscInt Nc; 1818 1819 PetscCall(DMGetField(dm, field, NULL, &obj)); 1820 PetscCall(PetscObjectGetClassId(obj, &id)); 1821 if (id == PETSCFE_CLASSID) { 1822 PetscFE fe = (PetscFE)obj; 1823 1824 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1825 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1826 } else if (id == PETSCFV_CLASSID) { 1827 PetscFV fv = (PetscFV)obj; 1828 1829 PetscCall(PetscFVGetQuadrature(fv, &quad)); 1830 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 1831 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1832 numComponents += Nc; 1833 } 1834 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 1835 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1836 PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ)); 1837 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1838 for (c = cStart; c < cEnd; ++c) { 1839 PetscScalar *x = NULL; 1840 PetscScalar elemDiff = 0.0; 1841 PetscInt qc = 0; 1842 1843 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1844 PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, NULL, PETSC_FALSE, localX, c, 0, NULL, &x)); 1845 1846 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1847 PetscObject obj; 1848 PetscClassId id; 1849 void *const ctx = ctxs ? ctxs[field] : NULL; 1850 PetscInt Nb, Nc, q, fc; 1851 1852 PetscCall(DMGetField(dm, field, NULL, &obj)); 1853 PetscCall(PetscObjectGetClassId(obj, &id)); 1854 if (id == PETSCFE_CLASSID) { 1855 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1856 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1857 } else if (id == PETSCFV_CLASSID) { 1858 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1859 Nb = 1; 1860 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1861 if (funcs[field]) { 1862 for (q = 0; q < Nq; ++q) { 1863 PetscFEGeom qgeom; 1864 1865 qgeom.dimEmbed = fegeom.dimEmbed; 1866 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1867 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1868 qgeom.detJ = &fegeom.detJ[q]; 1869 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1870 PetscCall((*funcs[field])(coordDim, time, &coords[q * coordDim], Nc, funcVal, ctx)); 1871 #if defined(needs_fix_with_return_code_argument) 1872 if (ierr) { 1873 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1874 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1875 PetscCall(DMRestoreLocalVector(dm, &localX)); 1876 } 1877 #endif 1878 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant)); 1879 else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant)); 1880 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1881 for (fc = 0; fc < Nc; ++fc) { 1882 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1883 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1884 } 1885 } 1886 } 1887 fieldOffset += Nb; 1888 qc += Nc; 1889 } 1890 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1891 PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES)); 1892 } 1893 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1894 PetscCall(DMRestoreLocalVector(dm, &localX)); 1895 PetscCall(VecSqrtAbs(D)); 1896 PetscFunctionReturn(PETSC_SUCCESS); 1897 } 1898 1899 /*@ 1900 DMPlexComputeL2FluxDiffVecLocal - This function computes the integral of the difference between the gradient of field `f`in `u` and field `mf` in `mu` 1901 1902 Collective 1903 1904 Input Parameters: 1905 + lu - The local `Vec` containing the primal solution 1906 . f - The field number for the potential 1907 . lmu - The local `Vec` containing the mixed solution 1908 - mf - The field number for the flux 1909 1910 Output Parameter: 1911 . eFlux - A global `Vec` which holds $||\nabla u_f - \mu_{mf}||$ 1912 1913 Level: advanced 1914 1915 Notes: 1916 We assume that the `DM` for each solution has the same topology, geometry, and quadrature. 1917 1918 This is usually used to get an error estimate for the primal solution, using the flux from a mixed solution. 1919 1920 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeL2FluxDiffVec()`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1921 @*/ 1922 PetscErrorCode DMPlexComputeL2FluxDiffVecLocal(Vec lu, PetscInt f, Vec lmu, PetscInt mf, Vec eFlux) 1923 { 1924 DM dm, mdm, edm; 1925 PetscFE fe, mfe; 1926 PetscFEGeom fegeom; 1927 PetscQuadrature quad; 1928 const PetscReal *quadWeights; 1929 PetscReal *coords; 1930 PetscScalar *interpolant, *minterpolant, *earray; 1931 PetscInt cdim, mcdim, cStart, cEnd, Nc, mNc, qNc, Nq; 1932 MPI_Comm comm; 1933 1934 PetscFunctionBegin; 1935 PetscCall(VecGetDM(lu, &dm)); 1936 PetscCall(VecGetDM(lmu, &mdm)); 1937 PetscCall(VecGetDM(eFlux, &edm)); 1938 PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); 1939 PetscCall(VecSet(eFlux, 0.0)); 1940 1941 // Check if the both problems are on the same mesh 1942 PetscCall(DMGetCoordinateDim(dm, &cdim)); 1943 PetscCall(DMGetCoordinateDim(mdm, &mcdim)); 1944 PetscCheck(cdim == mcdim, comm, PETSC_ERR_ARG_SIZ, "primal coordinate Dim %" PetscInt_FMT " != %" PetscInt_FMT " mixed coordinate Dim", cdim, mcdim); 1945 fegeom.dimEmbed = cdim; 1946 1947 PetscCall(DMGetField(dm, f, NULL, (PetscObject *)&fe)); 1948 PetscCall(DMGetField(mdm, mf, NULL, (PetscObject *)&mfe)); 1949 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1950 PetscCall(PetscFEGetNumComponents(mfe, &mNc)); 1951 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1952 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights)); 1953 PetscCheck(qNc == 1 || qNc == mNc, comm, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, mNc); 1954 1955 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1956 PetscCall(VecGetArrayWrite(eFlux, &earray)); 1957 PetscCall(PetscMalloc6(Nc * cdim, &interpolant, mNc * cdim, &minterpolant, cdim * (Nq + 1), &coords, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ, Nq, &fegeom.detJ)); 1958 for (PetscInt c = cStart; c < cEnd; ++c) { 1959 PetscScalar *x = NULL; 1960 PetscScalar *mx = NULL; 1961 PetscScalar *eval = NULL; 1962 PetscReal fluxElemDiff = 0.0; 1963 1964 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1965 PetscCall(DMPlexVecGetClosure(dm, NULL, lu, c, NULL, &x)); 1966 PetscCall(DMPlexVecGetClosure(mdm, NULL, lmu, c, NULL, &mx)); 1967 1968 for (PetscInt q = 0; q < Nq; ++q) { 1969 PetscFEGeom qgeom; 1970 1971 qgeom.dimEmbed = fegeom.dimEmbed; 1972 qgeom.J = &fegeom.J[q * cdim * cdim]; 1973 qgeom.invJ = &fegeom.invJ[q * cdim * cdim]; 1974 qgeom.detJ = &fegeom.detJ[q]; 1975 1976 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1977 1978 PetscCall(PetscFEInterpolate_Static(mfe, &mx[0], &qgeom, q, minterpolant)); 1979 PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[0], &qgeom, q, interpolant)); 1980 1981 /* Now take the elementwise difference and store that in a vector. */ 1982 for (PetscInt fc = 0; fc < mNc; ++fc) { 1983 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : fc)]; 1984 fluxElemDiff += PetscSqr(PetscRealPart(interpolant[fc] - minterpolant[fc])) * wt * fegeom.detJ[q]; 1985 } 1986 } 1987 PetscCall(DMPlexVecRestoreClosure(dm, NULL, lu, c, NULL, &x)); 1988 PetscCall(DMPlexVecRestoreClosure(mdm, NULL, lmu, c, NULL, &mx)); 1989 PetscCall(DMPlexPointGlobalRef(edm, c, earray, (void *)&eval)); 1990 if (eval) eval[0] = fluxElemDiff; 1991 } 1992 PetscCall(PetscFree6(interpolant, minterpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1993 PetscCall(VecRestoreArrayWrite(eFlux, &earray)); 1994 1995 PetscCall(VecAssemblyBegin(eFlux)); 1996 PetscCall(VecAssemblyEnd(eFlux)); 1997 PetscCall(VecSqrtAbs(eFlux)); 1998 PetscFunctionReturn(PETSC_SUCCESS); 1999 } 2000 2001 /*@ 2002 DMPlexComputeL2FluxDiffVec - This function computes the integral of the difference between the gradient of field `f`in `u` and field `mf` in `mu` 2003 2004 Collective 2005 2006 Input Parameters: 2007 + u - The global `Vec` containing the primal solution 2008 . f - The field number for the potential 2009 . mu - The global `Vec` containing the mixed solution 2010 - mf - The field number for the flux 2011 2012 Output Parameter: 2013 . eFlux - A global `Vec` which holds $||\nabla u_f - \mu_{mf}||$ 2014 2015 Level: advanced 2016 2017 Notes: 2018 We assume that the `DM` for each solution has the same topology, geometry, and quadrature. 2019 2020 This is usually used to get an error estimate for the primal solution, using the flux from a mixed solution. 2021 2022 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeL2FluxDiffVecLocal()`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 2023 @*/ 2024 PetscErrorCode DMPlexComputeL2FluxDiffVec(Vec u, PetscInt f, Vec mu, PetscInt mf, Vec eFlux) 2025 { 2026 DM dm, mdm; 2027 Vec lu, lmu; 2028 2029 PetscFunctionBegin; 2030 PetscCall(VecGetDM(u, &dm)); 2031 PetscCall(DMGetLocalVector(dm, &lu)); 2032 PetscCall(DMGlobalToLocal(dm, u, INSERT_VALUES, lu)); 2033 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, lu, 0.0, NULL, NULL, NULL)); 2034 2035 PetscCall(VecGetDM(mu, &mdm)); 2036 PetscCall(DMGetLocalVector(mdm, &lmu)); 2037 PetscCall(DMGlobalToLocal(mdm, mu, INSERT_VALUES, lmu)); 2038 PetscCall(DMPlexInsertBoundaryValues(mdm, PETSC_TRUE, lmu, 0.0, NULL, NULL, NULL)); 2039 2040 PetscCall(DMPlexComputeL2FluxDiffVecLocal(lu, f, lmu, mf, eFlux)); 2041 2042 PetscCall(DMRestoreLocalVector(dm, &lu)); 2043 PetscCall(DMRestoreLocalVector(mdm, &lmu)); 2044 PetscFunctionReturn(PETSC_SUCCESS); 2045 } 2046 2047 /*@ 2048 DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1 2049 2050 Collective 2051 2052 Input Parameters: 2053 + dm - The `DM` 2054 - locX - The coefficient vector u_h 2055 2056 Output Parameter: 2057 . locC - A `Vec` which holds the Clement interpolant of the function 2058 2059 Level: developer 2060 2061 Note: 2062 $ u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume 2063 2064 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 2065 @*/ 2066 PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC) 2067 { 2068 PetscInt debug = ((DM_Plex *)dm->data)->printFEM; 2069 DM dmc; 2070 PetscQuadrature quad; 2071 PetscScalar *interpolant, *valsum; 2072 PetscFEGeom fegeom; 2073 PetscReal *coords; 2074 const PetscReal *quadPoints, *quadWeights; 2075 PetscInt dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v; 2076 2077 PetscFunctionBegin; 2078 PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite)); 2079 PetscCall(VecGetDM(locC, &dmc)); 2080 PetscCall(VecSet(locC, 0.0)); 2081 PetscCall(DMGetDimension(dm, &dim)); 2082 PetscCall(DMGetCoordinateDim(dm, &cdim)); 2083 fegeom.dimEmbed = cdim; 2084 PetscCall(DMGetNumFields(dm, &Nf)); 2085 PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!"); 2086 for (f = 0; f < Nf; ++f) { 2087 PetscObject obj; 2088 PetscClassId id; 2089 PetscInt fNc; 2090 2091 PetscCall(DMGetField(dm, f, NULL, &obj)); 2092 PetscCall(PetscObjectGetClassId(obj, &id)); 2093 if (id == PETSCFE_CLASSID) { 2094 PetscFE fe = (PetscFE)obj; 2095 2096 PetscCall(PetscFEGetQuadrature(fe, &quad)); 2097 PetscCall(PetscFEGetNumComponents(fe, &fNc)); 2098 } else if (id == PETSCFV_CLASSID) { 2099 PetscFV fv = (PetscFV)obj; 2100 2101 PetscCall(PetscFVGetQuadrature(fv, &quad)); 2102 PetscCall(PetscFVGetNumComponents(fv, &fNc)); 2103 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 2104 Nc += fNc; 2105 } 2106 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 2107 PetscCheck(qNc == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc); 2108 PetscCall(PetscMalloc6(Nc * 2, &valsum, Nc, &interpolant, cdim * Nq, &coords, Nq, &fegeom.detJ, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ)); 2109 PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd)); 2110 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 2111 for (v = vStart; v < vEnd; ++v) { 2112 PetscScalar volsum = 0.0; 2113 PetscInt *star = NULL; 2114 PetscInt starSize, st, fc; 2115 2116 PetscCall(PetscArrayzero(valsum, Nc)); 2117 PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 2118 for (st = 0; st < starSize * 2; st += 2) { 2119 const PetscInt cell = star[st]; 2120 PetscScalar *val = &valsum[Nc]; 2121 PetscScalar *x = NULL; 2122 PetscReal vol = 0.0; 2123 PetscInt foff = 0; 2124 2125 if ((cell < cStart) || (cell >= cEnd)) continue; 2126 PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 2127 PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x)); 2128 for (f = 0; f < Nf; ++f) { 2129 PetscObject obj; 2130 PetscClassId id; 2131 PetscInt Nb, fNc, q; 2132 2133 PetscCall(PetscArrayzero(val, Nc)); 2134 PetscCall(DMGetField(dm, f, NULL, &obj)); 2135 PetscCall(PetscObjectGetClassId(obj, &id)); 2136 if (id == PETSCFE_CLASSID) { 2137 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &fNc)); 2138 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 2139 } else if (id == PETSCFV_CLASSID) { 2140 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &fNc)); 2141 Nb = 1; 2142 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 2143 for (q = 0; q < Nq; ++q) { 2144 const PetscReal wt = quadWeights[q] * fegeom.detJ[q]; 2145 PetscFEGeom qgeom; 2146 2147 qgeom.dimEmbed = fegeom.dimEmbed; 2148 qgeom.J = &fegeom.J[q * cdim * cdim]; 2149 qgeom.invJ = &fegeom.invJ[q * cdim * cdim]; 2150 qgeom.detJ = &fegeom.detJ[q]; 2151 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q); 2152 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[foff], &qgeom, q, interpolant)); 2153 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 2154 for (fc = 0; fc < fNc; ++fc) val[foff + fc] += interpolant[fc] * wt; 2155 vol += wt; 2156 } 2157 foff += Nb; 2158 } 2159 PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x)); 2160 for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc]; 2161 volsum += vol; 2162 if (debug) { 2163 PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell)); 2164 for (fc = 0; fc < Nc; ++fc) { 2165 if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", ")); 2166 PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(val[fc]))); 2167 } 2168 PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n")); 2169 } 2170 } 2171 for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum; 2172 PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 2173 PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES)); 2174 } 2175 PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 2176 PetscFunctionReturn(PETSC_SUCCESS); 2177 } 2178 2179 /*@ 2180 DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1 2181 2182 Collective 2183 2184 Input Parameters: 2185 + dm - The `DM` 2186 - locX - The coefficient vector u_h 2187 2188 Output Parameter: 2189 . locC - A `Vec` which holds the Clement interpolant of the gradient 2190 2191 Level: developer 2192 2193 Note: 2194 $\nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume 2195 2196 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 2197 @*/ 2198 PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC) 2199 { 2200 DM_Plex *mesh = (DM_Plex *)dm->data; 2201 PetscInt debug = mesh->printFEM; 2202 DM dmC; 2203 PetscQuadrature quad; 2204 PetscScalar *interpolant, *gradsum; 2205 PetscFEGeom fegeom; 2206 PetscReal *coords; 2207 const PetscReal *quadPoints, *quadWeights; 2208 PetscInt dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset; 2209 2210 PetscFunctionBegin; 2211 PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite)); 2212 PetscCall(VecGetDM(locC, &dmC)); 2213 PetscCall(VecSet(locC, 0.0)); 2214 PetscCall(DMGetDimension(dm, &dim)); 2215 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 2216 fegeom.dimEmbed = coordDim; 2217 PetscCall(DMGetNumFields(dm, &numFields)); 2218 PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!"); 2219 for (field = 0; field < numFields; ++field) { 2220 PetscObject obj; 2221 PetscClassId id; 2222 PetscInt Nc; 2223 2224 PetscCall(DMGetField(dm, field, NULL, &obj)); 2225 PetscCall(PetscObjectGetClassId(obj, &id)); 2226 if (id == PETSCFE_CLASSID) { 2227 PetscFE fe = (PetscFE)obj; 2228 2229 PetscCall(PetscFEGetQuadrature(fe, &quad)); 2230 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 2231 } else if (id == PETSCFV_CLASSID) { 2232 PetscFV fv = (PetscFV)obj; 2233 2234 PetscCall(PetscFVGetQuadrature(fv, &quad)); 2235 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 2236 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 2237 numComponents += Nc; 2238 } 2239 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 2240 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 2241 PetscCall(PetscMalloc6(coordDim * numComponents * 2, &gradsum, coordDim * numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ)); 2242 PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd)); 2243 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 2244 for (v = vStart; v < vEnd; ++v) { 2245 PetscScalar volsum = 0.0; 2246 PetscInt *star = NULL; 2247 PetscInt starSize, st, d, fc; 2248 2249 PetscCall(PetscArrayzero(gradsum, coordDim * numComponents)); 2250 PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 2251 for (st = 0; st < starSize * 2; st += 2) { 2252 const PetscInt cell = star[st]; 2253 PetscScalar *grad = &gradsum[coordDim * numComponents]; 2254 PetscScalar *x = NULL; 2255 PetscReal vol = 0.0; 2256 2257 if ((cell < cStart) || (cell >= cEnd)) continue; 2258 PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 2259 PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x)); 2260 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 2261 PetscObject obj; 2262 PetscClassId id; 2263 PetscInt Nb, Nc, q, qc = 0; 2264 2265 PetscCall(PetscArrayzero(grad, coordDim * numComponents)); 2266 PetscCall(DMGetField(dm, field, NULL, &obj)); 2267 PetscCall(PetscObjectGetClassId(obj, &id)); 2268 if (id == PETSCFE_CLASSID) { 2269 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 2270 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 2271 } else if (id == PETSCFV_CLASSID) { 2272 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 2273 Nb = 1; 2274 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 2275 for (q = 0; q < Nq; ++q) { 2276 PetscFEGeom qgeom; 2277 2278 qgeom.dimEmbed = fegeom.dimEmbed; 2279 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 2280 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 2281 qgeom.detJ = &fegeom.detJ[q]; 2282 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q); 2283 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolateGradient_Static((PetscFE)obj, 1, &x[fieldOffset], &qgeom, q, interpolant)); 2284 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 2285 for (fc = 0; fc < Nc; ++fc) { 2286 const PetscReal wt = quadWeights[q * qNc + qc]; 2287 2288 for (d = 0; d < coordDim; ++d) grad[fc * coordDim + d] += interpolant[fc * dim + d] * wt * fegeom.detJ[q]; 2289 } 2290 vol += quadWeights[q * qNc] * fegeom.detJ[q]; 2291 } 2292 fieldOffset += Nb; 2293 qc += Nc; 2294 } 2295 PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x)); 2296 for (fc = 0; fc < numComponents; ++fc) { 2297 for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] += grad[fc * coordDim + d]; 2298 } 2299 volsum += vol; 2300 if (debug) { 2301 PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell)); 2302 for (fc = 0; fc < numComponents; ++fc) { 2303 for (d = 0; d < coordDim; ++d) { 2304 if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", ")); 2305 PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc * coordDim + d]))); 2306 } 2307 } 2308 PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n")); 2309 } 2310 } 2311 for (fc = 0; fc < numComponents; ++fc) { 2312 for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] /= volsum; 2313 } 2314 PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 2315 PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES)); 2316 } 2317 PetscCall(PetscFree6(gradsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 2318 PetscFunctionReturn(PETSC_SUCCESS); 2319 } 2320 2321 PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec locX, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user) 2322 { 2323 DM dmAux = NULL, plexA = NULL; 2324 PetscDS prob, probAux = NULL; 2325 PetscSection section, sectionAux; 2326 Vec locA; 2327 PetscInt dim, numCells = cEnd - cStart, c, f; 2328 PetscBool useFVM = PETSC_FALSE; 2329 /* DS */ 2330 PetscInt Nf, totDim, *uOff, *uOff_x, numConstants; 2331 PetscInt NfAux, totDimAux, *aOff; 2332 PetscScalar *u, *a = NULL; 2333 const PetscScalar *constants; 2334 /* Geometry */ 2335 PetscFEGeom *cgeomFEM; 2336 DM dmGrad; 2337 PetscQuadrature affineQuad = NULL; 2338 Vec cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL; 2339 PetscFVCellGeom *cgeomFVM; 2340 const PetscScalar *lgrad; 2341 PetscInt maxDegree; 2342 DMField coordField; 2343 IS cellIS; 2344 2345 PetscFunctionBegin; 2346 PetscCall(DMGetDS(dm, &prob)); 2347 PetscCall(DMGetDimension(dm, &dim)); 2348 PetscCall(DMGetLocalSection(dm, §ion)); 2349 PetscCall(DMGetNumFields(dm, &Nf)); 2350 /* Determine which discretizations we have */ 2351 for (f = 0; f < Nf; ++f) { 2352 PetscObject obj; 2353 PetscClassId id; 2354 2355 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2356 PetscCall(PetscObjectGetClassId(obj, &id)); 2357 if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE; 2358 } 2359 /* Read DS information */ 2360 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 2361 PetscCall(PetscDSGetComponentOffsets(prob, &uOff)); 2362 PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x)); 2363 PetscCall(ISCreateStride(PETSC_COMM_SELF, numCells, cStart, 1, &cellIS)); 2364 PetscCall(PetscDSGetConstants(prob, &numConstants, &constants)); 2365 /* Read Auxiliary DS information */ 2366 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA)); 2367 if (locA) { 2368 PetscCall(VecGetDM(locA, &dmAux)); 2369 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 2370 PetscCall(DMGetDS(dmAux, &probAux)); 2371 PetscCall(PetscDSGetNumFields(probAux, &NfAux)); 2372 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 2373 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 2374 PetscCall(PetscDSGetComponentOffsets(probAux, &aOff)); 2375 } 2376 /* Allocate data arrays */ 2377 PetscCall(PetscCalloc1(numCells * totDim, &u)); 2378 if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a)); 2379 /* Read out geometry */ 2380 PetscCall(DMGetCoordinateField(dm, &coordField)); 2381 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 2382 if (maxDegree <= 1) { 2383 PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad)); 2384 if (affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, affineQuad, PETSC_FEGEOM_BASIC, &cgeomFEM)); 2385 } 2386 if (useFVM) { 2387 PetscFV fv = NULL; 2388 Vec grad; 2389 PetscInt fStart, fEnd; 2390 PetscBool compGrad; 2391 2392 for (f = 0; f < Nf; ++f) { 2393 PetscObject obj; 2394 PetscClassId id; 2395 2396 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2397 PetscCall(PetscObjectGetClassId(obj, &id)); 2398 if (id == PETSCFV_CLASSID) { 2399 fv = (PetscFV)obj; 2400 break; 2401 } 2402 } 2403 PetscCall(PetscFVGetComputeGradients(fv, &compGrad)); 2404 PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE)); 2405 PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM)); 2406 PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad)); 2407 PetscCall(PetscFVSetComputeGradients(fv, compGrad)); 2408 PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM)); 2409 /* Reconstruct and limit cell gradients */ 2410 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 2411 PetscCall(DMGetGlobalVector(dmGrad, &grad)); 2412 PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad)); 2413 /* Communicate gradient values */ 2414 PetscCall(DMGetLocalVector(dmGrad, &locGrad)); 2415 PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad)); 2416 PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad)); 2417 PetscCall(DMRestoreGlobalVector(dmGrad, &grad)); 2418 /* Handle non-essential (e.g. outflow) boundary values */ 2419 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad)); 2420 PetscCall(VecGetArrayRead(locGrad, &lgrad)); 2421 } 2422 /* Read out data from inputs */ 2423 for (c = cStart; c < cEnd; ++c) { 2424 PetscScalar *x = NULL; 2425 PetscInt i; 2426 2427 PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x)); 2428 for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i]; 2429 PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x)); 2430 if (dmAux) { 2431 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, c, NULL, &x)); 2432 for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i]; 2433 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, c, NULL, &x)); 2434 } 2435 } 2436 /* Do integration for each field */ 2437 for (f = 0; f < Nf; ++f) { 2438 PetscObject obj; 2439 PetscClassId id; 2440 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset; 2441 2442 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2443 PetscCall(PetscObjectGetClassId(obj, &id)); 2444 if (id == PETSCFE_CLASSID) { 2445 PetscFE fe = (PetscFE)obj; 2446 PetscQuadrature q; 2447 PetscFEGeom *chunkGeom = NULL; 2448 PetscInt Nq, Nb; 2449 2450 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 2451 PetscCall(PetscFEGetQuadrature(fe, &q)); 2452 PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL)); 2453 PetscCall(PetscFEGetDimension(fe, &Nb)); 2454 blockSize = Nb * Nq; 2455 batchSize = numBlocks * blockSize; 2456 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 2457 numChunks = numCells / (numBatches * batchSize); 2458 Ne = numChunks * numBatches * batchSize; 2459 Nr = numCells % (numBatches * batchSize); 2460 offset = numCells - Nr; 2461 if (!affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, q, PETSC_FEGEOM_BASIC, &cgeomFEM)); 2462 PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom)); 2463 PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral)); 2464 PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &chunkGeom)); 2465 PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset * totDim], probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), &cintegral[offset * Nf])); 2466 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &chunkGeom)); 2467 if (!affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM)); 2468 } else if (id == PETSCFV_CLASSID) { 2469 PetscInt foff; 2470 PetscPointFunc obj_func; 2471 2472 PetscCall(PetscDSGetObjective(prob, f, &obj_func)); 2473 PetscCall(PetscDSGetFieldOffset(prob, f, &foff)); 2474 if (obj_func) { 2475 for (c = 0; c < numCells; ++c) { 2476 PetscScalar *u_x; 2477 PetscScalar lint = 0.; 2478 2479 PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x)); 2480 obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim * c + foff], NULL, u_x, aOff, NULL, PetscSafePointerPlusOffset(a, totDimAux * c), NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint); 2481 cintegral[c * Nf + f] += PetscRealPart(lint) * cgeomFVM[c].volume; 2482 } 2483 } 2484 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 2485 } 2486 /* Cleanup data arrays */ 2487 if (useFVM) { 2488 PetscCall(VecRestoreArrayRead(locGrad, &lgrad)); 2489 PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM)); 2490 PetscCall(DMRestoreLocalVector(dmGrad, &locGrad)); 2491 PetscCall(VecDestroy(&faceGeometryFVM)); 2492 PetscCall(VecDestroy(&cellGeometryFVM)); 2493 PetscCall(DMDestroy(&dmGrad)); 2494 } 2495 if (dmAux) PetscCall(PetscFree(a)); 2496 PetscCall(DMDestroy(&plexA)); 2497 PetscCall(PetscFree(u)); 2498 /* Cleanup */ 2499 if (affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM)); 2500 PetscCall(PetscQuadratureDestroy(&affineQuad)); 2501 PetscCall(ISDestroy(&cellIS)); 2502 PetscFunctionReturn(PETSC_SUCCESS); 2503 } 2504 2505 /*@ 2506 DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user 2507 2508 Input Parameters: 2509 + dm - The mesh 2510 . X - Global input vector 2511 - user - The user context 2512 2513 Output Parameter: 2514 . integral - Integral for each field 2515 2516 Level: developer 2517 2518 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()` 2519 @*/ 2520 PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user) 2521 { 2522 PetscInt printFEM; 2523 PetscScalar *cintegral, *lintegral; 2524 PetscInt Nf, f, cellHeight, cStart, cEnd, cell; 2525 Vec locX; 2526 2527 PetscFunctionBegin; 2528 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2529 PetscValidHeaderSpecific(X, VEC_CLASSID, 2); 2530 PetscAssertPointer(integral, 3); 2531 PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2532 PetscCall(DMPlexConvertPlex(dm, &dm, PETSC_TRUE)); 2533 PetscCall(DMGetNumFields(dm, &Nf)); 2534 PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight)); 2535 PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd)); 2536 /* TODO Introduce a loop over large chunks (right now this is a single chunk) */ 2537 PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd - cStart) * Nf, &cintegral)); 2538 /* Get local solution with boundary values */ 2539 PetscCall(DMGetLocalVector(dm, &locX)); 2540 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL)); 2541 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX)); 2542 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX)); 2543 PetscCall(DMPlexComputeIntegral_Internal(dm, locX, cStart, cEnd, cintegral, user)); 2544 PetscCall(DMRestoreLocalVector(dm, &locX)); 2545 printFEM = ((DM_Plex *)dm->data)->printFEM; 2546 /* Sum up values */ 2547 for (cell = cStart; cell < cEnd; ++cell) { 2548 const PetscInt c = cell - cStart; 2549 2550 if (printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf])); 2551 for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c * Nf + f]; 2552 } 2553 PetscCallMPI(MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 2554 if (printFEM) { 2555 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "Integral:")); 2556 for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), " %g", (double)PetscRealPart(integral[f]))); 2557 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "\n")); 2558 } 2559 PetscCall(PetscFree2(lintegral, cintegral)); 2560 PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2561 PetscCall(DMDestroy(&dm)); 2562 PetscFunctionReturn(PETSC_SUCCESS); 2563 } 2564 2565 /*@ 2566 DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user 2567 2568 Input Parameters: 2569 + dm - The mesh 2570 . X - Global input vector 2571 - user - The user context 2572 2573 Output Parameter: 2574 . F - Cellwise integrals for each field 2575 2576 Level: developer 2577 2578 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()` 2579 @*/ 2580 PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user) 2581 { 2582 PetscInt printFEM; 2583 DM dmF; 2584 PetscSection sectionF = NULL; 2585 PetscScalar *cintegral, *af; 2586 PetscInt Nf, f, cellHeight, cStart, cEnd, cell, n; 2587 Vec locX; 2588 2589 PetscFunctionBegin; 2590 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2591 PetscValidHeaderSpecific(X, VEC_CLASSID, 2); 2592 PetscValidHeaderSpecific(F, VEC_CLASSID, 3); 2593 PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2594 PetscCall(DMPlexConvertPlex(dm, &dm, PETSC_TRUE)); 2595 PetscCall(DMGetNumFields(dm, &Nf)); 2596 PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight)); 2597 PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd)); 2598 /* TODO Introduce a loop over large chunks (right now this is a single chunk) */ 2599 PetscCall(PetscCalloc1((cEnd - cStart) * Nf, &cintegral)); 2600 /* Get local solution with boundary values */ 2601 PetscCall(DMGetLocalVector(dm, &locX)); 2602 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL)); 2603 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX)); 2604 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX)); 2605 PetscCall(DMPlexComputeIntegral_Internal(dm, locX, cStart, cEnd, cintegral, user)); 2606 PetscCall(DMRestoreLocalVector(dm, &locX)); 2607 /* Put values in F */ 2608 PetscCall(VecGetArray(F, &af)); 2609 PetscCall(VecGetDM(F, &dmF)); 2610 if (dmF) PetscCall(DMGetLocalSection(dmF, §ionF)); 2611 PetscCall(VecGetLocalSize(F, &n)); 2612 PetscCheck(n >= (cEnd - cStart) * Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Vector size %" PetscInt_FMT " < %" PetscInt_FMT, n, (cEnd - cStart) * Nf); 2613 printFEM = ((DM_Plex *)dm->data)->printFEM; 2614 for (cell = cStart; cell < cEnd; ++cell) { 2615 const PetscInt c = cell - cStart; 2616 PetscInt dof = Nf, off = c * Nf; 2617 2618 if (printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf])); 2619 if (sectionF) { 2620 PetscCall(PetscSectionGetDof(sectionF, cell, &dof)); 2621 PetscCall(PetscSectionGetOffset(sectionF, cell, &off)); 2622 } 2623 PetscCheck(dof == Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf); 2624 for (f = 0; f < Nf; ++f) af[off + f] = cintegral[c * Nf + f]; 2625 } 2626 PetscCall(VecRestoreArray(F, &af)); 2627 PetscCall(PetscFree(cintegral)); 2628 PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2629 PetscCall(DMDestroy(&dm)); 2630 PetscFunctionReturn(PETSC_SUCCESS); 2631 } 2632 2633 static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS, void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *fintegral, void *user) 2634 { 2635 DM plex = NULL, plexA = NULL; 2636 DMEnclosureType encAux; 2637 PetscDS prob, probAux = NULL; 2638 PetscSection section, sectionAux = NULL; 2639 Vec locA = NULL; 2640 DMField coordField; 2641 PetscInt Nf, totDim, *uOff, *uOff_x; 2642 PetscInt NfAux = 0, totDimAux = 0, *aOff = NULL; 2643 PetscScalar *u, *a = NULL; 2644 const PetscScalar *constants; 2645 PetscInt numConstants, f; 2646 2647 PetscFunctionBegin; 2648 PetscCall(DMGetCoordinateField(dm, &coordField)); 2649 PetscCall(DMConvert(dm, DMPLEX, &plex)); 2650 PetscCall(DMGetDS(dm, &prob)); 2651 PetscCall(DMGetLocalSection(dm, §ion)); 2652 PetscCall(PetscSectionGetNumFields(section, &Nf)); 2653 /* Determine which discretizations we have */ 2654 for (f = 0; f < Nf; ++f) { 2655 PetscObject obj; 2656 PetscClassId id; 2657 2658 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2659 PetscCall(PetscObjectGetClassId(obj, &id)); 2660 PetscCheck(id != PETSCFV_CLASSID, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f); 2661 } 2662 /* Read DS information */ 2663 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 2664 PetscCall(PetscDSGetComponentOffsets(prob, &uOff)); 2665 PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x)); 2666 PetscCall(PetscDSGetConstants(prob, &numConstants, &constants)); 2667 /* Read Auxiliary DS information */ 2668 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA)); 2669 if (locA) { 2670 DM dmAux; 2671 2672 PetscCall(VecGetDM(locA, &dmAux)); 2673 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 2674 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 2675 PetscCall(DMGetDS(dmAux, &probAux)); 2676 PetscCall(PetscDSGetNumFields(probAux, &NfAux)); 2677 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 2678 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 2679 PetscCall(PetscDSGetComponentOffsets(probAux, &aOff)); 2680 } 2681 /* Integrate over points */ 2682 { 2683 PetscFEGeom *fgeom, *chunkGeom = NULL; 2684 PetscInt maxDegree; 2685 PetscQuadrature qGeom = NULL; 2686 const PetscInt *points; 2687 PetscInt numFaces, face, Nq, field; 2688 PetscInt numChunks, chunkSize, chunk, Nr, offset; 2689 2690 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 2691 PetscCall(ISGetIndices(pointIS, &points)); 2692 PetscCall(PetscCalloc2(numFaces * totDim, &u, (locA ? (size_t)numFaces * totDimAux : 0), &a)); 2693 PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree)); 2694 for (face = 0; face < numFaces; ++face) { 2695 const PetscInt point = points[face], *support; 2696 PetscScalar *x = NULL; 2697 2698 PetscCall(DMPlexGetSupport(dm, point, &support)); 2699 PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x)); 2700 for (PetscInt i = 0; i < totDim; ++i) u[face * totDim + i] = x[i]; 2701 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x)); 2702 if (locA) { 2703 PetscInt subp; 2704 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp)); 2705 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x)); 2706 for (PetscInt i = 0; i < totDimAux; ++i) a[f * totDimAux + i] = x[i]; 2707 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x)); 2708 } 2709 } 2710 for (field = 0; field < Nf; ++field) { 2711 PetscFE fe; 2712 2713 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fe)); 2714 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom)); 2715 if (!qGeom) { 2716 PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom)); 2717 PetscCall(PetscObjectReference((PetscObject)qGeom)); 2718 } 2719 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 2720 PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom)); 2721 /* Get blocking */ 2722 { 2723 PetscQuadrature q; 2724 PetscInt numBatches, batchSize, numBlocks, blockSize; 2725 PetscInt Nq, Nb; 2726 2727 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 2728 PetscCall(PetscFEGetQuadrature(fe, &q)); 2729 PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL)); 2730 PetscCall(PetscFEGetDimension(fe, &Nb)); 2731 blockSize = Nb * Nq; 2732 batchSize = numBlocks * blockSize; 2733 chunkSize = numBatches * batchSize; 2734 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 2735 numChunks = numFaces / chunkSize; 2736 Nr = numFaces % chunkSize; 2737 offset = numFaces - Nr; 2738 } 2739 /* Do integration for each field */ 2740 for (chunk = 0; chunk < numChunks; ++chunk) { 2741 PetscCall(PetscFEGeomGetChunk(fgeom, chunk * chunkSize, (chunk + 1) * chunkSize, &chunkGeom)); 2742 PetscCall(PetscFEIntegrateBd(prob, field, funcs[field], chunkSize, chunkGeom, &u[chunk * chunkSize * totDim], probAux, PetscSafePointerPlusOffset(a, chunk * chunkSize * totDimAux), &fintegral[chunk * chunkSize * Nf])); 2743 PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom)); 2744 } 2745 PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom)); 2746 PetscCall(PetscFEIntegrateBd(prob, field, funcs[field], Nr, chunkGeom, &u[offset * totDim], probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), &fintegral[offset * Nf])); 2747 PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom)); 2748 /* Cleanup data arrays */ 2749 PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom)); 2750 PetscCall(PetscQuadratureDestroy(&qGeom)); 2751 } 2752 PetscCall(PetscFree2(u, a)); 2753 PetscCall(ISRestoreIndices(pointIS, &points)); 2754 } 2755 if (plex) PetscCall(DMDestroy(&plex)); 2756 if (plexA) PetscCall(DMDestroy(&plexA)); 2757 PetscFunctionReturn(PETSC_SUCCESS); 2758 } 2759 2760 /*@C 2761 DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user 2762 2763 Input Parameters: 2764 + dm - The mesh 2765 . X - Global input vector 2766 . label - The boundary `DMLabel` 2767 . numVals - The number of label values to use, or `PETSC_DETERMINE` for all values 2768 . vals - The label values to use, or NULL for all values 2769 . funcs - The functions to integrate along the boundary for each field 2770 - user - The user context 2771 2772 Output Parameter: 2773 . integral - Integral for each field 2774 2775 Level: developer 2776 2777 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeIntegralFEM()`, `DMPlexComputeBdResidualFEM()` 2778 @*/ 2779 PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[], void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *integral, void *user) 2780 { 2781 Vec locX; 2782 PetscSection section; 2783 DMLabel depthLabel; 2784 IS facetIS; 2785 PetscInt dim, Nf, f, v; 2786 2787 PetscFunctionBegin; 2788 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2789 PetscValidHeaderSpecific(X, VEC_CLASSID, 2); 2790 PetscValidHeaderSpecific(label, DMLABEL_CLASSID, 3); 2791 if (vals) PetscAssertPointer(vals, 5); 2792 PetscAssertPointer(integral, 7); 2793 PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2794 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 2795 PetscCall(DMGetDimension(dm, &dim)); 2796 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 2797 PetscCall(DMGetLocalSection(dm, §ion)); 2798 PetscCall(PetscSectionGetNumFields(section, &Nf)); 2799 /* Get local solution with boundary values */ 2800 PetscCall(DMGetLocalVector(dm, &locX)); 2801 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL)); 2802 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX)); 2803 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX)); 2804 /* Loop over label values */ 2805 PetscCall(PetscArrayzero(integral, Nf)); 2806 for (v = 0; v < numVals; ++v) { 2807 IS pointIS; 2808 PetscInt numFaces, face; 2809 PetscScalar *fintegral; 2810 2811 PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS)); 2812 if (!pointIS) continue; /* No points with that id on this process */ 2813 { 2814 IS isectIS; 2815 2816 /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */ 2817 PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS)); 2818 PetscCall(ISDestroy(&pointIS)); 2819 pointIS = isectIS; 2820 } 2821 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 2822 PetscCall(PetscCalloc1(numFaces * Nf, &fintegral)); 2823 PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, funcs, fintegral, user)); 2824 /* Sum point contributions into integral */ 2825 for (f = 0; f < Nf; ++f) 2826 for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face * Nf + f]; 2827 PetscCall(PetscFree(fintegral)); 2828 PetscCall(ISDestroy(&pointIS)); 2829 } 2830 PetscCall(DMRestoreLocalVector(dm, &locX)); 2831 PetscCall(ISDestroy(&facetIS)); 2832 PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2833 PetscFunctionReturn(PETSC_SUCCESS); 2834 } 2835 2836 /*@ 2837 DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix from the coarse `DM` to a uniformly refined `DM`. 2838 2839 Input Parameters: 2840 + dmc - The coarse mesh 2841 . dmf - The fine mesh 2842 . isRefined - Flag indicating regular refinement, rather than the same topology 2843 - user - The user context 2844 2845 Output Parameter: 2846 . In - The interpolation matrix 2847 2848 Level: developer 2849 2850 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorGeneral()` 2851 @*/ 2852 PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user) 2853 { 2854 DM_Plex *mesh = (DM_Plex *)dmc->data; 2855 const char *name = "Interpolator"; 2856 PetscFE *feRef; 2857 PetscFV *fvRef; 2858 PetscSection fsection, fglobalSection; 2859 PetscSection csection, cglobalSection; 2860 PetscScalar *elemMat; 2861 PetscInt dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c; 2862 PetscInt cTotDim = 0, rTotDim = 0; 2863 2864 PetscFunctionBegin; 2865 PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 2866 PetscCall(DMGetDimension(dmf, &dim)); 2867 PetscCall(DMGetLocalSection(dmf, &fsection)); 2868 PetscCall(DMGetGlobalSection(dmf, &fglobalSection)); 2869 PetscCall(DMGetLocalSection(dmc, &csection)); 2870 PetscCall(DMGetGlobalSection(dmc, &cglobalSection)); 2871 PetscCall(PetscSectionGetNumFields(fsection, &Nf)); 2872 PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd)); 2873 PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef)); 2874 for (f = 0; f < Nf; ++f) { 2875 PetscObject obj, objc; 2876 PetscClassId id, idc; 2877 PetscInt rNb = 0, Nc = 0, cNb = 0; 2878 2879 PetscCall(DMGetField(dmf, f, NULL, &obj)); 2880 PetscCall(PetscObjectGetClassId(obj, &id)); 2881 if (id == PETSCFE_CLASSID) { 2882 PetscFE fe = (PetscFE)obj; 2883 2884 if (isRefined) { 2885 PetscCall(PetscFERefine(fe, &feRef[f])); 2886 } else { 2887 PetscCall(PetscObjectReference((PetscObject)fe)); 2888 feRef[f] = fe; 2889 } 2890 PetscCall(PetscFEGetDimension(feRef[f], &rNb)); 2891 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 2892 } else if (id == PETSCFV_CLASSID) { 2893 PetscFV fv = (PetscFV)obj; 2894 PetscDualSpace Q; 2895 2896 if (isRefined) { 2897 PetscCall(PetscFVRefine(fv, &fvRef[f])); 2898 } else { 2899 PetscCall(PetscObjectReference((PetscObject)fv)); 2900 fvRef[f] = fv; 2901 } 2902 PetscCall(PetscFVGetDualSpace(fvRef[f], &Q)); 2903 PetscCall(PetscDualSpaceGetDimension(Q, &rNb)); 2904 PetscCall(PetscFVGetDualSpace(fv, &Q)); 2905 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 2906 } 2907 PetscCall(DMGetField(dmc, f, NULL, &objc)); 2908 PetscCall(PetscObjectGetClassId(objc, &idc)); 2909 if (idc == PETSCFE_CLASSID) { 2910 PetscFE fe = (PetscFE)objc; 2911 2912 PetscCall(PetscFEGetDimension(fe, &cNb)); 2913 } else if (id == PETSCFV_CLASSID) { 2914 PetscFV fv = (PetscFV)obj; 2915 PetscDualSpace Q; 2916 2917 PetscCall(PetscFVGetDualSpace(fv, &Q)); 2918 PetscCall(PetscDualSpaceGetDimension(Q, &cNb)); 2919 } 2920 rTotDim += rNb; 2921 cTotDim += cNb; 2922 } 2923 PetscCall(PetscMalloc1(rTotDim * cTotDim, &elemMat)); 2924 PetscCall(PetscArrayzero(elemMat, rTotDim * cTotDim)); 2925 for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) { 2926 PetscDualSpace Qref; 2927 PetscQuadrature f; 2928 const PetscReal *qpoints, *qweights; 2929 PetscReal *points; 2930 PetscInt npoints = 0, Nc, Np, fpdim, i, k, p, d; 2931 2932 /* Compose points from all dual basis functionals */ 2933 if (feRef[fieldI]) { 2934 PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref)); 2935 PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc)); 2936 } else { 2937 PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref)); 2938 PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc)); 2939 } 2940 PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim)); 2941 for (i = 0; i < fpdim; ++i) { 2942 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2943 PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL)); 2944 npoints += Np; 2945 } 2946 PetscCall(PetscMalloc1(npoints * dim, &points)); 2947 for (i = 0, k = 0; i < fpdim; ++i) { 2948 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2949 PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL)); 2950 for (p = 0; p < Np; ++p, ++k) 2951 for (d = 0; d < dim; ++d) points[k * dim + d] = qpoints[p * dim + d]; 2952 } 2953 2954 for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) { 2955 PetscObject obj; 2956 PetscClassId id; 2957 PetscInt NcJ = 0, cpdim = 0, j, qNc; 2958 2959 PetscCall(DMGetField(dmc, fieldJ, NULL, &obj)); 2960 PetscCall(PetscObjectGetClassId(obj, &id)); 2961 if (id == PETSCFE_CLASSID) { 2962 PetscFE fe = (PetscFE)obj; 2963 PetscTabulation T = NULL; 2964 2965 /* Evaluate basis at points */ 2966 PetscCall(PetscFEGetNumComponents(fe, &NcJ)); 2967 PetscCall(PetscFEGetDimension(fe, &cpdim)); 2968 /* For now, fields only interpolate themselves */ 2969 if (fieldI == fieldJ) { 2970 PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ); 2971 PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T)); 2972 for (i = 0, k = 0; i < fpdim; ++i) { 2973 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2974 PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights)); 2975 PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ); 2976 for (p = 0; p < Np; ++p, ++k) { 2977 for (j = 0; j < cpdim; ++j) { 2978 /* 2979 cTotDim: Total columns in element interpolation matrix, sum of number of dual basis functionals in each field 2980 offsetI, offsetJ: Offsets into the larger element interpolation matrix for different fields 2981 fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals 2982 qNC, Nc, Ncj, c: Number of components in this field 2983 Np, p: Number of quad points in the fine grid functional i 2984 k: i*Np + p, overall point number for the interpolation 2985 */ 2986 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += T->T[0][k * cpdim * NcJ + j * Nc + c] * qweights[p * qNc + c]; 2987 } 2988 } 2989 } 2990 PetscCall(PetscTabulationDestroy(&T)); 2991 } 2992 } else if (id == PETSCFV_CLASSID) { 2993 PetscFV fv = (PetscFV)obj; 2994 2995 /* Evaluate constant function at points */ 2996 PetscCall(PetscFVGetNumComponents(fv, &NcJ)); 2997 cpdim = 1; 2998 /* For now, fields only interpolate themselves */ 2999 if (fieldI == fieldJ) { 3000 PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ); 3001 for (i = 0, k = 0; i < fpdim; ++i) { 3002 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 3003 PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights)); 3004 PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ); 3005 for (p = 0; p < Np; ++p, ++k) { 3006 for (j = 0; j < cpdim; ++j) { 3007 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += 1.0 * qweights[p * qNc + c]; 3008 } 3009 } 3010 } 3011 } 3012 } 3013 offsetJ += cpdim; 3014 } 3015 offsetI += fpdim; 3016 PetscCall(PetscFree(points)); 3017 } 3018 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat)); 3019 /* Preallocate matrix */ 3020 { 3021 Mat preallocator; 3022 PetscScalar *vals; 3023 PetscInt *cellCIndices, *cellFIndices; 3024 PetscInt locRows, locCols, cell; 3025 3026 PetscCall(MatGetLocalSize(In, &locRows, &locCols)); 3027 PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &preallocator)); 3028 PetscCall(MatSetType(preallocator, MATPREALLOCATOR)); 3029 PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE)); 3030 PetscCall(MatSetUp(preallocator)); 3031 PetscCall(PetscCalloc3(rTotDim * cTotDim, &vals, cTotDim, &cellCIndices, rTotDim, &cellFIndices)); 3032 for (cell = cStart; cell < cEnd; ++cell) { 3033 if (isRefined) { 3034 PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices)); 3035 PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES)); 3036 } else { 3037 PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, PETSC_FALSE, dmc, csection, cglobalSection, PETSC_FALSE, preallocator, cell, vals, INSERT_VALUES)); 3038 } 3039 } 3040 PetscCall(PetscFree3(vals, cellCIndices, cellFIndices)); 3041 PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY)); 3042 PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY)); 3043 PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In)); 3044 PetscCall(MatDestroy(&preallocator)); 3045 } 3046 /* Fill matrix */ 3047 PetscCall(MatZeroEntries(In)); 3048 for (c = cStart; c < cEnd; ++c) { 3049 if (isRefined) { 3050 PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES)); 3051 } else { 3052 PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, PETSC_FALSE, dmc, csection, cglobalSection, PETSC_FALSE, In, c, elemMat, INSERT_VALUES)); 3053 } 3054 } 3055 for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f])); 3056 PetscCall(PetscFree2(feRef, fvRef)); 3057 PetscCall(PetscFree(elemMat)); 3058 PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY)); 3059 PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY)); 3060 if (mesh->printFEM > 1) { 3061 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name)); 3062 PetscCall(MatFilter(In, 1.0e-10, PETSC_FALSE, PETSC_FALSE)); 3063 PetscCall(MatView(In, NULL)); 3064 } 3065 PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 3066 PetscFunctionReturn(PETSC_SUCCESS); 3067 } 3068 3069 PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user) 3070 { 3071 SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_SUP, "Laziness"); 3072 } 3073 3074 /*@ 3075 DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix from the coarse `DM` to a non-nested fine `DM`. 3076 3077 Input Parameters: 3078 + dmf - The fine mesh 3079 . dmc - The coarse mesh 3080 - user - The user context 3081 3082 Output Parameter: 3083 . In - The interpolation matrix 3084 3085 Level: developer 3086 3087 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()` 3088 @*/ 3089 PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user) 3090 { 3091 DM_Plex *mesh = (DM_Plex *)dmf->data; 3092 const char *name = "Interpolator"; 3093 PetscDS prob; 3094 Mat interp; 3095 PetscSection fsection, globalFSection; 3096 PetscSection csection, globalCSection; 3097 PetscInt locRows, locCols; 3098 PetscReal *x, *v0, *J, *invJ, detJ; 3099 PetscReal *v0c, *Jc, *invJc, detJc; 3100 PetscScalar *elemMat; 3101 PetscInt dim, Nf, field, totDim, cStart, cEnd, cell, ccell, s; 3102 3103 PetscFunctionBegin; 3104 PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 3105 PetscCall(DMGetCoordinateDim(dmc, &dim)); 3106 PetscCall(DMGetDS(dmc, &prob)); 3107 PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL)); 3108 PetscCall(PetscDSGetNumFields(prob, &Nf)); 3109 PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ)); 3110 PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc)); 3111 PetscCall(DMGetLocalSection(dmf, &fsection)); 3112 PetscCall(DMGetGlobalSection(dmf, &globalFSection)); 3113 PetscCall(DMGetLocalSection(dmc, &csection)); 3114 PetscCall(DMGetGlobalSection(dmc, &globalCSection)); 3115 PetscCall(DMPlexGetSimplexOrBoxCells(dmf, 0, &cStart, &cEnd)); 3116 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 3117 PetscCall(PetscMalloc1(totDim, &elemMat)); 3118 3119 PetscCall(MatGetLocalSize(In, &locRows, &locCols)); 3120 PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &interp)); 3121 PetscCall(MatSetType(interp, MATPREALLOCATOR)); 3122 PetscCall(MatSetSizes(interp, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE)); 3123 PetscCall(MatSetUp(interp)); 3124 for (s = 0; s < 2; ++s) { 3125 for (field = 0; field < Nf; ++field) { 3126 PetscObject obj; 3127 PetscClassId id; 3128 PetscDualSpace Q = NULL; 3129 PetscTabulation T = NULL; 3130 PetscQuadrature f; 3131 const PetscReal *qpoints, *qweights; 3132 PetscInt Nc, qNc, Np, fpdim, off, i, d; 3133 3134 PetscCall(PetscDSGetFieldOffset(prob, field, &off)); 3135 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 3136 PetscCall(PetscObjectGetClassId(obj, &id)); 3137 if (id == PETSCFE_CLASSID) { 3138 PetscFE fe = (PetscFE)obj; 3139 3140 PetscCall(PetscFEGetDualSpace(fe, &Q)); 3141 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 3142 if (s) PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T)); 3143 } else if (id == PETSCFV_CLASSID) { 3144 PetscFV fv = (PetscFV)obj; 3145 3146 PetscCall(PetscFVGetDualSpace(fv, &Q)); 3147 Nc = 1; 3148 } else SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 3149 PetscCall(PetscDualSpaceGetDimension(Q, &fpdim)); 3150 /* For each fine grid cell */ 3151 for (cell = cStart; cell < cEnd; ++cell) { 3152 PetscInt *findices, *cindices; 3153 PetscInt numFIndices, numCIndices; 3154 3155 PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3156 PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ)); 3157 PetscCheck(numFIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim); 3158 for (i = 0; i < fpdim; ++i) { 3159 Vec pointVec; 3160 PetscScalar *pV; 3161 PetscSF coarseCellSF = NULL; 3162 const PetscSFNode *coarseCells; 3163 PetscInt numCoarseCells, cpdim, row = findices[i + off], q, c, j; 3164 3165 /* Get points from the dual basis functional quadrature */ 3166 PetscCall(PetscDualSpaceGetFunctional(Q, i, &f)); 3167 PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights)); 3168 PetscCheck(qNc == Nc, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc); 3169 PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np * dim, &pointVec)); 3170 PetscCall(VecSetBlockSize(pointVec, dim)); 3171 PetscCall(VecGetArray(pointVec, &pV)); 3172 for (q = 0; q < Np; ++q) { 3173 const PetscReal xi0[3] = {-1., -1., -1.}; 3174 3175 /* Transform point to real space */ 3176 CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x); 3177 for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d]; 3178 } 3179 PetscCall(VecRestoreArray(pointVec, &pV)); 3180 /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */ 3181 /* OPT: Read this out from preallocation information */ 3182 PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF)); 3183 /* Update preallocation info */ 3184 PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells)); 3185 PetscCheck(numCoarseCells == Np, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located"); 3186 PetscCall(VecGetArray(pointVec, &pV)); 3187 for (ccell = 0; ccell < numCoarseCells; ++ccell) { 3188 PetscReal pVReal[3]; 3189 const PetscReal xi0[3] = {-1., -1., -1.}; 3190 3191 PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3192 if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetDimension((PetscFE)obj, &cpdim)); 3193 else cpdim = 1; 3194 3195 if (s) { 3196 /* Transform points from real space to coarse reference space */ 3197 PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc)); 3198 for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]); 3199 CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x); 3200 3201 if (id == PETSCFE_CLASSID) { 3202 /* Evaluate coarse basis on contained point */ 3203 PetscCall(PetscFEComputeTabulation((PetscFE)obj, 1, x, 0, T)); 3204 PetscCall(PetscArrayzero(elemMat, cpdim)); 3205 /* Get elemMat entries by multiplying by weight */ 3206 for (j = 0; j < cpdim; ++j) { 3207 for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * qweights[ccell * qNc + c]; 3208 } 3209 } else { 3210 for (j = 0; j < cpdim; ++j) { 3211 for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * qweights[ccell * qNc + c]; 3212 } 3213 } 3214 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat)); 3215 } 3216 /* Update interpolator */ 3217 PetscCheck(numCIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, totDim); 3218 PetscCall(MatSetValues(interp, 1, &row, cpdim, &cindices[off], elemMat, INSERT_VALUES)); 3219 PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3220 } 3221 PetscCall(VecRestoreArray(pointVec, &pV)); 3222 PetscCall(PetscSFDestroy(&coarseCellSF)); 3223 PetscCall(VecDestroy(&pointVec)); 3224 } 3225 PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3226 } 3227 if (s && id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T)); 3228 } 3229 if (!s) { 3230 PetscCall(MatAssemblyBegin(interp, MAT_FINAL_ASSEMBLY)); 3231 PetscCall(MatAssemblyEnd(interp, MAT_FINAL_ASSEMBLY)); 3232 PetscCall(MatPreallocatorPreallocate(interp, PETSC_TRUE, In)); 3233 PetscCall(MatDestroy(&interp)); 3234 interp = In; 3235 } 3236 } 3237 PetscCall(PetscFree3(v0, J, invJ)); 3238 PetscCall(PetscFree3(v0c, Jc, invJc)); 3239 PetscCall(PetscFree(elemMat)); 3240 PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY)); 3241 PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY)); 3242 PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 3243 PetscFunctionReturn(PETSC_SUCCESS); 3244 } 3245 3246 /*@ 3247 DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix from the coarse `DM` to a non-nested fine `DM`. 3248 3249 Input Parameters: 3250 + dmf - The fine mesh 3251 . dmc - The coarse mesh 3252 - user - The user context 3253 3254 Output Parameter: 3255 . mass - The mass matrix 3256 3257 Level: developer 3258 3259 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeMassMatrixNested()`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeInterpolatorGeneral()` 3260 @*/ 3261 PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user) 3262 { 3263 DM_Plex *mesh = (DM_Plex *)dmf->data; 3264 const char *name = "Mass Matrix"; 3265 PetscDS prob; 3266 PetscSection fsection, csection, globalFSection, globalCSection; 3267 PetscHSetIJ ht; 3268 PetscLayout rLayout; 3269 PetscInt *dnz, *onz; 3270 PetscInt locRows, rStart, rEnd; 3271 PetscReal *x, *v0, *J, *invJ, detJ; 3272 PetscReal *v0c, *Jc, *invJc, detJc; 3273 PetscScalar *elemMat; 3274 PetscInt dim, Nf, field, totDim, cStart, cEnd, cell, ccell; 3275 3276 PetscFunctionBegin; 3277 PetscCall(DMGetCoordinateDim(dmc, &dim)); 3278 PetscCall(DMGetDS(dmc, &prob)); 3279 PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL)); 3280 PetscCall(PetscDSGetNumFields(prob, &Nf)); 3281 PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ)); 3282 PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc)); 3283 PetscCall(DMGetLocalSection(dmf, &fsection)); 3284 PetscCall(DMGetGlobalSection(dmf, &globalFSection)); 3285 PetscCall(DMGetLocalSection(dmc, &csection)); 3286 PetscCall(DMGetGlobalSection(dmc, &globalCSection)); 3287 PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd)); 3288 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 3289 PetscCall(PetscMalloc1(totDim, &elemMat)); 3290 3291 PetscCall(MatGetLocalSize(mass, &locRows, NULL)); 3292 PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)mass), &rLayout)); 3293 PetscCall(PetscLayoutSetLocalSize(rLayout, locRows)); 3294 PetscCall(PetscLayoutSetBlockSize(rLayout, 1)); 3295 PetscCall(PetscLayoutSetUp(rLayout)); 3296 PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd)); 3297 PetscCall(PetscLayoutDestroy(&rLayout)); 3298 PetscCall(PetscCalloc2(locRows, &dnz, locRows, &onz)); 3299 PetscCall(PetscHSetIJCreate(&ht)); 3300 for (field = 0; field < Nf; ++field) { 3301 PetscObject obj; 3302 PetscClassId id; 3303 PetscQuadrature quad; 3304 const PetscReal *qpoints; 3305 PetscInt Nq, Nc, i, d; 3306 3307 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 3308 PetscCall(PetscObjectGetClassId(obj, &id)); 3309 if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad)); 3310 else PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad)); 3311 PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL)); 3312 /* For each fine grid cell */ 3313 for (cell = cStart; cell < cEnd; ++cell) { 3314 Vec pointVec; 3315 PetscScalar *pV; 3316 PetscSF coarseCellSF = NULL; 3317 const PetscSFNode *coarseCells; 3318 PetscInt numCoarseCells, q, c; 3319 PetscInt *findices, *cindices; 3320 PetscInt numFIndices, numCIndices; 3321 3322 PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3323 PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ)); 3324 /* Get points from the quadrature */ 3325 PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec)); 3326 PetscCall(VecSetBlockSize(pointVec, dim)); 3327 PetscCall(VecGetArray(pointVec, &pV)); 3328 for (q = 0; q < Nq; ++q) { 3329 const PetscReal xi0[3] = {-1., -1., -1.}; 3330 3331 /* Transform point to real space */ 3332 CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x); 3333 for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d]; 3334 } 3335 PetscCall(VecRestoreArray(pointVec, &pV)); 3336 /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */ 3337 PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF)); 3338 PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view")); 3339 /* Update preallocation info */ 3340 PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells)); 3341 PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located"); 3342 { 3343 PetscHashIJKey key; 3344 PetscBool missing; 3345 3346 for (i = 0; i < numFIndices; ++i) { 3347 key.i = findices[i]; 3348 if (key.i >= 0) { 3349 /* Get indices for coarse elements */ 3350 for (ccell = 0; ccell < numCoarseCells; ++ccell) { 3351 PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3352 for (c = 0; c < numCIndices; ++c) { 3353 key.j = cindices[c]; 3354 if (key.j < 0) continue; 3355 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing)); 3356 if (missing) { 3357 if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i - rStart]; 3358 else ++onz[key.i - rStart]; 3359 } 3360 } 3361 PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3362 } 3363 } 3364 } 3365 } 3366 PetscCall(PetscSFDestroy(&coarseCellSF)); 3367 PetscCall(VecDestroy(&pointVec)); 3368 PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3369 } 3370 } 3371 PetscCall(PetscHSetIJDestroy(&ht)); 3372 PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL)); 3373 PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE)); 3374 PetscCall(PetscFree2(dnz, onz)); 3375 for (field = 0; field < Nf; ++field) { 3376 PetscObject obj; 3377 PetscClassId id; 3378 PetscTabulation T, Tfine; 3379 PetscQuadrature quad; 3380 const PetscReal *qpoints, *qweights; 3381 PetscInt Nq, Nc, i, d; 3382 3383 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 3384 PetscCall(PetscObjectGetClassId(obj, &id)); 3385 if (id == PETSCFE_CLASSID) { 3386 PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad)); 3387 PetscCall(PetscFEGetCellTabulation((PetscFE)obj, 1, &Tfine)); 3388 PetscCall(PetscFECreateTabulation((PetscFE)obj, 1, 1, x, 0, &T)); 3389 } else { 3390 PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad)); 3391 } 3392 PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights)); 3393 /* For each fine grid cell */ 3394 for (cell = cStart; cell < cEnd; ++cell) { 3395 Vec pointVec; 3396 PetscScalar *pV; 3397 PetscSF coarseCellSF = NULL; 3398 const PetscSFNode *coarseCells; 3399 PetscInt numCoarseCells, cpdim, q, c, j; 3400 PetscInt *findices, *cindices; 3401 PetscInt numFIndices, numCIndices; 3402 3403 PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3404 PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ)); 3405 /* Get points from the quadrature */ 3406 PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec)); 3407 PetscCall(VecSetBlockSize(pointVec, dim)); 3408 PetscCall(VecGetArray(pointVec, &pV)); 3409 for (q = 0; q < Nq; ++q) { 3410 const PetscReal xi0[3] = {-1., -1., -1.}; 3411 3412 /* Transform point to real space */ 3413 CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x); 3414 for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d]; 3415 } 3416 PetscCall(VecRestoreArray(pointVec, &pV)); 3417 /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */ 3418 PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF)); 3419 /* Update matrix */ 3420 PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells)); 3421 PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located"); 3422 PetscCall(VecGetArray(pointVec, &pV)); 3423 for (ccell = 0; ccell < numCoarseCells; ++ccell) { 3424 PetscReal pVReal[3]; 3425 const PetscReal xi0[3] = {-1., -1., -1.}; 3426 3427 PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3428 /* Transform points from real space to coarse reference space */ 3429 PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc)); 3430 for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]); 3431 CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x); 3432 3433 if (id == PETSCFE_CLASSID) { 3434 PetscFE fe = (PetscFE)obj; 3435 3436 /* Evaluate coarse basis on contained point */ 3437 PetscCall(PetscFEGetDimension(fe, &cpdim)); 3438 PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T)); 3439 /* Get elemMat entries by multiplying by weight */ 3440 for (i = 0; i < numFIndices; ++i) { 3441 PetscCall(PetscArrayzero(elemMat, cpdim)); 3442 for (j = 0; j < cpdim; ++j) { 3443 for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * Tfine->T[0][(ccell * numFIndices + i) * Nc + c] * qweights[ccell * Nc + c] * detJ; 3444 } 3445 /* Update interpolator */ 3446 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat)); 3447 PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim); 3448 PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES)); 3449 } 3450 } else { 3451 cpdim = 1; 3452 for (i = 0; i < numFIndices; ++i) { 3453 PetscCall(PetscArrayzero(elemMat, cpdim)); 3454 for (j = 0; j < cpdim; ++j) { 3455 for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * 1.0 * qweights[ccell * Nc + c] * detJ; 3456 } 3457 /* Update interpolator */ 3458 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat)); 3459 PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices)); 3460 PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim); 3461 PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES)); 3462 } 3463 } 3464 PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3465 } 3466 PetscCall(VecRestoreArray(pointVec, &pV)); 3467 PetscCall(PetscSFDestroy(&coarseCellSF)); 3468 PetscCall(VecDestroy(&pointVec)); 3469 PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3470 } 3471 if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T)); 3472 } 3473 PetscCall(PetscFree3(v0, J, invJ)); 3474 PetscCall(PetscFree3(v0c, Jc, invJc)); 3475 PetscCall(PetscFree(elemMat)); 3476 PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY)); 3477 PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY)); 3478 PetscFunctionReturn(PETSC_SUCCESS); 3479 } 3480 3481 /*@ 3482 DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns 3483 3484 Input Parameters: 3485 + dmc - The coarse mesh 3486 . dmf - The fine mesh 3487 - user - The user context 3488 3489 Output Parameter: 3490 . sc - The mapping 3491 3492 Level: developer 3493 3494 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()` 3495 @*/ 3496 PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user) 3497 { 3498 PetscDS prob; 3499 PetscFE *feRef; 3500 PetscFV *fvRef; 3501 Vec fv, cv; 3502 IS fis, cis; 3503 PetscSection fsection, fglobalSection, csection, cglobalSection; 3504 PetscInt *cmap, *cellCIndices, *cellFIndices, *cindices, *findices; 3505 PetscInt cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m; 3506 PetscBool *needAvg; 3507 3508 PetscFunctionBegin; 3509 PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM, dmc, dmf, 0, 0)); 3510 PetscCall(DMGetDimension(dmf, &dim)); 3511 PetscCall(DMGetLocalSection(dmf, &fsection)); 3512 PetscCall(DMGetGlobalSection(dmf, &fglobalSection)); 3513 PetscCall(DMGetLocalSection(dmc, &csection)); 3514 PetscCall(DMGetGlobalSection(dmc, &cglobalSection)); 3515 PetscCall(PetscSectionGetNumFields(fsection, &Nf)); 3516 PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd)); 3517 PetscCall(DMGetDS(dmc, &prob)); 3518 PetscCall(PetscCalloc3(Nf, &feRef, Nf, &fvRef, Nf, &needAvg)); 3519 for (f = 0; f < Nf; ++f) { 3520 PetscObject obj; 3521 PetscClassId id; 3522 PetscInt fNb = 0, Nc = 0; 3523 3524 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 3525 PetscCall(PetscObjectGetClassId(obj, &id)); 3526 if (id == PETSCFE_CLASSID) { 3527 PetscFE fe = (PetscFE)obj; 3528 PetscSpace sp; 3529 PetscInt maxDegree; 3530 3531 PetscCall(PetscFERefine(fe, &feRef[f])); 3532 PetscCall(PetscFEGetDimension(feRef[f], &fNb)); 3533 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 3534 PetscCall(PetscFEGetBasisSpace(fe, &sp)); 3535 PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree)); 3536 if (!maxDegree) needAvg[f] = PETSC_TRUE; 3537 } else if (id == PETSCFV_CLASSID) { 3538 PetscFV fv = (PetscFV)obj; 3539 PetscDualSpace Q; 3540 3541 PetscCall(PetscFVRefine(fv, &fvRef[f])); 3542 PetscCall(PetscFVGetDualSpace(fvRef[f], &Q)); 3543 PetscCall(PetscDualSpaceGetDimension(Q, &fNb)); 3544 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 3545 needAvg[f] = PETSC_TRUE; 3546 } 3547 fTotDim += fNb; 3548 } 3549 PetscCall(PetscDSGetTotalDimension(prob, &cTotDim)); 3550 PetscCall(PetscMalloc1(cTotDim, &cmap)); 3551 for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) { 3552 PetscFE feC; 3553 PetscFV fvC; 3554 PetscDualSpace QF, QC; 3555 PetscInt order = -1, NcF, NcC, fpdim, cpdim; 3556 3557 if (feRef[field]) { 3558 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&feC)); 3559 PetscCall(PetscFEGetNumComponents(feC, &NcC)); 3560 PetscCall(PetscFEGetNumComponents(feRef[field], &NcF)); 3561 PetscCall(PetscFEGetDualSpace(feRef[field], &QF)); 3562 PetscCall(PetscDualSpaceGetOrder(QF, &order)); 3563 PetscCall(PetscDualSpaceGetDimension(QF, &fpdim)); 3564 PetscCall(PetscFEGetDualSpace(feC, &QC)); 3565 PetscCall(PetscDualSpaceGetDimension(QC, &cpdim)); 3566 } else { 3567 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fvC)); 3568 PetscCall(PetscFVGetNumComponents(fvC, &NcC)); 3569 PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF)); 3570 PetscCall(PetscFVGetDualSpace(fvRef[field], &QF)); 3571 PetscCall(PetscDualSpaceGetDimension(QF, &fpdim)); 3572 PetscCall(PetscFVGetDualSpace(fvC, &QC)); 3573 PetscCall(PetscDualSpaceGetDimension(QC, &cpdim)); 3574 } 3575 PetscCheck(NcF == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC); 3576 for (c = 0; c < cpdim; ++c) { 3577 PetscQuadrature cfunc; 3578 const PetscReal *cqpoints, *cqweights; 3579 PetscInt NqcC, NpC; 3580 PetscBool found = PETSC_FALSE; 3581 3582 PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc)); 3583 PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights)); 3584 PetscCheck(NqcC == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC); 3585 PetscCheck(NpC == 1 || !feRef[field], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments"); 3586 for (f = 0; f < fpdim; ++f) { 3587 PetscQuadrature ffunc; 3588 const PetscReal *fqpoints, *fqweights; 3589 PetscReal sum = 0.0; 3590 PetscInt NqcF, NpF; 3591 3592 PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc)); 3593 PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights)); 3594 PetscCheck(NqcF == NcF, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF); 3595 if (NpC != NpF) continue; 3596 for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]); 3597 if (sum > 1.0e-9) continue; 3598 for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d] * fqweights[d]); 3599 if (sum < 1.0e-9) continue; 3600 cmap[offsetC + c] = offsetF + f; 3601 found = PETSC_TRUE; 3602 break; 3603 } 3604 if (!found) { 3605 /* TODO We really want the average here, but some asshole put VecScatter in the interface */ 3606 if (fvRef[field] || (feRef[field] && order == 0)) { 3607 cmap[offsetC + c] = offsetF + 0; 3608 } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection"); 3609 } 3610 } 3611 offsetC += cpdim; 3612 offsetF += fpdim; 3613 } 3614 for (f = 0; f < Nf; ++f) { 3615 PetscCall(PetscFEDestroy(&feRef[f])); 3616 PetscCall(PetscFVDestroy(&fvRef[f])); 3617 } 3618 PetscCall(PetscFree3(feRef, fvRef, needAvg)); 3619 3620 PetscCall(DMGetGlobalVector(dmf, &fv)); 3621 PetscCall(DMGetGlobalVector(dmc, &cv)); 3622 PetscCall(VecGetOwnershipRange(cv, &startC, &endC)); 3623 PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m)); 3624 PetscCall(PetscMalloc2(cTotDim, &cellCIndices, fTotDim, &cellFIndices)); 3625 PetscCall(PetscMalloc1(m, &cindices)); 3626 PetscCall(PetscMalloc1(m, &findices)); 3627 for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1; 3628 for (c = cStart; c < cEnd; ++c) { 3629 PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices)); 3630 for (d = 0; d < cTotDim; ++d) { 3631 if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue; 3632 PetscCheck(!(findices[cellCIndices[d] - startC] >= 0) || !(findices[cellCIndices[d] - startC] != cellFIndices[cmap[d]]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, c, cindices[cellCIndices[d] - startC], findices[cellCIndices[d] - startC], cellFIndices[cmap[d]]); 3633 cindices[cellCIndices[d] - startC] = cellCIndices[d]; 3634 findices[cellCIndices[d] - startC] = cellFIndices[cmap[d]]; 3635 } 3636 } 3637 PetscCall(PetscFree(cmap)); 3638 PetscCall(PetscFree2(cellCIndices, cellFIndices)); 3639 3640 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis)); 3641 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis)); 3642 PetscCall(VecScatterCreate(cv, cis, fv, fis, sc)); 3643 PetscCall(ISDestroy(&cis)); 3644 PetscCall(ISDestroy(&fis)); 3645 PetscCall(DMRestoreGlobalVector(dmf, &fv)); 3646 PetscCall(DMRestoreGlobalVector(dmc, &cv)); 3647 PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM, dmc, dmf, 0, 0)); 3648 PetscFunctionReturn(PETSC_SUCCESS); 3649 } 3650 3651 /*@C 3652 DMPlexGetCellFields - Retrieve the field values values for a chunk of cells 3653 3654 Input Parameters: 3655 + dm - The `DM` 3656 . cellIS - The cells to include 3657 . locX - A local vector with the solution fields 3658 . locX_t - A local vector with solution field time derivatives, or `NULL` 3659 - locA - A local vector with auxiliary fields, or `NULL` 3660 3661 Output Parameters: 3662 + u - The field coefficients 3663 . u_t - The fields derivative coefficients 3664 - a - The auxiliary field coefficients 3665 3666 Level: developer 3667 3668 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 3669 @*/ 3670 PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, PeOp Vec locX_t, PeOp Vec locA, PetscScalar *u[], PetscScalar *u_t[], PetscScalar *a[]) 3671 { 3672 DM plex, plexA = NULL; 3673 DMEnclosureType encAux; 3674 PetscSection section, sectionAux; 3675 PetscDS prob; 3676 const PetscInt *cells; 3677 PetscInt cStart, cEnd, numCells, totDim, totDimAux, c; 3678 3679 PetscFunctionBegin; 3680 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 3681 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 3682 if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); 3683 if (locA) PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); 3684 PetscAssertPointer(u, 6); 3685 PetscAssertPointer(u_t, 7); 3686 PetscAssertPointer(a, 8); 3687 PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE)); 3688 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 3689 PetscCall(DMGetLocalSection(dm, §ion)); 3690 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL)); 3691 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 3692 if (locA) { 3693 DM dmAux; 3694 PetscDS probAux; 3695 3696 PetscCall(VecGetDM(locA, &dmAux)); 3697 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 3698 PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE)); 3699 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 3700 PetscCall(DMGetDS(dmAux, &probAux)); 3701 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 3702 } 3703 numCells = cEnd - cStart; 3704 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u)); 3705 if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t)); 3706 else *u_t = NULL; 3707 if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a)); 3708 else *a = NULL; 3709 for (c = cStart; c < cEnd; ++c) { 3710 const PetscInt cell = cells ? cells[c] : c; 3711 const PetscInt cind = c - cStart; 3712 PetscScalar *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a; 3713 PetscInt i; 3714 3715 PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x)); 3716 for (i = 0; i < totDim; ++i) ul[cind * totDim + i] = x[i]; 3717 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x)); 3718 if (locX_t) { 3719 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t)); 3720 for (i = 0; i < totDim; ++i) ul_t[cind * totDim + i] = x_t[i]; 3721 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t)); 3722 } 3723 if (locA) { 3724 PetscInt subcell; 3725 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell)); 3726 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x)); 3727 for (i = 0; i < totDimAux; ++i) al[cind * totDimAux + i] = x[i]; 3728 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x)); 3729 } 3730 } 3731 PetscCall(DMDestroy(&plex)); 3732 if (locA) PetscCall(DMDestroy(&plexA)); 3733 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 3734 PetscFunctionReturn(PETSC_SUCCESS); 3735 } 3736 3737 /*@C 3738 DMPlexRestoreCellFields - Restore the field values values for a chunk of cells 3739 3740 Input Parameters: 3741 + dm - The `DM` 3742 . cellIS - The cells to include 3743 . locX - A local vector with the solution fields 3744 . locX_t - A local vector with solution field time derivatives, or `NULL` 3745 - locA - A local vector with auxiliary fields, or `NULL` 3746 3747 Output Parameters: 3748 + u - The field coefficients 3749 . u_t - The fields derivative coefficients 3750 - a - The auxiliary field coefficients 3751 3752 Level: developer 3753 3754 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 3755 @*/ 3756 PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, PeOp Vec locX_t, PeOp Vec locA, PetscScalar *u[], PetscScalar *u_t[], PetscScalar *a[]) 3757 { 3758 PetscFunctionBegin; 3759 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u)); 3760 if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t)); 3761 if (locA) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a)); 3762 PetscFunctionReturn(PETSC_SUCCESS); 3763 } 3764 3765 static PetscErrorCode DMPlexGetHybridCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a) 3766 { 3767 DM plex, plexA = NULL; 3768 DMEnclosureType encAux; 3769 PetscSection section, sectionAux; 3770 PetscDS ds, dsIn; 3771 const PetscInt *cells; 3772 PetscInt cStart, cEnd, numCells, c, totDim, totDimAux, Nf, f; 3773 3774 PetscFunctionBegin; 3775 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 3776 PetscValidHeaderSpecific(cellIS, IS_CLASSID, 2); 3777 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 3778 if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); } 3779 if (locA) { PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); } 3780 PetscAssertPointer(u, 6); 3781 PetscAssertPointer(u_t, 7); 3782 PetscAssertPointer(a, 8); 3783 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 3784 numCells = cEnd - cStart; 3785 PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE)); 3786 PetscCall(DMGetLocalSection(dm, §ion)); 3787 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn)); 3788 PetscCall(PetscDSGetNumFields(dsIn, &Nf)); 3789 PetscCall(PetscDSGetTotalDimension(dsIn, &totDim)); 3790 if (locA) { 3791 DM dmAux; 3792 PetscDS probAux; 3793 3794 PetscCall(VecGetDM(locA, &dmAux)); 3795 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 3796 PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE)); 3797 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 3798 PetscCall(DMGetDS(dmAux, &probAux)); 3799 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 3800 } 3801 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u)); 3802 if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t)); 3803 else { 3804 *u_t = NULL; 3805 } 3806 if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a)); 3807 else { 3808 *a = NULL; 3809 } 3810 // Loop over cohesive cells 3811 for (c = cStart; c < cEnd; ++c) { 3812 const PetscInt cell = cells ? cells[c] : c; 3813 const PetscInt cind = c - cStart; 3814 PetscScalar *xf = NULL, *xc = NULL, *x = NULL, *xf_t = NULL, *xc_t = NULL; 3815 PetscScalar *ul = &(*u)[cind * totDim], *ul_t = PetscSafePointerPlusOffset(*u_t, cind * totDim); 3816 const PetscInt *cone, *ornt; 3817 PetscInt Nx = 0, Nxf, s; 3818 3819 PetscCall(DMPlexGetCone(dm, cell, &cone)); 3820 PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt)); 3821 // Put in cohesive unknowns 3822 PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, &Nxf, &xf)); 3823 if (locX_t) PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &xf_t)); 3824 for (f = 0; f < Nf; ++f) { 3825 PetscInt fdofIn, foff, foffIn; 3826 PetscBool cohesive; 3827 3828 PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive)); 3829 if (!cohesive) continue; 3830 PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn)); 3831 PetscCall(PetscDSGetFieldOffsetCohesive(ds, f, &foff)); 3832 PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn)); 3833 for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + i] = xf[foff + i]; 3834 if (locX_t) 3835 for (PetscInt i = 0; i < fdofIn; ++i) ul_t[foffIn + i] = xf_t[foff + i]; 3836 Nx += fdofIn; 3837 } 3838 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, &Nxf, &xf)); 3839 if (locX_t) PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &xf_t)); 3840 // Loop over sides of surface 3841 for (s = 0; s < 2; ++s) { 3842 const PetscInt *support; 3843 const PetscInt face = cone[s]; 3844 PetscDS dsC; 3845 PetscInt ssize, ncell, Nxc; 3846 3847 // I don't think I need the face to have 0 orientation in the hybrid cell 3848 //PetscCheck(!ornt[s], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", face, cell, ornt[s]); 3849 PetscCall(DMPlexGetSupport(dm, face, &support)); 3850 PetscCall(DMPlexGetSupportSize(dm, face, &ssize)); 3851 if (support[0] == cell) ncell = support[1]; 3852 else if (support[1] == cell) ncell = support[0]; 3853 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell); 3854 // Get closure of both face and cell, stick in cell for normal fields and face for cohesive fields 3855 PetscCall(DMGetCellDS(dm, ncell, &dsC, NULL)); 3856 PetscCall(DMPlexVecGetClosure(plex, section, locX, ncell, &Nxc, &xc)); 3857 if (locX_t) PetscCall(DMPlexVecGetClosure(plex, section, locX_t, ncell, NULL, &xc_t)); 3858 for (f = 0; f < Nf; ++f) { 3859 PetscInt fdofIn, foffIn, foff; 3860 PetscBool cohesive; 3861 3862 PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive)); 3863 if (cohesive) continue; 3864 PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn)); 3865 PetscCall(PetscDSGetFieldOffset(dsC, f, &foff)); 3866 PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn)); 3867 for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + s * fdofIn + i] = xc[foff + i]; 3868 if (locX_t) 3869 for (PetscInt i = 0; i < fdofIn; ++i) ul_t[foffIn + s * fdofIn + i] = xc_t[foff + i]; 3870 Nx += fdofIn; 3871 } 3872 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, ncell, &Nxc, &xc)); 3873 if (locX_t) PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, ncell, NULL, &xc_t)); 3874 } 3875 PetscCheck(Nx == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for cell %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, cell, totDim); 3876 3877 if (locA) { 3878 PetscScalar *al = &(*a)[cind * totDimAux]; 3879 PetscInt subcell; 3880 3881 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell)); 3882 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, &Nx, &x)); 3883 PetscCheck(Nx == totDimAux, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subcell %" PetscInt_FMT "does not match DS size %" PetscInt_FMT, Nx, subcell, totDimAux); 3884 for (PetscInt i = 0; i < totDimAux; ++i) al[i] = x[i]; 3885 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, &Nx, &x)); 3886 } 3887 } 3888 PetscCall(DMDestroy(&plex)); 3889 PetscCall(DMDestroy(&plexA)); 3890 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 3891 PetscFunctionReturn(PETSC_SUCCESS); 3892 } 3893 3894 /* 3895 DMPlexGetHybridFields - Get the field values for the negative side (s = 0) and positive side (s = 1) of the interface 3896 3897 Input Parameters: 3898 + dm - The full domain DM 3899 . dmX - An array of DM for the field, say an auxiliary DM, indexed by s 3900 . dsX - An array of PetscDS for the field, indexed by s 3901 . cellIS - The interface cells for which we want values 3902 . locX - An array of local vectors with the field values, indexed by s 3903 - useCell - Flag to have values come from neighboring cell rather than endcap face 3904 3905 Output Parameter: 3906 . x - An array of field values, indexed by s 3907 3908 Note: 3909 The arrays in `x` will be allocated using `DMGetWorkArray()`, and must be returned using `DMPlexRestoreHybridFields()`. 3910 3911 Level: advanced 3912 3913 .seealso: `DMPlexRestoreHybridFields()`, `DMGetWorkArray()` 3914 */ 3915 static PetscErrorCode DMPlexGetHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[]) 3916 { 3917 DM plexX[2]; 3918 DMEnclosureType encX[2]; 3919 PetscSection sectionX[2]; 3920 const PetscInt *cells; 3921 PetscInt cStart, cEnd, numCells, c, s, totDimX[2]; 3922 3923 PetscFunctionBegin; 3924 PetscAssertPointer(locX, 5); 3925 if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS); 3926 PetscAssertPointer(dmX, 2); 3927 PetscAssertPointer(dsX, 3); 3928 PetscValidHeaderSpecific(cellIS, IS_CLASSID, 4); 3929 PetscAssertPointer(x, 7); 3930 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 3931 numCells = cEnd - cStart; 3932 for (s = 0; s < 2; ++s) { 3933 PetscValidHeaderSpecific(dmX[s], DM_CLASSID, 2); 3934 PetscValidHeaderSpecific(dsX[s], PETSCDS_CLASSID, 3); 3935 PetscValidHeaderSpecific(locX[s], VEC_CLASSID, 5); 3936 PetscCall(DMPlexConvertPlex(dmX[s], &plexX[s], PETSC_FALSE)); 3937 PetscCall(DMGetEnclosureRelation(dmX[s], dm, &encX[s])); 3938 PetscCall(DMGetLocalSection(dmX[s], §ionX[s])); 3939 PetscCall(PetscDSGetTotalDimension(dsX[s], &totDimX[s])); 3940 PetscCall(DMGetWorkArray(dmX[s], numCells * totDimX[s], MPIU_SCALAR, &x[s])); 3941 } 3942 for (c = cStart; c < cEnd; ++c) { 3943 const PetscInt cell = cells ? cells[c] : c; 3944 const PetscInt cind = c - cStart; 3945 const PetscInt *cone, *ornt; 3946 3947 PetscCall(DMPlexGetCone(dm, cell, &cone)); 3948 PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt)); 3949 //PetscCheck(!ornt[0], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", cone[0], cell, ornt[0]); 3950 for (s = 0; s < 2; ++s) { 3951 const PetscInt tdX = totDimX[s]; 3952 PetscScalar *closure = NULL, *xl = &x[s][cind * tdX]; 3953 PetscInt face = cone[s], point = face, subpoint, Nx, i; 3954 3955 if (useCell) { 3956 const PetscInt *support; 3957 PetscInt ssize; 3958 3959 PetscCall(DMPlexGetSupport(dm, face, &support)); 3960 PetscCall(DMPlexGetSupportSize(dm, face, &ssize)); 3961 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", face, cell, ssize); 3962 if (support[0] == cell) point = support[1]; 3963 else if (support[1] == cell) point = support[0]; 3964 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell); 3965 } 3966 PetscCall(DMGetEnclosurePoint(plexX[s], dm, encX[s], point, &subpoint)); 3967 PetscCall(DMPlexVecGetOrientedClosure_Internal(plexX[s], sectionX[s], PETSC_FALSE, locX[s], subpoint, ornt[s], &Nx, &closure)); 3968 PetscCheck(Nx == tdX, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subpoint %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, subpoint, tdX); 3969 for (i = 0; i < Nx; ++i) xl[i] = closure[i]; 3970 PetscCall(DMPlexVecRestoreClosure(plexX[s], sectionX[s], locX[s], subpoint, &Nx, &closure)); 3971 } 3972 } 3973 for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexX[s])); 3974 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 3975 PetscFunctionReturn(PETSC_SUCCESS); 3976 } 3977 3978 static PetscErrorCode DMPlexRestoreHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[]) 3979 { 3980 PetscFunctionBegin; 3981 if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS); 3982 PetscCall(DMRestoreWorkArray(dmX[0], 0, MPIU_SCALAR, &x[0])); 3983 PetscCall(DMRestoreWorkArray(dmX[1], 0, MPIU_SCALAR, &x[1])); 3984 PetscFunctionReturn(PETSC_SUCCESS); 3985 } 3986 3987 /*@C 3988 DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces 3989 3990 Input Parameters: 3991 + dm - The `DM` 3992 . fStart - The first face to include 3993 . fEnd - The first face to exclude 3994 . locX - A local vector with the solution fields 3995 . locX_t - A local vector with solution field time derivatives, or `NULL` 3996 . faceGeometry - A local vector with face geometry 3997 . cellGeometry - A local vector with cell geometry 3998 - locGrad - A local vector with field gradients, or `NULL` 3999 4000 Output Parameters: 4001 + Nface - The number of faces with field values 4002 . uL - The field values at the left side of the face 4003 - uR - The field values at the right side of the face 4004 4005 Level: developer 4006 4007 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()` 4008 @*/ 4009 PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, PeOp Vec locX_t, Vec faceGeometry, Vec cellGeometry, PeOp Vec locGrad, PetscInt *Nface, PetscScalar *uL[], PetscScalar *uR[]) 4010 { 4011 DM dmFace, dmCell, dmGrad = NULL; 4012 PetscSection section; 4013 PetscDS prob; 4014 DMLabel ghostLabel; 4015 const PetscScalar *facegeom, *cellgeom, *x, *lgrad; 4016 PetscBool *isFE; 4017 PetscInt dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face; 4018 4019 PetscFunctionBegin; 4020 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4021 PetscValidHeaderSpecific(locX, VEC_CLASSID, 4); 4022 if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5); 4023 PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6); 4024 PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7); 4025 if (locGrad) PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8); 4026 PetscAssertPointer(uL, 10); 4027 PetscAssertPointer(uR, 11); 4028 PetscCall(DMGetDimension(dm, &dim)); 4029 PetscCall(DMGetDS(dm, &prob)); 4030 PetscCall(DMGetLocalSection(dm, §ion)); 4031 PetscCall(PetscDSGetNumFields(prob, &Nf)); 4032 PetscCall(PetscDSGetTotalComponents(prob, &Nc)); 4033 PetscCall(PetscMalloc1(Nf, &isFE)); 4034 for (f = 0; f < Nf; ++f) { 4035 PetscObject obj; 4036 PetscClassId id; 4037 4038 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4039 PetscCall(PetscObjectGetClassId(obj, &id)); 4040 if (id == PETSCFE_CLASSID) { 4041 isFE[f] = PETSC_TRUE; 4042 } else if (id == PETSCFV_CLASSID) { 4043 isFE[f] = PETSC_FALSE; 4044 } else { 4045 isFE[f] = PETSC_FALSE; 4046 } 4047 } 4048 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 4049 PetscCall(VecGetArrayRead(locX, &x)); 4050 PetscCall(VecGetDM(faceGeometry, &dmFace)); 4051 PetscCall(VecGetArrayRead(faceGeometry, &facegeom)); 4052 PetscCall(VecGetDM(cellGeometry, &dmCell)); 4053 PetscCall(VecGetArrayRead(cellGeometry, &cellgeom)); 4054 if (locGrad) { 4055 PetscCall(VecGetDM(locGrad, &dmGrad)); 4056 PetscCall(VecGetArrayRead(locGrad, &lgrad)); 4057 } 4058 PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uL)); 4059 PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uR)); 4060 /* Right now just eat the extra work for FE (could make a cell loop) */ 4061 for (face = fStart, iface = 0; face < fEnd; ++face) { 4062 const PetscInt *cells; 4063 PetscFVFaceGeom *fg; 4064 PetscFVCellGeom *cgL, *cgR; 4065 PetscScalar *xL, *xR, *gL, *gR; 4066 PetscScalar *uLl = *uL, *uRl = *uR; 4067 PetscInt ghost, nsupp, nchild; 4068 4069 PetscCall(DMLabelGetValue(ghostLabel, face, &ghost)); 4070 PetscCall(DMPlexGetSupportSize(dm, face, &nsupp)); 4071 PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL)); 4072 if (ghost >= 0 || nsupp > 2 || nchild > 0) continue; 4073 PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg)); 4074 PetscCall(DMPlexGetSupport(dm, face, &cells)); 4075 PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL)); 4076 PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR)); 4077 for (f = 0; f < Nf; ++f) { 4078 PetscInt off; 4079 4080 PetscCall(PetscDSGetComponentOffset(prob, f, &off)); 4081 if (isFE[f]) { 4082 const PetscInt *cone; 4083 PetscInt comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d; 4084 4085 xL = xR = NULL; 4086 PetscCall(PetscSectionGetFieldComponents(section, f, &comp)); 4087 PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, &xL)); 4088 PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, &xR)); 4089 PetscCall(DMPlexGetCone(dm, cells[0], &cone)); 4090 PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL)); 4091 for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL) 4092 if (cone[faceLocL] == face) break; 4093 PetscCall(DMPlexGetCone(dm, cells[1], &cone)); 4094 PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR)); 4095 for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR) 4096 if (cone[faceLocR] == face) break; 4097 PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]); 4098 /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */ 4099 /* TODO: this is a hack that might not be right for nonconforming */ 4100 if (faceLocL < coneSizeL) { 4101 PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface * Nc + off])); 4102 if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off])); 4103 else { 4104 for (d = 0; d < comp; ++d) uRl[iface * Nc + off + d] = uLl[iface * Nc + off + d]; 4105 } 4106 } else { 4107 PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off])); 4108 PetscCall(PetscSectionGetFieldComponents(section, f, &comp)); 4109 for (d = 0; d < comp; ++d) uLl[iface * Nc + off + d] = uRl[iface * Nc + off + d]; 4110 } 4111 PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, &xL)); 4112 PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, &xR)); 4113 } else { 4114 PetscFV fv; 4115 PetscInt numComp, c; 4116 4117 PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *)&fv)); 4118 PetscCall(PetscFVGetNumComponents(fv, &numComp)); 4119 PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL)); 4120 PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR)); 4121 if (dmGrad) { 4122 PetscReal dxL[3], dxR[3]; 4123 4124 PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL)); 4125 PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR)); 4126 DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL); 4127 DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR); 4128 for (c = 0; c < numComp; ++c) { 4129 uLl[iface * Nc + off + c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c * dim], dxL); 4130 uRl[iface * Nc + off + c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c * dim], dxR); 4131 } 4132 } else { 4133 for (c = 0; c < numComp; ++c) { 4134 uLl[iface * Nc + off + c] = xL[c]; 4135 uRl[iface * Nc + off + c] = xR[c]; 4136 } 4137 } 4138 } 4139 } 4140 ++iface; 4141 } 4142 *Nface = iface; 4143 PetscCall(VecRestoreArrayRead(locX, &x)); 4144 PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom)); 4145 PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom)); 4146 if (locGrad) PetscCall(VecRestoreArrayRead(locGrad, &lgrad)); 4147 PetscCall(PetscFree(isFE)); 4148 PetscFunctionReturn(PETSC_SUCCESS); 4149 } 4150 4151 /*@C 4152 DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces 4153 4154 Input Parameters: 4155 + dm - The `DM` 4156 . fStart - The first face to include 4157 . fEnd - The first face to exclude 4158 . locX - A local vector with the solution fields 4159 . locX_t - A local vector with solution field time derivatives, or `NULL` 4160 . faceGeometry - A local vector with face geometry 4161 . cellGeometry - A local vector with cell geometry 4162 - locGrad - A local vector with field gradients, or `NULL` 4163 4164 Output Parameters: 4165 + Nface - The number of faces with field values 4166 . uL - The field values at the left side of the face 4167 - uR - The field values at the right side of the face 4168 4169 Level: developer 4170 4171 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 4172 @*/ 4173 PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, PeOp Vec locX_t, Vec faceGeometry, Vec cellGeometry, PeOp Vec locGrad, PetscInt *Nface, PetscScalar *uL[], PetscScalar *uR[]) 4174 { 4175 PetscFunctionBegin; 4176 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL)); 4177 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR)); 4178 PetscFunctionReturn(PETSC_SUCCESS); 4179 } 4180 4181 /*@C 4182 DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces 4183 4184 Input Parameters: 4185 + dm - The `DM` 4186 . fStart - The first face to include 4187 . fEnd - The first face to exclude 4188 . faceGeometry - A local vector with face geometry 4189 - cellGeometry - A local vector with cell geometry 4190 4191 Output Parameters: 4192 + Nface - The number of faces with field values 4193 . fgeom - The face centroid and normals 4194 - vol - The cell volumes 4195 4196 Level: developer 4197 4198 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()` 4199 @*/ 4200 PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom *fgeom[], PetscReal *vol[]) 4201 { 4202 DM dmFace, dmCell; 4203 DMLabel ghostLabel; 4204 const PetscScalar *facegeom, *cellgeom; 4205 PetscInt dim, numFaces = fEnd - fStart, iface, face; 4206 4207 PetscFunctionBegin; 4208 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4209 PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4); 4210 PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5); 4211 PetscAssertPointer(fgeom, 7); 4212 PetscAssertPointer(vol, 8); 4213 PetscCall(DMGetDimension(dm, &dim)); 4214 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 4215 PetscCall(VecGetDM(faceGeometry, &dmFace)); 4216 PetscCall(VecGetArrayRead(faceGeometry, &facegeom)); 4217 PetscCall(VecGetDM(cellGeometry, &dmCell)); 4218 PetscCall(VecGetArrayRead(cellGeometry, &cellgeom)); 4219 PetscCall(PetscMalloc1(numFaces, fgeom)); 4220 PetscCall(DMGetWorkArray(dm, numFaces * 2, MPIU_SCALAR, vol)); 4221 for (face = fStart, iface = 0; face < fEnd; ++face) { 4222 const PetscInt *cells; 4223 PetscFVFaceGeom *fg; 4224 PetscFVCellGeom *cgL, *cgR; 4225 PetscFVFaceGeom *fgeoml = *fgeom; 4226 PetscReal *voll = *vol; 4227 PetscInt ghost, d, nchild, nsupp; 4228 4229 PetscCall(DMLabelGetValue(ghostLabel, face, &ghost)); 4230 PetscCall(DMPlexGetSupportSize(dm, face, &nsupp)); 4231 PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL)); 4232 if (ghost >= 0 || nsupp > 2 || nchild > 0) continue; 4233 PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg)); 4234 PetscCall(DMPlexGetSupport(dm, face, &cells)); 4235 PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL)); 4236 PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR)); 4237 for (d = 0; d < dim; ++d) { 4238 fgeoml[iface].centroid[d] = fg->centroid[d]; 4239 fgeoml[iface].normal[d] = fg->normal[d]; 4240 } 4241 voll[iface * 2 + 0] = cgL->volume; 4242 voll[iface * 2 + 1] = cgR->volume; 4243 ++iface; 4244 } 4245 *Nface = iface; 4246 PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom)); 4247 PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom)); 4248 PetscFunctionReturn(PETSC_SUCCESS); 4249 } 4250 4251 /*@C 4252 DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces 4253 4254 Input Parameters: 4255 + dm - The `DM` 4256 . fStart - The first face to include 4257 . fEnd - The first face to exclude 4258 . faceGeometry - A local vector with face geometry 4259 - cellGeometry - A local vector with cell geometry 4260 4261 Output Parameters: 4262 + Nface - The number of faces with field values 4263 . fgeom - The face centroid and normals 4264 - vol - The cell volumes 4265 4266 Level: developer 4267 4268 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 4269 @*/ 4270 PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom *fgeom[], PetscReal *vol[]) 4271 { 4272 PetscFunctionBegin; 4273 PetscCall(PetscFree(*fgeom)); 4274 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol)); 4275 PetscFunctionReturn(PETSC_SUCCESS); 4276 } 4277 4278 PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom) 4279 { 4280 char composeStr[33] = {0}; 4281 PetscObjectId id; 4282 PetscContainer container; 4283 4284 PetscFunctionBegin; 4285 PetscCall(PetscObjectGetId((PetscObject)quad, &id)); 4286 PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id)); 4287 PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container)); 4288 if (container) { 4289 PetscCall(PetscContainerGetPointer(container, (void **)geom)); 4290 } else { 4291 PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, mode, geom)); 4292 PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container)); 4293 PetscCall(PetscContainerSetPointer(container, (void *)*geom)); 4294 PetscCall(PetscContainerSetCtxDestroy(container, PetscContainerCtxDestroy_PetscFEGeom)); 4295 PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container)); 4296 PetscCall(PetscContainerDestroy(&container)); 4297 } 4298 PetscFunctionReturn(PETSC_SUCCESS); 4299 } 4300 4301 PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) 4302 { 4303 PetscFunctionBegin; 4304 *geom = NULL; 4305 PetscFunctionReturn(PETSC_SUCCESS); 4306 } 4307 4308 PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user) 4309 { 4310 DM_Plex *mesh = (DM_Plex *)dm->data; 4311 const char *name = "Residual"; 4312 DM dmAux = NULL; 4313 DMLabel ghostLabel = NULL; 4314 PetscDS prob = NULL; 4315 PetscDS probAux = NULL; 4316 PetscBool useFEM = PETSC_FALSE; 4317 PetscBool isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE; 4318 DMField coordField = NULL; 4319 Vec locA; 4320 PetscScalar *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL; 4321 IS chunkIS; 4322 const PetscInt *cells; 4323 PetscInt cStart, cEnd, numCells; 4324 PetscInt Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd; 4325 PetscInt maxDegree = PETSC_INT_MAX; 4326 PetscFormKey key; 4327 PetscQuadrature affineQuad = NULL, *quads = NULL; 4328 PetscFEGeom *affineGeom = NULL, **geoms = NULL; 4329 4330 PetscFunctionBegin; 4331 PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 4332 /* FEM+FVM */ 4333 /* 1: Get sizes from dm and dmAux */ 4334 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 4335 PetscCall(DMGetDS(dm, &prob)); 4336 PetscCall(PetscDSGetNumFields(prob, &Nf)); 4337 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 4338 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA)); 4339 if (locA) { 4340 PetscCall(VecGetDM(locA, &dmAux)); 4341 PetscCall(DMGetDS(dmAux, &probAux)); 4342 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 4343 } 4344 /* 2: Get geometric data */ 4345 for (f = 0; f < Nf; ++f) { 4346 PetscObject obj; 4347 PetscClassId id; 4348 PetscBool fimp; 4349 4350 PetscCall(PetscDSGetImplicit(prob, f, &fimp)); 4351 if (isImplicit != fimp) continue; 4352 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4353 PetscCall(PetscObjectGetClassId(obj, &id)); 4354 if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE; 4355 PetscCheck(id != PETSCFV_CLASSID, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Use of FVM with PCPATCH not yet implemented"); 4356 } 4357 if (useFEM) { 4358 PetscCall(DMGetCoordinateField(dm, &coordField)); 4359 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 4360 if (maxDegree <= 1) { 4361 PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad)); 4362 if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FEGEOM_BASIC, &affineGeom)); 4363 } else { 4364 PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms)); 4365 for (f = 0; f < Nf; ++f) { 4366 PetscObject obj; 4367 PetscClassId id; 4368 PetscBool fimp; 4369 4370 PetscCall(PetscDSGetImplicit(prob, f, &fimp)); 4371 if (isImplicit != fimp) continue; 4372 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4373 PetscCall(PetscObjectGetClassId(obj, &id)); 4374 if (id == PETSCFE_CLASSID) { 4375 PetscFE fe = (PetscFE)obj; 4376 4377 PetscCall(PetscFEGetQuadrature(fe, &quads[f])); 4378 PetscCall(PetscObjectReference((PetscObject)quads[f])); 4379 PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FEGEOM_BASIC, &geoms[f])); 4380 } 4381 } 4382 } 4383 } 4384 /* Loop over chunks */ 4385 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 4386 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 4387 if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS)); 4388 numCells = cEnd - cStart; 4389 numChunks = 1; 4390 cellChunkSize = numCells / numChunks; 4391 numChunks = PetscMin(1, numCells); 4392 key.label = NULL; 4393 key.value = 0; 4394 key.part = 0; 4395 for (chunk = 0; chunk < numChunks; ++chunk) { 4396 PetscScalar *elemVec, *fluxL = NULL, *fluxR = NULL; 4397 PetscReal *vol = NULL; 4398 PetscFVFaceGeom *fgeom = NULL; 4399 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 4400 PetscInt numFaces = 0; 4401 4402 /* Extract field coefficients */ 4403 if (useFEM) { 4404 PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells)); 4405 PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 4406 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 4407 PetscCall(PetscArrayzero(elemVec, numCells * totDim)); 4408 } 4409 /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */ 4410 /* Loop over fields */ 4411 for (f = 0; f < Nf; ++f) { 4412 PetscObject obj; 4413 PetscClassId id; 4414 PetscBool fimp; 4415 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset; 4416 4417 key.field = f; 4418 PetscCall(PetscDSGetImplicit(prob, f, &fimp)); 4419 if (isImplicit != fimp) continue; 4420 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4421 PetscCall(PetscObjectGetClassId(obj, &id)); 4422 if (id == PETSCFE_CLASSID) { 4423 PetscFE fe = (PetscFE)obj; 4424 PetscFEGeom *geom = affineGeom ? affineGeom : geoms[f]; 4425 PetscFEGeom *chunkGeom = NULL; 4426 PetscQuadrature quad = affineQuad ? affineQuad : quads[f]; 4427 PetscInt Nq, Nb; 4428 4429 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4430 PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL)); 4431 PetscCall(PetscFEGetDimension(fe, &Nb)); 4432 blockSize = Nb; 4433 batchSize = numBlocks * blockSize; 4434 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4435 numChunks = numCells / (numBatches * batchSize); 4436 Ne = numChunks * numBatches * batchSize; 4437 Nr = numCells % (numBatches * batchSize); 4438 offset = numCells - Nr; 4439 /* Integrate FE residual to get elemVec (need fields at quadrature points) */ 4440 /* For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */ 4441 PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom)); 4442 PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec)); 4443 PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom)); 4444 PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, &a[offset * totDimAux], t, &elemVec[offset * totDim])); 4445 PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom)); 4446 } else if (id == PETSCFV_CLASSID) { 4447 PetscFV fv = (PetscFV)obj; 4448 4449 Ne = numFaces; 4450 /* Riemann solve over faces (need fields at face centroids) */ 4451 /* We need to evaluate FE fields at those coordinates */ 4452 PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR)); 4453 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 4454 } 4455 /* Loop over domain */ 4456 if (useFEM) { 4457 /* Add elemVec to locX */ 4458 for (c = cS; c < cE; ++c) { 4459 const PetscInt cell = cells ? cells[c] : c; 4460 const PetscInt cind = c - cStart; 4461 4462 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim])); 4463 if (ghostLabel) { 4464 PetscInt ghostVal; 4465 4466 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 4467 if (ghostVal > 0) continue; 4468 } 4469 PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES)); 4470 } 4471 } 4472 /* Handle time derivative */ 4473 if (locX_t) { 4474 PetscScalar *x_t, *fa; 4475 4476 PetscCall(VecGetArray(locF, &fa)); 4477 PetscCall(VecGetArray(locX_t, &x_t)); 4478 for (f = 0; f < Nf; ++f) { 4479 PetscFV fv; 4480 PetscObject obj; 4481 PetscClassId id; 4482 PetscInt pdim, d; 4483 4484 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4485 PetscCall(PetscObjectGetClassId(obj, &id)); 4486 if (id != PETSCFV_CLASSID) continue; 4487 fv = (PetscFV)obj; 4488 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 4489 for (c = cS; c < cE; ++c) { 4490 const PetscInt cell = cells ? cells[c] : c; 4491 PetscScalar *u_t, *r; 4492 4493 if (ghostLabel) { 4494 PetscInt ghostVal; 4495 4496 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 4497 if (ghostVal > 0) continue; 4498 } 4499 PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t)); 4500 PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r)); 4501 for (d = 0; d < pdim; ++d) r[d] += u_t[d]; 4502 } 4503 } 4504 PetscCall(VecRestoreArray(locX_t, &x_t)); 4505 PetscCall(VecRestoreArray(locF, &fa)); 4506 } 4507 if (useFEM) { 4508 PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 4509 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 4510 } 4511 } 4512 if (useFEM) PetscCall(ISDestroy(&chunkIS)); 4513 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 4514 /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */ 4515 if (useFEM) { 4516 if (maxDegree <= 1) { 4517 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 4518 PetscCall(PetscQuadratureDestroy(&affineQuad)); 4519 } else { 4520 for (f = 0; f < Nf; ++f) { 4521 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 4522 PetscCall(PetscQuadratureDestroy(&quads[f])); 4523 } 4524 PetscCall(PetscFree2(quads, geoms)); 4525 } 4526 } 4527 PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 4528 PetscFunctionReturn(PETSC_SUCCESS); 4529 } 4530 4531 /* 4532 We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac 4533 4534 X - The local solution vector 4535 X_t - The local solution time derivative vector, or NULL 4536 */ 4537 PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx) 4538 { 4539 DM_Plex *mesh = (DM_Plex *)dm->data; 4540 const char *name = "Jacobian", *nameP = "JacobianPre"; 4541 DM dmAux = NULL; 4542 PetscDS prob, probAux = NULL; 4543 PetscSection sectionAux = NULL; 4544 Vec A; 4545 DMField coordField; 4546 PetscFEGeom *cgeomFEM; 4547 PetscQuadrature qGeom = NULL; 4548 Mat J = Jac, JP = JacP; 4549 PetscScalar *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL; 4550 PetscBool hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE; 4551 const PetscInt *cells; 4552 PetscFormKey key; 4553 PetscInt Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0; 4554 4555 PetscFunctionBegin; 4556 PetscCall(ISGetLocalSize(cellIS, &numCells)); 4557 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 4558 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 4559 PetscCall(DMGetDS(dm, &prob)); 4560 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A)); 4561 if (A) { 4562 PetscCall(VecGetDM(A, &dmAux)); 4563 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 4564 PetscCall(DMGetDS(dmAux, &probAux)); 4565 } 4566 /* Get flags */ 4567 PetscCall(PetscDSGetNumFields(prob, &Nf)); 4568 PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE)); 4569 for (fieldI = 0; fieldI < Nf; ++fieldI) { 4570 PetscObject disc; 4571 PetscClassId id; 4572 PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc)); 4573 PetscCall(PetscObjectGetClassId(disc, &id)); 4574 if (id == PETSCFE_CLASSID) { 4575 isFE[fieldI] = PETSC_TRUE; 4576 } else if (id == PETSCFV_CLASSID) { 4577 hasFV = PETSC_TRUE; 4578 isFE[fieldI] = PETSC_FALSE; 4579 } 4580 } 4581 PetscCall(PetscDSHasJacobian(prob, &hasJac)); 4582 PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec)); 4583 PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn)); 4584 assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE; 4585 hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE; 4586 if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */ 4587 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 4588 if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 4589 /* Compute batch sizes */ 4590 if (isFE[0]) { 4591 PetscFE fe; 4592 PetscQuadrature q; 4593 PetscInt numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb; 4594 4595 PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe)); 4596 PetscCall(PetscFEGetQuadrature(fe, &q)); 4597 PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL)); 4598 PetscCall(PetscFEGetDimension(fe, &Nb)); 4599 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4600 blockSize = Nb * numQuadPoints; 4601 batchSize = numBlocks * blockSize; 4602 chunkSize = numBatches * batchSize; 4603 numChunks = numCells / chunkSize + numCells % chunkSize; 4604 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4605 } else { 4606 chunkSize = numCells; 4607 numChunks = 1; 4608 } 4609 /* Get work space */ 4610 wsz = (((X ? 1 : 0) + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize; 4611 PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work)); 4612 PetscCall(PetscArrayzero(work, wsz)); 4613 off = 0; 4614 u = X ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL; 4615 u_t = X_t ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL; 4616 a = dmAux ? (sz = chunkSize * totDimAux, off += sz, work + off - sz) : NULL; 4617 elemMat = hasJac ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL; 4618 elemMatP = hasPrec ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL; 4619 elemMatD = hasDyn ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL; 4620 PetscCheck(off == wsz, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz); 4621 /* Setup geometry */ 4622 PetscCall(DMGetCoordinateField(dm, &coordField)); 4623 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 4624 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom)); 4625 if (!qGeom) { 4626 PetscFE fe; 4627 4628 PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe)); 4629 PetscCall(PetscFEGetQuadrature(fe, &qGeom)); 4630 PetscCall(PetscObjectReference((PetscObject)qGeom)); 4631 } 4632 PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM)); 4633 /* Compute volume integrals */ 4634 if (assembleJac) PetscCall(MatZeroEntries(J)); 4635 PetscCall(MatZeroEntries(JP)); 4636 key.label = NULL; 4637 key.value = 0; 4638 key.part = 0; 4639 for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) { 4640 const PetscInt Ncell = PetscMin(chunkSize, numCells - offCell); 4641 PetscInt c; 4642 4643 /* Extract values */ 4644 for (c = 0; c < Ncell; ++c) { 4645 const PetscInt cell = cells ? cells[c + offCell] : c + offCell; 4646 PetscScalar *x = NULL, *x_t = NULL; 4647 PetscInt i; 4648 4649 if (X) { 4650 PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x)); 4651 for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i]; 4652 PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x)); 4653 } 4654 if (X_t) { 4655 PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t)); 4656 for (i = 0; i < totDim; ++i) u_t[c * totDim + i] = x_t[i]; 4657 PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t)); 4658 } 4659 if (dmAux) { 4660 PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x)); 4661 for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i]; 4662 PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x)); 4663 } 4664 } 4665 for (fieldI = 0; fieldI < Nf; ++fieldI) { 4666 PetscFE fe; 4667 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 4668 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 4669 key.field = fieldI * Nf + fieldJ; 4670 if (hasJac) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat)); 4671 if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP)); 4672 if (hasDyn) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD)); 4673 } 4674 /* For finite volume, add the identity */ 4675 if (!isFE[fieldI]) { 4676 PetscFV fv; 4677 PetscInt eOffset = 0, Nc, fc, foff; 4678 4679 PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff)); 4680 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv)); 4681 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 4682 for (c = 0; c < chunkSize; ++c, eOffset += totDim * totDim) { 4683 for (fc = 0; fc < Nc; ++fc) { 4684 const PetscInt i = foff + fc; 4685 if (hasJac) elemMat[eOffset + i * totDim + i] = 1.0; 4686 if (hasPrec) elemMatP[eOffset + i * totDim + i] = 1.0; 4687 } 4688 } 4689 } 4690 } 4691 /* Add contribution from X_t */ 4692 if (hasDyn) { 4693 for (c = 0; c < chunkSize * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c]; 4694 } 4695 /* Insert values into matrix */ 4696 for (c = 0; c < Ncell; ++c) { 4697 const PetscInt cell = cells ? cells[c + offCell] : c + offCell; 4698 if (mesh->printFEM > 1) { 4699 if (hasJac) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[(c - cStart) * totDim * totDim])); 4700 if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c - cStart) * totDim * totDim])); 4701 } 4702 if (assembleJac) PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES)); 4703 PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JP, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES)); 4704 } 4705 } 4706 /* Cleanup */ 4707 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 4708 PetscCall(PetscQuadratureDestroy(&qGeom)); 4709 if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE)); 4710 PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE)); 4711 PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize, MPIU_SCALAR, &work)); 4712 /* Compute boundary integrals */ 4713 /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */ 4714 /* Assemble matrix */ 4715 if (assembleJac) { 4716 PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY)); 4717 PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY)); 4718 } 4719 PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY)); 4720 PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY)); 4721 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 4722 PetscFunctionReturn(PETSC_SUCCESS); 4723 } 4724 4725 /* FEM Assembly Function */ 4726 4727 static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy) 4728 { 4729 PetscBool isPlex; 4730 4731 PetscFunctionBegin; 4732 PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex)); 4733 if (isPlex) { 4734 *plex = dm; 4735 PetscCall(PetscObjectReference((PetscObject)dm)); 4736 } else { 4737 PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex)); 4738 if (!*plex) { 4739 PetscCall(DMConvert(dm, DMPLEX, plex)); 4740 PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex)); 4741 } else { 4742 PetscCall(PetscObjectReference((PetscObject)*plex)); 4743 } 4744 if (copy) PetscCall(DMCopyAuxiliaryVec(dm, *plex)); 4745 } 4746 PetscFunctionReturn(PETSC_SUCCESS); 4747 } 4748 4749 /*@ 4750 DMPlexGetGeometryFVM - Return precomputed geometric data 4751 4752 Collective 4753 4754 Input Parameter: 4755 . dm - The `DM` 4756 4757 Output Parameters: 4758 + facegeom - The values precomputed from face geometry 4759 . cellgeom - The values precomputed from cell geometry 4760 - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell, or `NULL` if not needed 4761 4762 Level: developer 4763 4764 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMTSSetRHSFunctionLocal()` 4765 @*/ 4766 PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PeOp PetscReal *minRadius) 4767 { 4768 DM plex; 4769 4770 PetscFunctionBegin; 4771 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4772 PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE)); 4773 PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL)); 4774 if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius)); 4775 PetscCall(DMDestroy(&plex)); 4776 PetscFunctionReturn(PETSC_SUCCESS); 4777 } 4778 4779 /*@ 4780 DMPlexGetGradientDM - Return gradient data layout 4781 4782 Collective 4783 4784 Input Parameters: 4785 + dm - The `DM` 4786 - fv - The `PetscFV` 4787 4788 Output Parameter: 4789 . dmGrad - The layout for gradient values 4790 4791 Level: developer 4792 4793 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetGeometryFVM()` 4794 @*/ 4795 PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad) 4796 { 4797 DM plex; 4798 PetscBool computeGradients; 4799 4800 PetscFunctionBegin; 4801 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4802 PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2); 4803 PetscAssertPointer(dmGrad, 3); 4804 PetscCall(PetscFVGetComputeGradients(fv, &computeGradients)); 4805 if (!computeGradients) { 4806 *dmGrad = NULL; 4807 PetscFunctionReturn(PETSC_SUCCESS); 4808 } 4809 PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE)); 4810 PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad)); 4811 PetscCall(DMDestroy(&plex)); 4812 PetscFunctionReturn(PETSC_SUCCESS); 4813 } 4814 4815 static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS) 4816 { 4817 DM_Plex *mesh = (DM_Plex *)dm->data; 4818 DM plex = NULL, plexA = NULL; 4819 const char *name = "BdResidual"; 4820 DMEnclosureType encAux; 4821 PetscDS prob, probAux = NULL; 4822 PetscSection section, sectionAux = NULL; 4823 Vec locA = NULL; 4824 PetscScalar *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL; 4825 PetscInt totDim, totDimAux = 0; 4826 4827 PetscFunctionBegin; 4828 PetscCall(DMConvert(dm, DMPLEX, &plex)); 4829 PetscCall(DMGetLocalSection(dm, §ion)); 4830 PetscCall(DMGetDS(dm, &prob)); 4831 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 4832 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA)); 4833 if (locA) { 4834 DM dmAux; 4835 4836 PetscCall(VecGetDM(locA, &dmAux)); 4837 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 4838 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 4839 PetscCall(DMGetDS(plexA, &probAux)); 4840 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 4841 PetscCall(DMGetLocalSection(plexA, §ionAux)); 4842 } 4843 { 4844 PetscFEGeom *fgeom; 4845 PetscInt maxDegree; 4846 PetscQuadrature qGeom = NULL; 4847 IS pointIS; 4848 const PetscInt *points; 4849 PetscInt numFaces, face, Nq; 4850 4851 PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS)); 4852 if (!pointIS) goto end; /* No points with that id on this process */ 4853 { 4854 IS isectIS; 4855 4856 /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */ 4857 PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS)); 4858 PetscCall(ISDestroy(&pointIS)); 4859 pointIS = isectIS; 4860 } 4861 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 4862 PetscCall(ISGetIndices(pointIS, &points)); 4863 PetscCall(PetscMalloc4(numFaces * totDim, &u, (locX_t ? (size_t)numFaces * totDim : 0), &u_t, numFaces * totDim, &elemVec, (locA ? (size_t)numFaces * totDimAux : 0), &a)); 4864 PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree)); 4865 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom)); 4866 if (!qGeom) { 4867 PetscFE fe; 4868 4869 PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe)); 4870 PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom)); 4871 PetscCall(PetscObjectReference((PetscObject)qGeom)); 4872 } 4873 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 4874 PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom)); 4875 for (face = 0; face < numFaces; ++face) { 4876 const PetscInt point = points[face], *support; 4877 PetscScalar *x = NULL; 4878 PetscInt i; 4879 4880 PetscCall(DMPlexGetSupport(dm, point, &support)); 4881 PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x)); 4882 for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i]; 4883 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x)); 4884 if (locX_t) { 4885 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x)); 4886 for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i]; 4887 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x)); 4888 } 4889 if (locA) { 4890 PetscInt subp; 4891 4892 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp)); 4893 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x)); 4894 for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i]; 4895 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x)); 4896 } 4897 } 4898 PetscCall(PetscArrayzero(elemVec, numFaces * totDim)); 4899 { 4900 PetscFE fe; 4901 PetscInt Nb; 4902 PetscFEGeom *chunkGeom = NULL; 4903 /* Conforming batches */ 4904 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 4905 /* Remainder */ 4906 PetscInt Nr, offset; 4907 4908 PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe)); 4909 PetscCall(PetscFEGetDimension(fe, &Nb)); 4910 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4911 /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */ 4912 blockSize = Nb; 4913 batchSize = numBlocks * blockSize; 4914 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4915 numChunks = numFaces / (numBatches * batchSize); 4916 Ne = numChunks * numBatches * batchSize; 4917 Nr = numFaces % (numBatches * batchSize); 4918 offset = numFaces - Nr; 4919 PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom)); 4920 PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec)); 4921 PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom)); 4922 PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom)); 4923 PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, &elemVec[offset * totDim])); 4924 PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom)); 4925 } 4926 for (face = 0; face < numFaces; ++face) { 4927 const PetscInt point = points[face], *support; 4928 4929 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, name, totDim, &elemVec[face * totDim])); 4930 PetscCall(DMPlexGetSupport(plex, point, &support)); 4931 PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face * totDim], ADD_ALL_VALUES)); 4932 } 4933 PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 4934 PetscCall(PetscQuadratureDestroy(&qGeom)); 4935 PetscCall(ISRestoreIndices(pointIS, &points)); 4936 PetscCall(ISDestroy(&pointIS)); 4937 PetscCall(PetscFree4(u, u_t, elemVec, a)); 4938 } 4939 end: 4940 if (mesh->printFEM) { 4941 PetscSection s; 4942 Vec locFbc; 4943 PetscInt pStart, pEnd, maxDof; 4944 PetscScalar *zeroes; 4945 4946 PetscCall(DMGetLocalSection(dm, &s)); 4947 PetscCall(VecDuplicate(locF, &locFbc)); 4948 PetscCall(VecCopy(locF, locFbc)); 4949 PetscCall(PetscSectionGetChart(s, &pStart, &pEnd)); 4950 PetscCall(PetscSectionGetMaxDof(s, &maxDof)); 4951 PetscCall(PetscCalloc1(maxDof, &zeroes)); 4952 for (PetscInt p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, s, p, zeroes, INSERT_BC_VALUES)); 4953 PetscCall(PetscFree(zeroes)); 4954 PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc)); 4955 PetscCall(VecDestroy(&locFbc)); 4956 } 4957 PetscCall(DMDestroy(&plex)); 4958 PetscCall(DMDestroy(&plexA)); 4959 PetscFunctionReturn(PETSC_SUCCESS); 4960 } 4961 4962 PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF) 4963 { 4964 DMField coordField; 4965 DMLabel depthLabel; 4966 IS facetIS; 4967 PetscInt dim; 4968 4969 PetscFunctionBegin; 4970 PetscCall(DMGetDimension(dm, &dim)); 4971 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 4972 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 4973 PetscCall(DMGetCoordinateField(dm, &coordField)); 4974 PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS)); 4975 PetscCall(ISDestroy(&facetIS)); 4976 PetscFunctionReturn(PETSC_SUCCESS); 4977 } 4978 4979 static PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) 4980 { 4981 PetscDS prob; 4982 PetscInt numBd, bd; 4983 DMField coordField = NULL; 4984 IS facetIS = NULL; 4985 DMLabel depthLabel; 4986 PetscInt dim; 4987 4988 PetscFunctionBegin; 4989 PetscCall(DMGetDS(dm, &prob)); 4990 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 4991 PetscCall(DMGetDimension(dm, &dim)); 4992 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 4993 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 4994 for (bd = 0; bd < numBd; ++bd) { 4995 PetscWeakForm wf; 4996 DMBoundaryConditionType type; 4997 DMLabel label; 4998 const PetscInt *values; 4999 PetscInt field, numValues, v; 5000 PetscObject obj; 5001 PetscClassId id; 5002 PetscFormKey key; 5003 5004 PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL)); 5005 if (type & DM_BC_ESSENTIAL) continue; 5006 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 5007 PetscCall(PetscObjectGetClassId(obj, &id)); 5008 if (id != PETSCFE_CLASSID) continue; 5009 if (!facetIS) { 5010 DMLabel depthLabel; 5011 PetscInt dim; 5012 5013 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 5014 PetscCall(DMGetDimension(dm, &dim)); 5015 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 5016 } 5017 PetscCall(DMGetCoordinateField(dm, &coordField)); 5018 for (v = 0; v < numValues; ++v) { 5019 key.label = label; 5020 key.value = values[v]; 5021 key.field = field; 5022 key.part = 0; 5023 PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS)); 5024 } 5025 } 5026 PetscCall(ISDestroy(&facetIS)); 5027 PetscFunctionReturn(PETSC_SUCCESS); 5028 } 5029 5030 PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) 5031 { 5032 DM_Plex *mesh = (DM_Plex *)dm->data; 5033 const char *name = "Residual"; 5034 DM dmAux = NULL; 5035 DM dmGrad = NULL; 5036 DMLabel ghostLabel = NULL; 5037 PetscDS ds = NULL; 5038 PetscDS dsAux = NULL; 5039 PetscSection section = NULL; 5040 PetscBool useFEM = PETSC_FALSE; 5041 PetscBool useFVM = PETSC_FALSE; 5042 PetscBool isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE; 5043 PetscFV fvm = NULL; 5044 DMField coordField = NULL; 5045 Vec locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL; 5046 PetscScalar *u = NULL, *u_t, *a, *uL, *uR; 5047 IS chunkIS; 5048 const PetscInt *cells; 5049 PetscInt cStart, cEnd, numCells; 5050 PetscInt Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd; 5051 PetscInt maxDegree = PETSC_INT_MAX; 5052 PetscQuadrature affineQuad = NULL, *quads = NULL; 5053 PetscFEGeom *affineGeom = NULL, **geoms = NULL; 5054 5055 PetscFunctionBegin; 5056 PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5057 if (!cellIS) goto end; 5058 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 5059 if (cStart >= cEnd) goto end; 5060 /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */ 5061 /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */ 5062 /* FEM+FVM */ 5063 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 5064 /* 1: Get sizes from dm and dmAux */ 5065 PetscCall(DMGetLocalSection(dm, §ion)); 5066 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 5067 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, NULL)); 5068 PetscCall(PetscDSGetNumFields(ds, &Nf)); 5069 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 5070 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA)); 5071 if (locA) { 5072 PetscInt subcell; 5073 PetscCall(VecGetDM(locA, &dmAux)); 5074 PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cells ? cells[cStart] : cStart, &subcell)); 5075 PetscCall(DMGetCellDS(dmAux, subcell, &dsAux, NULL)); 5076 PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux)); 5077 } 5078 /* 2: Get geometric data */ 5079 for (f = 0; f < Nf; ++f) { 5080 PetscObject obj; 5081 PetscClassId id; 5082 PetscBool fimp; 5083 5084 PetscCall(PetscDSGetImplicit(ds, f, &fimp)); 5085 if (isImplicit != fimp) continue; 5086 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 5087 PetscCall(PetscObjectGetClassId(obj, &id)); 5088 if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE; 5089 if (id == PETSCFV_CLASSID) { 5090 useFVM = PETSC_TRUE; 5091 fvm = (PetscFV)obj; 5092 } 5093 } 5094 if (useFEM) { 5095 PetscCall(DMGetCoordinateField(dm, &coordField)); 5096 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 5097 if (maxDegree <= 1) { 5098 PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad)); 5099 if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FEGEOM_BASIC, &affineGeom)); 5100 } else { 5101 PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms)); 5102 for (f = 0; f < Nf; ++f) { 5103 PetscObject obj; 5104 PetscClassId id; 5105 PetscBool fimp; 5106 5107 PetscCall(PetscDSGetImplicit(ds, f, &fimp)); 5108 if (isImplicit != fimp) continue; 5109 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 5110 PetscCall(PetscObjectGetClassId(obj, &id)); 5111 if (id == PETSCFE_CLASSID) { 5112 PetscFE fe = (PetscFE)obj; 5113 5114 PetscCall(PetscFEGetQuadrature(fe, &quads[f])); 5115 PetscCall(PetscObjectReference((PetscObject)quads[f])); 5116 PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FEGEOM_BASIC, &geoms[f])); 5117 } 5118 } 5119 } 5120 } 5121 // Handle non-essential (e.g. outflow) boundary values 5122 if (useFVM) { 5123 PetscCall(DMPlexInsertBoundaryValuesFVM(dm, fvm, locX, time, &locGrad)); 5124 PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL)); 5125 PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad)); 5126 } 5127 /* Loop over chunks */ 5128 if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS)); 5129 numCells = cEnd - cStart; 5130 numChunks = 1; 5131 cellChunkSize = numCells / numChunks; 5132 faceChunkSize = (fEnd - fStart) / numChunks; 5133 numChunks = PetscMin(1, numCells); 5134 for (chunk = 0; chunk < numChunks; ++chunk) { 5135 PetscScalar *elemVec, *fluxL, *fluxR; 5136 PetscReal *vol; 5137 PetscFVFaceGeom *fgeom; 5138 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 5139 PetscInt fS = fStart + chunk * faceChunkSize, fE = PetscMin(fS + faceChunkSize, fEnd), numFaces = 0, face; 5140 5141 /* Extract field coefficients */ 5142 if (useFEM) { 5143 PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells)); 5144 PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 5145 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 5146 PetscCall(PetscArrayzero(elemVec, numCells * totDim)); 5147 } 5148 if (useFVM) { 5149 PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR)); 5150 PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol)); 5151 PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL)); 5152 PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR)); 5153 PetscCall(PetscArrayzero(fluxL, numFaces * totDim)); 5154 PetscCall(PetscArrayzero(fluxR, numFaces * totDim)); 5155 } 5156 /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */ 5157 /* Loop over fields */ 5158 for (f = 0; f < Nf; ++f) { 5159 PetscObject obj; 5160 PetscClassId id; 5161 PetscBool fimp; 5162 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset; 5163 5164 key.field = f; 5165 PetscCall(PetscDSGetImplicit(ds, f, &fimp)); 5166 if (isImplicit != fimp) continue; 5167 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 5168 PetscCall(PetscObjectGetClassId(obj, &id)); 5169 if (id == PETSCFE_CLASSID) { 5170 PetscFE fe = (PetscFE)obj; 5171 PetscFEGeom *geom = affineGeom ? affineGeom : geoms[f]; 5172 PetscFEGeom *chunkGeom = NULL; 5173 PetscQuadrature quad = affineQuad ? affineQuad : quads[f]; 5174 PetscInt Nq, Nb; 5175 5176 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 5177 PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL)); 5178 PetscCall(PetscFEGetDimension(fe, &Nb)); 5179 blockSize = Nb; 5180 batchSize = numBlocks * blockSize; 5181 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 5182 numChunks = numCells / (numBatches * batchSize); 5183 Ne = numChunks * numBatches * batchSize; 5184 Nr = numCells % (numBatches * batchSize); 5185 offset = numCells - Nr; 5186 /* Integrate FE residual to get elemVec (need fields at quadrature points) */ 5187 /* For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */ 5188 PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom)); 5189 PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec)); 5190 PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom)); 5191 PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, &elemVec[offset * totDim])); 5192 PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom)); 5193 } else if (id == PETSCFV_CLASSID) { 5194 PetscFV fv = (PetscFV)obj; 5195 5196 Ne = numFaces; 5197 /* Riemann solve over faces (need fields at face centroids) */ 5198 /* We need to evaluate FE fields at those coordinates */ 5199 PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR)); 5200 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 5201 } 5202 /* Loop over domain */ 5203 if (useFEM) { 5204 /* Add elemVec to locX */ 5205 for (c = cS; c < cE; ++c) { 5206 const PetscInt cell = cells ? cells[c] : c; 5207 const PetscInt cind = c - cStart; 5208 5209 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim])); 5210 if (ghostLabel) { 5211 PetscInt ghostVal; 5212 5213 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 5214 if (ghostVal > 0) continue; 5215 } 5216 PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES)); 5217 } 5218 } 5219 if (useFVM) { 5220 PetscScalar *fa; 5221 PetscInt iface; 5222 5223 PetscCall(VecGetArray(locF, &fa)); 5224 for (f = 0; f < Nf; ++f) { 5225 PetscFV fv; 5226 PetscObject obj; 5227 PetscClassId id; 5228 PetscInt cdim, foff, pdim; 5229 5230 PetscCall(DMGetCoordinateDim(dm, &cdim)); 5231 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 5232 PetscCall(PetscDSGetFieldOffset(ds, f, &foff)); 5233 PetscCall(PetscObjectGetClassId(obj, &id)); 5234 if (id != PETSCFV_CLASSID) continue; 5235 fv = (PetscFV)obj; 5236 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 5237 /* Accumulate fluxes to cells */ 5238 for (face = fS, iface = 0; face < fE; ++face) { 5239 const PetscInt *scells; 5240 PetscScalar *fL = NULL, *fR = NULL; 5241 PetscInt ghost, d, nsupp, nchild; 5242 5243 PetscCall(DMLabelGetValue(ghostLabel, face, &ghost)); 5244 PetscCall(DMPlexGetSupportSize(dm, face, &nsupp)); 5245 PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL)); 5246 if (ghost >= 0 || nsupp > 2 || nchild > 0) continue; 5247 PetscCall(DMPlexGetSupport(dm, face, &scells)); 5248 PetscCall(DMLabelGetValue(ghostLabel, scells[0], &ghost)); 5249 if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL)); 5250 PetscCall(DMLabelGetValue(ghostLabel, scells[1], &ghost)); 5251 if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR)); 5252 if (mesh->printFVM > 1) { 5253 PetscCall(DMPrintCellVectorReal(face, "Residual: normal", cdim, fgeom[iface].normal)); 5254 PetscCall(DMPrintCellVector(face, "Residual: left state", pdim, &uL[iface * totDim + foff])); 5255 PetscCall(DMPrintCellVector(face, "Residual: right state", pdim, &uR[iface * totDim + foff])); 5256 PetscCall(DMPrintCellVector(face, "Residual: left flux", pdim, &fluxL[iface * totDim + foff])); 5257 PetscCall(DMPrintCellVector(face, "Residual: right flux", pdim, &fluxR[iface * totDim + foff])); 5258 } 5259 for (d = 0; d < pdim; ++d) { 5260 if (fL) fL[d] -= fluxL[iface * totDim + foff + d]; 5261 if (fR) fR[d] += fluxR[iface * totDim + foff + d]; 5262 } 5263 ++iface; 5264 } 5265 } 5266 PetscCall(VecRestoreArray(locF, &fa)); 5267 } 5268 /* Handle time derivative */ 5269 if (locX_t) { 5270 PetscScalar *x_t, *fa; 5271 5272 PetscCall(VecGetArray(locF, &fa)); 5273 PetscCall(VecGetArray(locX_t, &x_t)); 5274 for (f = 0; f < Nf; ++f) { 5275 PetscFV fv; 5276 PetscObject obj; 5277 PetscClassId id; 5278 PetscInt pdim, d; 5279 5280 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 5281 PetscCall(PetscObjectGetClassId(obj, &id)); 5282 if (id != PETSCFV_CLASSID) continue; 5283 fv = (PetscFV)obj; 5284 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 5285 for (c = cS; c < cE; ++c) { 5286 const PetscInt cell = cells ? cells[c] : c; 5287 PetscScalar *u_t, *r; 5288 5289 if (ghostLabel) { 5290 PetscInt ghostVal; 5291 5292 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 5293 if (ghostVal > 0) continue; 5294 } 5295 PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t)); 5296 PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r)); 5297 for (d = 0; d < pdim; ++d) r[d] += u_t[d]; 5298 } 5299 } 5300 PetscCall(VecRestoreArray(locX_t, &x_t)); 5301 PetscCall(VecRestoreArray(locF, &fa)); 5302 } 5303 if (useFEM) { 5304 PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 5305 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 5306 } 5307 if (useFVM) { 5308 PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR)); 5309 PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol)); 5310 PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL)); 5311 PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR)); 5312 if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad)); 5313 } 5314 } 5315 if (useFEM) PetscCall(ISDestroy(&chunkIS)); 5316 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 5317 5318 if (useFEM) { 5319 PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user)); 5320 5321 if (maxDegree <= 1) { 5322 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 5323 PetscCall(PetscQuadratureDestroy(&affineQuad)); 5324 } else { 5325 for (f = 0; f < Nf; ++f) { 5326 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 5327 PetscCall(PetscQuadratureDestroy(&quads[f])); 5328 } 5329 PetscCall(PetscFree2(quads, geoms)); 5330 } 5331 } 5332 5333 /* FEM */ 5334 /* 1: Get sizes from dm and dmAux */ 5335 /* 2: Get geometric data */ 5336 /* 3: Handle boundary values */ 5337 /* 4: Loop over domain */ 5338 /* Extract coefficients */ 5339 /* Loop over fields */ 5340 /* Set tiling for FE*/ 5341 /* Integrate FE residual to get elemVec */ 5342 /* Loop over subdomain */ 5343 /* Loop over quad points */ 5344 /* Transform coords to real space */ 5345 /* Evaluate field and aux fields at point */ 5346 /* Evaluate residual at point */ 5347 /* Transform residual to real space */ 5348 /* Add residual to elemVec */ 5349 /* Loop over domain */ 5350 /* Add elemVec to locX */ 5351 5352 /* FVM */ 5353 /* Get geometric data */ 5354 /* If using gradients */ 5355 /* Compute gradient data */ 5356 /* Loop over domain faces */ 5357 /* Count computational faces */ 5358 /* Reconstruct cell gradient */ 5359 /* Loop over domain cells */ 5360 /* Limit cell gradients */ 5361 /* Handle boundary values */ 5362 /* Loop over domain faces */ 5363 /* Read out field, centroid, normal, volume for each side of face */ 5364 /* Riemann solve over faces */ 5365 /* Loop over domain faces */ 5366 /* Accumulate fluxes to cells */ 5367 /* TODO Change printFEM to printDisc here */ 5368 if (mesh->printFEM) { 5369 Vec locFbc; 5370 PetscInt pStart, pEnd, p, maxDof; 5371 PetscScalar *zeroes; 5372 5373 PetscCall(VecDuplicate(locF, &locFbc)); 5374 PetscCall(VecCopy(locF, locFbc)); 5375 PetscCall(PetscSectionGetChart(section, &pStart, &pEnd)); 5376 PetscCall(PetscSectionGetMaxDof(section, &maxDof)); 5377 PetscCall(PetscCalloc1(maxDof, &zeroes)); 5378 for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES)); 5379 PetscCall(PetscFree(zeroes)); 5380 PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc)); 5381 PetscCall(VecDestroy(&locFbc)); 5382 } 5383 end: 5384 PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5385 PetscFunctionReturn(PETSC_SUCCESS); 5386 } 5387 5388 /* 5389 1) Allow multiple kernels for BdResidual for hybrid DS 5390 5391 DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux 5392 5393 DONE 3) Change DMGetCellFields() to get different aux data a[] for each side 5394 - I think I just need to replace a[] with the closure from each face 5395 5396 4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before 5397 */ 5398 PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) 5399 { 5400 DM_Plex *mesh = (DM_Plex *)dm->data; 5401 const char *name = "Hybrid Residual"; 5402 DM dmAux[3] = {NULL, NULL, NULL}; 5403 DMLabel ghostLabel = NULL; 5404 PetscDS ds = NULL; 5405 PetscDS dsIn = NULL; 5406 PetscDS dsAux[3] = {NULL, NULL, NULL}; 5407 Vec locA[3] = {NULL, NULL, NULL}; 5408 DM dmScale[3] = {NULL, NULL, NULL}; 5409 PetscDS dsScale[3] = {NULL, NULL, NULL}; 5410 Vec locS[3] = {NULL, NULL, NULL}; 5411 PetscSection section = NULL; 5412 DMField coordField = NULL; 5413 PetscScalar *a[3] = {NULL, NULL, NULL}; 5414 PetscScalar *s[3] = {NULL, NULL, NULL}; 5415 PetscScalar *u = NULL, *u_t; 5416 PetscScalar *elemVecNeg, *elemVecPos, *elemVecCoh; 5417 IS chunkISF, chunkISN; 5418 const PetscInt *cells; 5419 PetscInt *faces, *neighbors; 5420 PetscInt cStart, cEnd, numCells; 5421 PetscInt Nf, f, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk; 5422 PetscInt maxDegree = PETSC_INT_MAX; 5423 PetscQuadrature affineQuadF = NULL, *quadsF = NULL; 5424 PetscFEGeom *affineGeomF = NULL, **geomsF = NULL; 5425 PetscQuadrature affineQuadN = NULL, *quadsN = NULL; 5426 PetscFEGeom *affineGeomN = NULL, **geomsN = NULL; 5427 5428 PetscFunctionBegin; 5429 PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5430 if (!cellIS) goto end; 5431 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 5432 PetscCall(ISGetLocalSize(cellIS, &numCells)); 5433 if (cStart >= cEnd) goto end; 5434 if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) { 5435 const char *name; 5436 PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name)); 5437 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part); 5438 } 5439 /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */ 5440 /* FEM */ 5441 /* 1: Get sizes from dm and dmAux */ 5442 PetscCall(DMGetLocalSection(dm, §ion)); 5443 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 5444 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn)); 5445 PetscCall(PetscDSGetNumFields(ds, &Nf)); 5446 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 5447 PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn)); 5448 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2])); 5449 if (locA[2]) { 5450 const PetscInt cellStart = cells ? cells[cStart] : cStart; 5451 5452 PetscCall(VecGetDM(locA[2], &dmAux[2])); 5453 PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL)); 5454 PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2])); 5455 { 5456 const PetscInt *cone; 5457 PetscInt c; 5458 5459 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 5460 for (c = 0; c < 2; ++c) { 5461 const PetscInt *support; 5462 PetscInt ssize, s; 5463 5464 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 5465 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 5466 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 5467 if (support[0] == cellStart) s = 1; 5468 else if (support[1] == cellStart) s = 0; 5469 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 5470 PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c])); 5471 PetscCheck(locA[c], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Must have auxiliary vector for (%p, %" PetscInt_FMT ", %" PetscInt_FMT ")", (void *)key[c].label, key[c].value, key[c].part); 5472 if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c])); 5473 else dmAux[c] = dmAux[2]; 5474 PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL)); 5475 PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c])); 5476 } 5477 } 5478 } 5479 /* Handle mass matrix scaling 5480 The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */ 5481 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2])); 5482 if (locS[2]) { 5483 const PetscInt cellStart = cells ? cells[cStart] : cStart; 5484 PetscInt Nb, Nbs; 5485 5486 PetscCall(VecGetDM(locS[2], &dmScale[2])); 5487 PetscCall(DMGetCellDS(dmScale[2], cellStart, &dsScale[2], NULL)); 5488 PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2])); 5489 // BRAD: This is not set correctly 5490 key[2].field = 2; 5491 PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb)); 5492 PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs)); 5493 PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs); 5494 { 5495 const PetscInt *cone; 5496 PetscInt c; 5497 5498 locS[1] = locS[0] = locS[2]; 5499 dmScale[1] = dmScale[0] = dmScale[2]; 5500 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 5501 for (c = 0; c < 2; ++c) { 5502 const PetscInt *support; 5503 PetscInt ssize, s; 5504 5505 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 5506 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 5507 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 5508 if (support[0] == cellStart) s = 1; 5509 else if (support[1] == cellStart) s = 0; 5510 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 5511 PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL)); 5512 PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c])); 5513 } 5514 } 5515 } 5516 /* 2: Setup geometric data */ 5517 PetscCall(DMGetCoordinateField(dm, &coordField)); 5518 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 5519 if (maxDegree > 1) { 5520 PetscCall(PetscCalloc4(Nf, &quadsF, Nf, &geomsF, Nf, &quadsN, Nf, &geomsN)); 5521 for (f = 0; f < Nf; ++f) { 5522 PetscFE fe; 5523 PetscBool isCohesiveField; 5524 5525 PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe)); 5526 if (fe) { 5527 PetscCall(PetscFEGetQuadrature(fe, &quadsF[f])); 5528 PetscCall(PetscObjectReference((PetscObject)quadsF[f])); 5529 } 5530 PetscCall(PetscDSGetDiscretization(dsIn, f, (PetscObject *)&fe)); 5531 PetscCall(PetscDSGetCohesive(dsIn, f, &isCohesiveField)); 5532 if (fe) { 5533 if (isCohesiveField) { 5534 for (PetscInt g = 0; g < Nf; ++g) { 5535 PetscCall(PetscDSGetDiscretization(dsIn, g, (PetscObject *)&fe)); 5536 PetscCall(PetscDSGetCohesive(dsIn, g, &isCohesiveField)); 5537 if (!isCohesiveField) break; 5538 } 5539 } 5540 PetscCall(PetscFEGetQuadrature(fe, &quadsN[f])); 5541 PetscCall(PetscObjectReference((PetscObject)quadsN[f])); 5542 } 5543 } 5544 } 5545 /* Loop over chunks */ 5546 cellChunkSize = numCells; 5547 numChunks = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize); 5548 PetscCall(PetscCalloc2(2 * cellChunkSize, &faces, 2 * cellChunkSize, &neighbors)); 5549 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkISF)); 5550 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER, &chunkISN)); 5551 /* Extract field coefficients */ 5552 /* NOTE This needs the end cap faces to have identical orientations */ 5553 PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 5554 PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 5555 PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s)); 5556 PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecNeg)); 5557 PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecPos)); 5558 PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecCoh)); 5559 for (chunk = 0; chunk < numChunks; ++chunk) { 5560 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 5561 5562 PetscCall(PetscArrayzero(elemVecNeg, cellChunkSize * totDim)); 5563 PetscCall(PetscArrayzero(elemVecPos, cellChunkSize * totDim)); 5564 PetscCall(PetscArrayzero(elemVecCoh, cellChunkSize * totDim)); 5565 /* Get faces and neighbors */ 5566 for (c = cS; c < cE; ++c) { 5567 const PetscInt cell = cells ? cells[c] : c; 5568 const PetscInt *cone, *support; 5569 PetscCall(DMPlexGetCone(dm, cell, &cone)); 5570 faces[(c - cS) * 2 + 0] = cone[0]; 5571 faces[(c - cS) * 2 + 1] = cone[1]; 5572 PetscCall(DMPlexGetSupport(dm, cone[0], &support)); 5573 neighbors[(c - cS) * 2 + 0] = support[0] == cell ? support[1] : support[0]; 5574 PetscCall(DMPlexGetSupport(dm, cone[1], &support)); 5575 neighbors[(c - cS) * 2 + 1] = support[0] == cell ? support[1] : support[0]; 5576 } 5577 PetscCall(ISGeneralSetIndices(chunkISF, 2 * cellChunkSize, faces, PETSC_USE_POINTER)); 5578 PetscCall(ISGeneralSetIndices(chunkISN, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER)); 5579 /* Get geometric data */ 5580 if (maxDegree <= 1) { 5581 if (!affineQuadF) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkISF, &affineQuadF)); 5582 if (affineQuadF) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, affineQuadF, PETSC_FEGEOM_COHESIVE, &affineGeomF)); 5583 if (!affineQuadN) { 5584 PetscInt dim; 5585 PetscCall(PetscQuadratureGetData(affineQuadF, &dim, NULL, NULL, NULL, NULL)); 5586 PetscCall(DMFieldCreateDefaultFaceQuadrature(coordField, chunkISN, &affineQuadN)); 5587 PetscCall(PetscQuadratureSetData(affineQuadN, dim + 1, PETSC_DECIDE, PETSC_DECIDE, NULL, NULL)); 5588 } 5589 if (affineQuadN) PetscCall(DMSNESGetFEGeom(coordField, chunkISN, affineQuadN, PETSC_FEGEOM_BASIC, &affineGeomN)); 5590 } else { 5591 for (f = 0; f < Nf; ++f) { 5592 if (quadsF[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, quadsF[f], PETSC_FEGEOM_COHESIVE, &geomsF[f])); 5593 if (quadsN[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkISN, quadsN[f], PETSC_FEGEOM_BASIC, &geomsN[f])); 5594 } 5595 } 5596 /* Loop over fields */ 5597 for (f = 0; f < Nf; ++f) { 5598 PetscFE fe; 5599 PetscFEGeom *geomF = affineGeomF ? affineGeomF : geomsF[f]; 5600 PetscFEGeom *chunkGeomF = NULL, *remGeomF = NULL; 5601 PetscFEGeom *geomN = affineGeomN ? affineGeomN : geomsN[f]; 5602 PetscFEGeom *chunkGeomN = NULL, *remGeomN = NULL; 5603 PetscQuadrature quadF = affineQuadF ? affineQuadF : quadsF[f]; 5604 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb; 5605 PetscBool isCohesiveField; 5606 5607 PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe)); 5608 if (!fe) continue; 5609 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 5610 PetscCall(PetscQuadratureGetData(quadF, NULL, NULL, &Nq, NULL, NULL)); 5611 PetscCall(PetscFEGetDimension(fe, &Nb)); 5612 blockSize = Nb; 5613 batchSize = numBlocks * blockSize; 5614 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 5615 numChunks = numCells / (numBatches * batchSize); 5616 Ne = numChunks * numBatches * batchSize; 5617 Nr = numCells % (numBatches * batchSize); 5618 offset = numCells - Nr; 5619 PetscCall(PetscFEGeomGetChunk(geomF, 0, offset * 2, &chunkGeomF)); 5620 PetscCall(PetscFEGeomGetChunk(geomF, offset * 2, numCells * 2, &remGeomF)); 5621 PetscCall(PetscFEGeomGetChunk(geomN, 0, offset * 2, &chunkGeomN)); 5622 PetscCall(PetscFEGeomGetChunk(geomN, offset * 2, numCells * 2, &remGeomN)); 5623 PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField)); 5624 // TODO Do I need to set isCohesive on the chunks? 5625 key[0].field = f; 5626 key[1].field = f; 5627 key[2].field = f; 5628 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[0], a[0], t, elemVecNeg)); 5629 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], PetscSafePointerPlusOffset(a[0], offset * totDimAux[0]), t, &elemVecNeg[offset * totDim])); 5630 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[1], a[1], t, elemVecPos)); 5631 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], PetscSafePointerPlusOffset(a[1], offset * totDimAux[1]), t, &elemVecPos[offset * totDim])); 5632 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[2], a[2], t, elemVecCoh)); 5633 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], PetscSafePointerPlusOffset(a[2], offset * totDimAux[2]), t, &elemVecCoh[offset * totDim])); 5634 PetscCall(PetscFEGeomRestoreChunk(geomF, offset, numCells, &remGeomF)); 5635 PetscCall(PetscFEGeomRestoreChunk(geomF, 0, offset, &chunkGeomF)); 5636 PetscCall(PetscFEGeomRestoreChunk(geomN, offset, numCells, &remGeomN)); 5637 PetscCall(PetscFEGeomRestoreChunk(geomN, 0, offset, &chunkGeomN)); 5638 } 5639 /* Add elemVec to locX */ 5640 for (c = cS; c < cE; ++c) { 5641 const PetscInt cell = cells ? cells[c] : c; 5642 const PetscInt cind = c - cStart; 5643 PetscInt i; 5644 5645 /* Scale element values */ 5646 if (locS[0]) { 5647 PetscInt Nb, off = cind * totDim, soff = cind * totDimScale[0]; 5648 PetscBool cohesive; 5649 5650 for (f = 0; f < Nf; ++f) { 5651 PetscCall(PetscDSGetFieldSize(ds, f, &Nb)); 5652 PetscCall(PetscDSGetCohesive(ds, f, &cohesive)); 5653 if (f == key[2].field) { 5654 PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields"); 5655 // No cohesive scaling field is currently input 5656 for (i = 0; i < Nb; ++i) elemVecCoh[off + i] += s[0][soff + i] * elemVecNeg[off + i] + s[1][soff + i] * elemVecPos[off + i]; 5657 off += Nb; 5658 } else { 5659 const PetscInt N = cohesive ? Nb : Nb * 2; 5660 5661 for (i = 0; i < N; ++i) elemVecCoh[off + i] += elemVecNeg[off + i] + elemVecPos[off + i]; 5662 off += N; 5663 } 5664 } 5665 } else { 5666 for (i = cind * totDim; i < (cind + 1) * totDim; ++i) elemVecCoh[i] += elemVecNeg[i] + elemVecPos[i]; 5667 } 5668 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVecCoh[cind * totDim])); 5669 if (ghostLabel) { 5670 PetscInt ghostVal; 5671 5672 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 5673 if (ghostVal > 0) continue; 5674 } 5675 PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVecCoh[cind * totDim], ADD_ALL_VALUES)); 5676 } 5677 } 5678 PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 5679 PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 5680 PetscCall(DMPlexRestoreHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s)); 5681 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecNeg)); 5682 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecPos)); 5683 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecCoh)); 5684 PetscCall(PetscFree2(faces, neighbors)); 5685 PetscCall(ISDestroy(&chunkISF)); 5686 PetscCall(ISDestroy(&chunkISN)); 5687 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 5688 if (maxDegree <= 1) { 5689 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadF, PETSC_FALSE, &affineGeomF)); 5690 PetscCall(PetscQuadratureDestroy(&affineQuadF)); 5691 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadN, PETSC_FALSE, &affineGeomN)); 5692 PetscCall(PetscQuadratureDestroy(&affineQuadN)); 5693 } else { 5694 for (f = 0; f < Nf; ++f) { 5695 if (geomsF) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quadsF[f], PETSC_FALSE, &geomsF[f])); 5696 if (quadsF) PetscCall(PetscQuadratureDestroy(&quadsF[f])); 5697 if (geomsN) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quadsN[f], PETSC_FALSE, &geomsN[f])); 5698 if (quadsN) PetscCall(PetscQuadratureDestroy(&quadsN[f])); 5699 } 5700 PetscCall(PetscFree4(quadsF, geomsF, quadsN, geomsN)); 5701 } 5702 if (mesh->printFEM) { 5703 Vec locFbc; 5704 PetscInt pStart, pEnd, p, maxDof; 5705 PetscScalar *zeroes; 5706 5707 PetscCall(VecDuplicate(locF, &locFbc)); 5708 PetscCall(VecCopy(locF, locFbc)); 5709 PetscCall(PetscSectionGetChart(section, &pStart, &pEnd)); 5710 PetscCall(PetscSectionGetMaxDof(section, &maxDof)); 5711 PetscCall(PetscCalloc1(maxDof, &zeroes)); 5712 for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES)); 5713 PetscCall(PetscFree(zeroes)); 5714 PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc)); 5715 PetscCall(VecDestroy(&locFbc)); 5716 } 5717 end: 5718 PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5719 PetscFunctionReturn(PETSC_SUCCESS); 5720 } 5721 5722 static PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS) 5723 { 5724 DM_Plex *mesh = (DM_Plex *)dm->data; 5725 DM plex = NULL, plexA = NULL, tdm; 5726 DMEnclosureType encAux; 5727 PetscDS ds, dsAux = NULL; 5728 PetscSection section, sectionAux = NULL; 5729 PetscSection globalSection; 5730 Vec locA = NULL, tv; 5731 PetscScalar *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL; 5732 PetscInt v; 5733 PetscInt Nf, totDim, totDimAux = 0; 5734 PetscBool hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, transform; 5735 5736 PetscFunctionBegin; 5737 PetscCall(DMHasBasisTransform(dm, &transform)); 5738 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 5739 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 5740 PetscCall(DMGetLocalSection(dm, §ion)); 5741 PetscCall(DMGetDS(dm, &ds)); 5742 PetscCall(PetscDSGetNumFields(ds, &Nf)); 5743 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 5744 PetscCall(PetscWeakFormHasBdJacobian(wf, &hasJac)); 5745 PetscCall(PetscWeakFormHasBdJacobianPreconditioner(wf, &hasPrec)); 5746 if (!hasJac && !hasPrec) PetscFunctionReturn(PETSC_SUCCESS); 5747 PetscCall(DMConvert(dm, DMPLEX, &plex)); 5748 PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA)); 5749 if (locA) { 5750 DM dmAux; 5751 5752 PetscCall(VecGetDM(locA, &dmAux)); 5753 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 5754 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 5755 PetscCall(DMGetDS(plexA, &dsAux)); 5756 PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux)); 5757 PetscCall(DMGetLocalSection(plexA, §ionAux)); 5758 } 5759 5760 PetscCall(DMGetGlobalSection(dm, &globalSection)); 5761 for (v = 0; v < numValues; ++v) { 5762 PetscFEGeom *fgeom; 5763 PetscInt maxDegree; 5764 PetscQuadrature qGeom = NULL; 5765 IS pointIS; 5766 const PetscInt *points; 5767 PetscFormKey key; 5768 PetscInt numFaces, face, Nq; 5769 5770 key.label = label; 5771 key.value = values[v]; 5772 key.part = 0; 5773 PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS)); 5774 if (!pointIS) continue; /* No points with that id on this process */ 5775 { 5776 IS isectIS; 5777 5778 /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */ 5779 PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS)); 5780 PetscCall(ISDestroy(&pointIS)); 5781 pointIS = isectIS; 5782 } 5783 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 5784 PetscCall(ISGetIndices(pointIS, &points)); 5785 PetscCall(PetscMalloc5(numFaces * totDim, &u, (locX_t ? (size_t)numFaces * totDim : 0), &u_t, (hasJac ? (size_t)numFaces * totDim * totDim : 0), &elemMat, (hasPrec ? (size_t)numFaces * totDim * totDim : 0), &elemMatP, (locA ? (size_t)numFaces * totDimAux : 0), &a)); 5786 PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree)); 5787 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom)); 5788 if (!qGeom) { 5789 PetscFE fe; 5790 5791 PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&fe)); 5792 PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom)); 5793 PetscCall(PetscObjectReference((PetscObject)qGeom)); 5794 } 5795 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 5796 PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_FEGEOM_BOUNDARY, &fgeom)); 5797 for (face = 0; face < numFaces; ++face) { 5798 const PetscInt point = points[face], *support; 5799 PetscScalar *x = NULL; 5800 PetscInt i; 5801 5802 PetscCall(DMPlexGetSupport(dm, point, &support)); 5803 PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x)); 5804 for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i]; 5805 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x)); 5806 if (locX_t) { 5807 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x)); 5808 for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i]; 5809 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x)); 5810 } 5811 if (locA) { 5812 PetscInt subp; 5813 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp)); 5814 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x)); 5815 for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i]; 5816 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x)); 5817 } 5818 } 5819 if (elemMat) PetscCall(PetscArrayzero(elemMat, numFaces * totDim * totDim)); 5820 if (elemMatP) PetscCall(PetscArrayzero(elemMatP, numFaces * totDim * totDim)); 5821 { 5822 PetscFE fe; 5823 PetscInt Nb; 5824 /* Conforming batches */ 5825 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 5826 /* Remainder */ 5827 PetscFEGeom *chunkGeom = NULL; 5828 PetscInt fieldJ, Nr, offset; 5829 5830 PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&fe)); 5831 PetscCall(PetscFEGetDimension(fe, &Nb)); 5832 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 5833 blockSize = Nb; 5834 batchSize = numBlocks * blockSize; 5835 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 5836 numChunks = numFaces / (numBatches * batchSize); 5837 Ne = numChunks * numBatches * batchSize; 5838 Nr = numFaces % (numBatches * batchSize); 5839 offset = numFaces - Nr; 5840 PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom)); 5841 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 5842 key.field = fieldI * Nf + fieldJ; 5843 if (hasJac) PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMat)); 5844 if (hasPrec) PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, dsAux, a, t, X_tShift, elemMatP)); 5845 } 5846 PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom)); 5847 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 5848 key.field = fieldI * Nf + fieldJ; 5849 if (hasJac) 5850 PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim])); 5851 if (hasPrec) 5852 PetscCall(PetscFEIntegrateBdJacobian(ds, wf, PETSCFE_JACOBIAN_PRE, key, Nr, chunkGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), dsAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * totDim])); 5853 } 5854 PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom)); 5855 } 5856 for (face = 0; face < numFaces; ++face) { 5857 const PetscInt point = points[face], *support; 5858 5859 /* Transform to global basis before insertion in Jacobian */ 5860 PetscCall(DMPlexGetSupport(plex, point, &support)); 5861 if (hasJac && transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face * totDim * totDim])); 5862 if (hasPrec && transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMatP[face * totDim * totDim])); 5863 if (hasPrec) { 5864 if (hasJac) { 5865 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim])); 5866 PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, support[0], &elemMat[face * totDim * totDim], ADD_VALUES)); 5867 } 5868 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMatP[face * totDim * totDim])); 5869 PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, JacP, support[0], &elemMatP[face * totDim * totDim], ADD_VALUES)); 5870 } else { 5871 if (hasJac) { 5872 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim])); 5873 PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, support[0], &elemMat[face * totDim * totDim], ADD_VALUES)); 5874 } 5875 } 5876 } 5877 PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 5878 PetscCall(PetscQuadratureDestroy(&qGeom)); 5879 PetscCall(ISRestoreIndices(pointIS, &points)); 5880 PetscCall(ISDestroy(&pointIS)); 5881 PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, a)); 5882 } 5883 if (plex) PetscCall(DMDestroy(&plex)); 5884 if (plexA) PetscCall(DMDestroy(&plexA)); 5885 PetscFunctionReturn(PETSC_SUCCESS); 5886 } 5887 5888 PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP) 5889 { 5890 DMField coordField; 5891 DMLabel depthLabel; 5892 IS facetIS; 5893 PetscInt dim; 5894 5895 PetscFunctionBegin; 5896 PetscCall(DMGetDimension(dm, &dim)); 5897 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 5898 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 5899 PetscCall(DMGetCoordinateField(dm, &coordField)); 5900 PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS)); 5901 PetscCall(ISDestroy(&facetIS)); 5902 PetscFunctionReturn(PETSC_SUCCESS); 5903 } 5904 5905 static PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user) 5906 { 5907 PetscDS prob; 5908 PetscInt dim, numBd, bd; 5909 DMLabel depthLabel; 5910 DMField coordField = NULL; 5911 IS facetIS; 5912 5913 PetscFunctionBegin; 5914 PetscCall(DMGetDS(dm, &prob)); 5915 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 5916 PetscCall(DMGetDimension(dm, &dim)); 5917 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 5918 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 5919 PetscCall(DMGetCoordinateField(dm, &coordField)); 5920 for (bd = 0; bd < numBd; ++bd) { 5921 PetscWeakForm wf; 5922 DMBoundaryConditionType type; 5923 DMLabel label; 5924 const PetscInt *values; 5925 PetscInt fieldI, numValues; 5926 PetscObject obj; 5927 PetscClassId id; 5928 5929 PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL)); 5930 if (type & DM_BC_ESSENTIAL) continue; 5931 PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj)); 5932 PetscCall(PetscObjectGetClassId(obj, &id)); 5933 if (id != PETSCFE_CLASSID) continue; 5934 PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS)); 5935 } 5936 PetscCall(ISDestroy(&facetIS)); 5937 PetscFunctionReturn(PETSC_SUCCESS); 5938 } 5939 5940 PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *user) 5941 { 5942 DM_Plex *mesh = (DM_Plex *)dm->data; 5943 const char *name = "Jacobian"; 5944 DM dmAux = NULL, plex, tdm; 5945 DMEnclosureType encAux; 5946 Vec A, tv; 5947 DMField coordField; 5948 PetscDS prob, probAux = NULL; 5949 PetscSection section, globalSection, sectionAux; 5950 PetscScalar *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL; 5951 const PetscInt *cells; 5952 PetscInt Nf, fieldI, fieldJ; 5953 PetscInt totDim, totDimAux = 0, cStart, cEnd, numCells, c; 5954 PetscBool hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, hasDyn, hasFV = PETSC_FALSE, transform; 5955 5956 PetscFunctionBegin; 5957 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 5958 PetscCall(DMGetLocalSection(dm, §ion)); 5959 PetscCall(DMGetGlobalSection(dm, &globalSection)); 5960 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A)); 5961 if (A) { 5962 PetscCall(VecGetDM(A, &dmAux)); 5963 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 5964 PetscCall(DMConvert(dmAux, DMPLEX, &plex)); 5965 PetscCall(DMGetLocalSection(plex, §ionAux)); 5966 PetscCall(DMGetDS(dmAux, &probAux)); 5967 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 5968 } 5969 PetscCall(DMGetCoordinateField(dm, &coordField)); 5970 if (!cellIS) goto end; 5971 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 5972 PetscCall(ISGetLocalSize(cellIS, &numCells)); 5973 if (cStart >= cEnd) goto end; 5974 PetscCall(DMHasBasisTransform(dm, &transform)); 5975 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 5976 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 5977 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL)); 5978 PetscCall(PetscDSGetNumFields(prob, &Nf)); 5979 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 5980 PetscCall(PetscDSHasJacobian(prob, &hasJac)); 5981 PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec)); 5982 /* user passed in the same matrix, avoid double contributions and 5983 only assemble the Jacobian */ 5984 if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE; 5985 PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn)); 5986 hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE; 5987 PetscCall(PetscMalloc5(numCells * totDim, &u, (X_t ? (size_t)numCells * totDim : 0), &u_t, (hasJac ? (size_t)numCells * totDim * totDim : 0), &elemMat, (hasPrec ? (size_t)numCells * totDim * totDim : 0), &elemMatP, (hasDyn ? (size_t)numCells * totDim * totDim : 0), &elemMatD)); 5988 if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a)); 5989 for (c = cStart; c < cEnd; ++c) { 5990 const PetscInt cell = cells ? cells[c] : c; 5991 const PetscInt cind = c - cStart; 5992 PetscScalar *x = NULL, *x_t = NULL; 5993 PetscInt i; 5994 5995 PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x)); 5996 for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i]; 5997 PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x)); 5998 if (X_t) { 5999 PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t)); 6000 for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i]; 6001 PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t)); 6002 } 6003 if (dmAux) { 6004 PetscInt subcell; 6005 PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell)); 6006 PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x)); 6007 for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i]; 6008 PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x)); 6009 } 6010 } 6011 if (hasJac) PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim)); 6012 if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells * totDim * totDim)); 6013 if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim)); 6014 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6015 PetscClassId id; 6016 PetscFE fe; 6017 PetscQuadrature qGeom = NULL; 6018 PetscInt Nb; 6019 /* Conforming batches */ 6020 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 6021 /* Remainder */ 6022 PetscInt Nr, offset, Nq; 6023 PetscInt maxDegree; 6024 PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL; 6025 6026 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 6027 PetscCall(PetscObjectGetClassId((PetscObject)fe, &id)); 6028 if (id == PETSCFV_CLASSID) { 6029 hasFV = PETSC_TRUE; 6030 continue; 6031 } 6032 PetscCall(PetscFEGetDimension(fe, &Nb)); 6033 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 6034 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 6035 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom)); 6036 if (!qGeom) { 6037 PetscCall(PetscFEGetQuadrature(fe, &qGeom)); 6038 PetscCall(PetscObjectReference((PetscObject)qGeom)); 6039 } 6040 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 6041 PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM)); 6042 blockSize = Nb; 6043 batchSize = numBlocks * blockSize; 6044 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 6045 numChunks = numCells / (numBatches * batchSize); 6046 Ne = numChunks * numBatches * batchSize; 6047 Nr = numCells % (numBatches * batchSize); 6048 offset = numCells - Nr; 6049 PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom)); 6050 PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom)); 6051 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 6052 key.field = fieldI * Nf + fieldJ; 6053 if (hasJac) { 6054 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat)); 6055 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim])); 6056 } 6057 if (hasPrec) { 6058 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP)); 6059 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatP[offset * totDim * totDim])); 6060 } 6061 if (hasDyn) { 6062 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD)); 6063 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMatD[offset * totDim * totDim])); 6064 } 6065 } 6066 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom)); 6067 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom)); 6068 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 6069 PetscCall(PetscQuadratureDestroy(&qGeom)); 6070 } 6071 /* Add contribution from X_t */ 6072 if (hasDyn) { 6073 for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c]; 6074 } 6075 if (hasFV) { 6076 PetscClassId id; 6077 PetscFV fv; 6078 PetscInt offsetI, NcI, NbI = 1, fc, f; 6079 6080 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6081 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv)); 6082 PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI)); 6083 PetscCall(PetscObjectGetClassId((PetscObject)fv, &id)); 6084 if (id != PETSCFV_CLASSID) continue; 6085 /* Put in the weighted identity */ 6086 PetscCall(PetscFVGetNumComponents(fv, &NcI)); 6087 for (c = cStart; c < cEnd; ++c) { 6088 const PetscInt cind = c - cStart; 6089 const PetscInt eOffset = cind * totDim * totDim; 6090 PetscReal vol; 6091 6092 PetscCall(DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL)); 6093 for (fc = 0; fc < NcI; ++fc) { 6094 for (f = 0; f < NbI; ++f) { 6095 const PetscInt i = offsetI + f * NcI + fc; 6096 if (hasPrec) { 6097 if (hasJac) elemMat[eOffset + i * totDim + i] = vol; 6098 elemMatP[eOffset + i * totDim + i] = vol; 6099 } else { 6100 elemMat[eOffset + i * totDim + i] = vol; 6101 } 6102 } 6103 } 6104 } 6105 } 6106 /* No allocated space for FV stuff, so ignore the zero entries */ 6107 PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); 6108 } 6109 /* Insert values into matrix */ 6110 for (c = cStart; c < cEnd; ++c) { 6111 const PetscInt cell = cells ? cells[c] : c; 6112 const PetscInt cind = c - cStart; 6113 6114 /* Transform to global basis before insertion in Jacobian */ 6115 if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind * totDim * totDim])); 6116 if (hasPrec) { 6117 if (hasJac) { 6118 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim])); 6119 PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMat[cind * totDim * totDim], ADD_VALUES)); 6120 } 6121 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind * totDim * totDim])); 6122 PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMatP[cind * totDim * totDim], ADD_VALUES)); 6123 } else { 6124 if (hasJac) { 6125 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim])); 6126 PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMat[cind * totDim * totDim], ADD_VALUES)); 6127 } 6128 } 6129 } 6130 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 6131 if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE)); 6132 PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, elemMatD)); 6133 if (dmAux) PetscCall(PetscFree(a)); 6134 /* Compute boundary integrals */ 6135 PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user)); 6136 /* Assemble matrix */ 6137 end: { 6138 PetscBool assOp = hasJac && hasPrec ? PETSC_TRUE : PETSC_FALSE, gassOp; 6139 6140 if (dmAux) PetscCall(DMDestroy(&plex)); 6141 PetscCallMPI(MPIU_Allreduce(&assOp, &gassOp, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm))); 6142 if (hasJac && hasPrec) { 6143 PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY)); 6144 PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY)); 6145 } 6146 } 6147 PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY)); 6148 PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY)); 6149 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6150 PetscFunctionReturn(PETSC_SUCCESS); 6151 } 6152 6153 PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user) 6154 { 6155 DM_Plex *mesh = (DM_Plex *)dm->data; 6156 const char *name = "Hybrid Jacobian"; 6157 DM dmAux[3] = {NULL, NULL, NULL}; 6158 DMLabel ghostLabel = NULL; 6159 DM plex = NULL; 6160 DM plexA = NULL; 6161 PetscDS ds = NULL; 6162 PetscDS dsIn = NULL; 6163 PetscDS dsAux[3] = {NULL, NULL, NULL}; 6164 Vec locA[3] = {NULL, NULL, NULL}; 6165 DM dmScale[3] = {NULL, NULL, NULL}; 6166 PetscDS dsScale[3] = {NULL, NULL, NULL}; 6167 Vec locS[3] = {NULL, NULL, NULL}; 6168 PetscSection section = NULL; 6169 PetscSection sectionAux[3] = {NULL, NULL, NULL}; 6170 DMField coordField = NULL; 6171 PetscScalar *a[3] = {NULL, NULL, NULL}; 6172 PetscScalar *s[3] = {NULL, NULL, NULL}; 6173 PetscScalar *u = NULL, *u_t; 6174 PetscScalar *elemMatNeg, *elemMatPos, *elemMatCoh; 6175 PetscScalar *elemMatNegP, *elemMatPosP, *elemMatCohP; 6176 PetscSection globalSection; 6177 IS chunkISF, chunkISN; 6178 const PetscInt *cells; 6179 PetscInt *faces, *neighbors; 6180 PetscInt cStart, cEnd, numCells; 6181 PetscInt Nf, fieldI, fieldJ, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk; 6182 PetscInt maxDegree = PETSC_INT_MAX; 6183 PetscQuadrature affineQuadF = NULL, *quadsF = NULL; 6184 PetscFEGeom *affineGeomF = NULL, **geomsF = NULL; 6185 PetscQuadrature affineQuadN = NULL; 6186 PetscFEGeom *affineGeomN = NULL; 6187 PetscBool hasBdJac, hasBdPrec; 6188 6189 PetscFunctionBegin; 6190 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6191 if (!cellIS) goto end; 6192 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 6193 PetscCall(ISGetLocalSize(cellIS, &numCells)); 6194 if (cStart >= cEnd) goto end; 6195 if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) { 6196 const char *name; 6197 PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name)); 6198 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part); 6199 } 6200 PetscCall(DMConvert(dm, DMPLEX, &plex)); 6201 PetscCall(DMGetLocalSection(dm, §ion)); 6202 PetscCall(DMGetGlobalSection(dm, &globalSection)); 6203 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 6204 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn)); 6205 PetscCall(PetscDSGetNumFields(ds, &Nf)); 6206 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 6207 PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn)); 6208 PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac)); 6209 PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec)); 6210 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2])); 6211 if (locA[2]) { 6212 const PetscInt cellStart = cells ? cells[cStart] : cStart; 6213 6214 PetscCall(VecGetDM(locA[2], &dmAux[2])); 6215 PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA)); 6216 PetscCall(DMGetLocalSection(dmAux[2], §ionAux[2])); 6217 PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL)); 6218 PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2])); 6219 { 6220 const PetscInt *cone; 6221 PetscInt c; 6222 6223 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 6224 for (c = 0; c < 2; ++c) { 6225 const PetscInt *support; 6226 PetscInt ssize, s; 6227 6228 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 6229 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 6230 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 6231 if (support[0] == cellStart) s = 1; 6232 else if (support[1] == cellStart) s = 0; 6233 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 6234 PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c])); 6235 if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c])); 6236 else dmAux[c] = dmAux[2]; 6237 PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL)); 6238 PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c])); 6239 } 6240 } 6241 } 6242 /* Handle mass matrix scaling 6243 The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */ 6244 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2])); 6245 if (locS[2]) { 6246 const PetscInt cellStart = cells ? cells[cStart] : cStart; 6247 PetscInt Nb, Nbs; 6248 6249 PetscCall(VecGetDM(locS[2], &dmScale[2])); 6250 PetscCall(DMGetCellDS(dmScale[2], cells ? cells[cStart] : cStart, &dsScale[2], NULL)); 6251 PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2])); 6252 // BRAD: This is not set correctly 6253 key[2].field = 2; 6254 PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb)); 6255 PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs)); 6256 PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs); 6257 { 6258 const PetscInt *cone; 6259 PetscInt c; 6260 6261 locS[1] = locS[0] = locS[2]; 6262 dmScale[1] = dmScale[0] = dmScale[2]; 6263 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 6264 for (c = 0; c < 2; ++c) { 6265 const PetscInt *support; 6266 PetscInt ssize, s; 6267 6268 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 6269 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 6270 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 6271 if (support[0] == cellStart) s = 1; 6272 else if (support[1] == cellStart) s = 0; 6273 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 6274 PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL)); 6275 PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c])); 6276 } 6277 } 6278 } 6279 /* 2: Setup geometric data */ 6280 PetscCall(DMGetCoordinateField(dm, &coordField)); 6281 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 6282 if (maxDegree > 1) { 6283 PetscInt f; 6284 PetscCall(PetscCalloc2(Nf, &quadsF, Nf, &geomsF)); 6285 for (f = 0; f < Nf; ++f) { 6286 PetscFE fe; 6287 6288 PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe)); 6289 if (fe) { 6290 PetscCall(PetscFEGetQuadrature(fe, &quadsF[f])); 6291 PetscCall(PetscObjectReference((PetscObject)quadsF[f])); 6292 } 6293 } 6294 } 6295 /* Loop over chunks */ 6296 cellChunkSize = numCells; 6297 numChunks = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize); 6298 PetscCall(PetscCalloc2(2 * cellChunkSize, &faces, 2 * cellChunkSize, &neighbors)); 6299 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkISF)); 6300 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER, &chunkISN)); 6301 /* Extract field coefficients */ 6302 /* NOTE This needs the end cap faces to have identical orientations */ 6303 PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 6304 PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 6305 PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s)); 6306 PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg)); 6307 PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos)); 6308 PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh)); 6309 PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP)); 6310 PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP)); 6311 PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP)); 6312 for (chunk = 0; chunk < numChunks; ++chunk) { 6313 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 6314 6315 if (hasBdJac) { 6316 PetscCall(PetscArrayzero(elemMatNeg, cellChunkSize * totDim * totDim)); 6317 PetscCall(PetscArrayzero(elemMatPos, cellChunkSize * totDim * totDim)); 6318 PetscCall(PetscArrayzero(elemMatCoh, cellChunkSize * totDim * totDim)); 6319 } 6320 if (hasBdPrec) { 6321 PetscCall(PetscArrayzero(elemMatNegP, cellChunkSize * totDim * totDim)); 6322 PetscCall(PetscArrayzero(elemMatPosP, cellChunkSize * totDim * totDim)); 6323 PetscCall(PetscArrayzero(elemMatCohP, cellChunkSize * totDim * totDim)); 6324 } 6325 /* Get faces */ 6326 for (c = cS; c < cE; ++c) { 6327 const PetscInt cell = cells ? cells[c] : c; 6328 const PetscInt *cone, *support; 6329 PetscCall(DMPlexGetCone(plex, cell, &cone)); 6330 faces[(c - cS) * 2 + 0] = cone[0]; 6331 faces[(c - cS) * 2 + 1] = cone[1]; 6332 PetscCall(DMPlexGetSupport(dm, cone[0], &support)); 6333 neighbors[(c - cS) * 2 + 0] = support[0] == cell ? support[1] : support[0]; 6334 PetscCall(DMPlexGetSupport(dm, cone[1], &support)); 6335 neighbors[(c - cS) * 2 + 1] = support[0] == cell ? support[1] : support[0]; 6336 } 6337 PetscCall(ISGeneralSetIndices(chunkISF, 2 * cellChunkSize, faces, PETSC_USE_POINTER)); 6338 PetscCall(ISGeneralSetIndices(chunkISN, 2 * cellChunkSize, neighbors, PETSC_USE_POINTER)); 6339 if (maxDegree <= 1) { 6340 if (!affineQuadF) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkISF, &affineQuadF)); 6341 if (affineQuadF) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, affineQuadF, PETSC_FEGEOM_COHESIVE, &affineGeomF)); 6342 if (!affineQuadN) { 6343 PetscInt dim; 6344 PetscCall(PetscQuadratureGetData(affineQuadF, &dim, NULL, NULL, NULL, NULL)); 6345 PetscCall(DMFieldCreateDefaultFaceQuadrature(coordField, chunkISN, &affineQuadN)); 6346 PetscCall(PetscQuadratureSetData(affineQuadN, dim + 1, PETSC_DECIDE, PETSC_DECIDE, NULL, NULL)); 6347 } 6348 if (affineQuadN) PetscCall(DMSNESGetFEGeom(coordField, chunkISN, affineQuadN, PETSC_FEGEOM_BASIC, &affineGeomN)); 6349 } else { 6350 PetscInt f; 6351 for (f = 0; f < Nf; ++f) { 6352 if (quadsF[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkISF, quadsF[f], PETSC_FEGEOM_COHESIVE, &geomsF[f])); 6353 } 6354 } 6355 6356 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6357 PetscFE feI; 6358 PetscFEGeom *geomF = affineGeomF ? affineGeomF : geomsF[fieldI]; 6359 PetscFEGeom *chunkGeomF = NULL, *remGeomF = NULL; 6360 PetscFEGeom *geomN = affineGeomN ? affineGeomN : geomsF[fieldI]; 6361 PetscFEGeom *chunkGeomN = NULL, *remGeomN = NULL; 6362 PetscQuadrature quadF = affineQuadF ? affineQuadF : quadsF[fieldI]; 6363 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb; 6364 PetscBool isCohesiveField; 6365 6366 PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&feI)); 6367 if (!feI) continue; 6368 PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches)); 6369 PetscCall(PetscQuadratureGetData(quadF, NULL, NULL, &Nq, NULL, NULL)); 6370 PetscCall(PetscFEGetDimension(feI, &Nb)); 6371 blockSize = Nb; 6372 batchSize = numBlocks * blockSize; 6373 PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches)); 6374 numChunks = numCells / (numBatches * batchSize); 6375 Ne = numChunks * numBatches * batchSize; 6376 Nr = numCells % (numBatches * batchSize); 6377 offset = numCells - Nr; 6378 PetscCall(PetscFEGeomGetChunk(geomF, 0, offset * 2, &chunkGeomF)); 6379 PetscCall(PetscFEGeomGetChunk(geomF, offset * 2, numCells * 2, &remGeomF)); 6380 PetscCall(PetscFEGeomGetChunk(geomN, 0, offset * 2, &chunkGeomN)); 6381 PetscCall(PetscFEGeomGetChunk(geomN, offset * 2, numCells * 2, &remGeomN)); 6382 PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField)); 6383 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 6384 PetscFE feJ; 6385 6386 PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *)&feJ)); 6387 if (!feJ) continue; 6388 key[0].field = fieldI * Nf + fieldJ; 6389 key[1].field = fieldI * Nf + fieldJ; 6390 key[2].field = fieldI * Nf + fieldJ; 6391 if (hasBdJac) { 6392 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNeg)); 6393 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], PetscSafePointerPlusOffset(a[0], offset * totDimAux[0]), t, X_tShift, &elemMatNeg[offset * totDim * totDim])); 6394 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPos)); 6395 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], PetscSafePointerPlusOffset(a[1], offset * totDimAux[1]), t, X_tShift, &elemMatPos[offset * totDim * totDim])); 6396 } 6397 if (hasBdPrec) { 6398 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNegP)); 6399 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMatNegP[offset * totDim * totDim])); 6400 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPosP)); 6401 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMatPosP[offset * totDim * totDim])); 6402 } 6403 if (hasBdJac) { 6404 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCoh)); 6405 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], PetscSafePointerPlusOffset(a[2], offset * totDimAux[2]), t, X_tShift, &elemMatCoh[offset * totDim * totDim])); 6406 } 6407 if (hasBdPrec) { 6408 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeomF, chunkGeomN, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCohP)); 6409 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeomF, remGeomN, &u[offset * totDimIn], PetscSafePointerPlusOffset(u_t, offset * totDimIn), dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMatCohP[offset * totDim * totDim])); 6410 } 6411 } 6412 PetscCall(PetscFEGeomRestoreChunk(geomF, offset, numCells, &remGeomF)); 6413 PetscCall(PetscFEGeomRestoreChunk(geomF, 0, offset, &chunkGeomF)); 6414 PetscCall(PetscFEGeomRestoreChunk(geomN, offset, numCells, &remGeomN)); 6415 PetscCall(PetscFEGeomRestoreChunk(geomN, 0, offset, &chunkGeomN)); 6416 } 6417 /* Insert values into matrix */ 6418 for (c = cS; c < cE; ++c) { 6419 const PetscInt cell = cells ? cells[c] : c; 6420 const PetscInt cind = c - cS, coff = cind * totDim * totDim; 6421 PetscInt i, j; 6422 6423 /* Scale element values */ 6424 if (locS[0]) { 6425 PetscInt Nb, soff = cind * totDimScale[0], off = 0; 6426 PetscBool cohesive; 6427 6428 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6429 PetscCall(PetscDSGetFieldSize(ds, fieldI, &Nb)); 6430 PetscCall(PetscDSGetCohesive(ds, fieldI, &cohesive)); 6431 6432 if (fieldI == key[2].field) { 6433 PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields"); 6434 for (i = 0; i < Nb; ++i) { 6435 for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNeg[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPos[coff + (off + i) * totDim + j]; 6436 if (hasBdPrec) 6437 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNegP[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPosP[coff + (off + i) * totDim + j]; 6438 } 6439 off += Nb; 6440 } else { 6441 const PetscInt N = cohesive ? Nb : Nb * 2; 6442 6443 for (i = 0; i < N; ++i) { 6444 for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += elemMatNeg[coff + (off + i) * totDim + j] + elemMatPos[coff + (off + i) * totDim + j]; 6445 if (hasBdPrec) 6446 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += elemMatNegP[coff + (off + i) * totDim + j] + elemMatPosP[coff + (off + i) * totDim + j]; 6447 } 6448 off += N; 6449 } 6450 } 6451 } else { 6452 for (i = 0; i < totDim * totDim; ++i) elemMatCoh[coff + i] += elemMatNeg[coff + i] + elemMatPos[coff + i]; 6453 if (hasBdPrec) 6454 for (i = 0; i < totDim * totDim; ++i) elemMatCohP[coff + i] += elemMatNegP[coff + i] + elemMatPosP[coff + i]; 6455 } 6456 if (hasBdPrec) { 6457 if (hasBdJac) { 6458 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim])); 6459 PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, Jac, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES)); 6460 } 6461 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCohP[cind * totDim * totDim])); 6462 PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatCohP[cind * totDim * totDim], ADD_VALUES)); 6463 } else if (hasBdJac) { 6464 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim])); 6465 PetscCall(DMPlexMatSetClosure_Internal(plex, section, globalSection, mesh->useMatClPerm, JacP, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES)); 6466 } 6467 } 6468 } 6469 PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 6470 PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 6471 PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg)); 6472 PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos)); 6473 PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh)); 6474 PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP)); 6475 PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP)); 6476 PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP)); 6477 PetscCall(PetscFree2(faces, neighbors)); 6478 PetscCall(ISDestroy(&chunkISF)); 6479 PetscCall(ISDestroy(&chunkISN)); 6480 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 6481 if (maxDegree <= 1) { 6482 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadF, PETSC_FALSE, &affineGeomF)); 6483 PetscCall(PetscQuadratureDestroy(&affineQuadF)); 6484 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuadN, PETSC_FALSE, &affineGeomN)); 6485 PetscCall(PetscQuadratureDestroy(&affineQuadN)); 6486 } else { 6487 PetscInt f; 6488 for (f = 0; f < Nf; ++f) { 6489 if (geomsF) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quadsF[f], PETSC_FALSE, &geomsF[f])); 6490 if (quadsF) PetscCall(PetscQuadratureDestroy(&quadsF[f])); 6491 } 6492 PetscCall(PetscFree2(quadsF, geomsF)); 6493 } 6494 if (dmAux[2]) PetscCall(DMDestroy(&plexA)); 6495 PetscCall(DMDestroy(&plex)); 6496 end: 6497 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6498 PetscFunctionReturn(PETSC_SUCCESS); 6499 } 6500 6501 /* 6502 DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user. 6503 6504 Input Parameters: 6505 + dm - The mesh 6506 . key - The PetscWeakFormKey indicating where integration should happen 6507 . cellIS - The cells to integrate over 6508 . t - The time 6509 . X_tShift - The multiplier for the Jacobian with respect to X_t 6510 . X - Local solution vector 6511 . X_t - Time-derivative of the local solution vector 6512 . Y - Local input vector 6513 - user - the user context 6514 6515 Output Parameter: 6516 . Z - Local output vector 6517 6518 Note: 6519 We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator, 6520 like a GPU, or vectorize on a multicore machine. 6521 */ 6522 PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user) 6523 { 6524 DM_Plex *mesh = (DM_Plex *)dm->data; 6525 const char *name = "Jacobian"; 6526 DM dmAux = NULL, plex, plexAux = NULL; 6527 DMEnclosureType encAux; 6528 Vec A; 6529 DMField coordField; 6530 PetscDS prob, probAux = NULL; 6531 PetscQuadrature quad; 6532 PetscSection section, globalSection, sectionAux; 6533 PetscScalar *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z; 6534 const PetscInt *cells; 6535 PetscInt Nf, fieldI, fieldJ; 6536 PetscInt totDim, totDimAux = 0, cStart, cEnd, numCells, c; 6537 PetscBool hasDyn; 6538 6539 PetscFunctionBegin; 6540 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6541 PetscCall(DMConvert(dm, DMPLEX, &plex)); 6542 PetscCall(ISGetLocalSize(cellIS, &numCells)); 6543 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 6544 PetscCall(DMGetLocalSection(dm, §ion)); 6545 PetscCall(DMGetGlobalSection(dm, &globalSection)); 6546 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL)); 6547 PetscCall(PetscDSGetNumFields(prob, &Nf)); 6548 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 6549 PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn)); 6550 hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE; 6551 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A)); 6552 if (A) { 6553 PetscCall(VecGetDM(A, &dmAux)); 6554 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 6555 PetscCall(DMConvert(dmAux, DMPLEX, &plexAux)); 6556 PetscCall(DMGetLocalSection(plexAux, §ionAux)); 6557 PetscCall(DMGetDS(dmAux, &probAux)); 6558 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 6559 } 6560 PetscCall(VecSet(Z, 0.0)); 6561 PetscCall(PetscMalloc6(numCells * totDim, &u, (X_t ? (size_t)numCells * totDim : 0), &u_t, numCells * totDim * totDim, &elemMat, (hasDyn ? (size_t)numCells * totDim * totDim : 0), &elemMatD, numCells * totDim, &y, totDim, &z)); 6562 if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a)); 6563 PetscCall(DMGetCoordinateField(dm, &coordField)); 6564 for (c = cStart; c < cEnd; ++c) { 6565 const PetscInt cell = cells ? cells[c] : c; 6566 const PetscInt cind = c - cStart; 6567 PetscScalar *x = NULL, *x_t = NULL; 6568 PetscInt i; 6569 6570 PetscCall(DMPlexVecGetClosure(plex, section, X, cell, NULL, &x)); 6571 for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i]; 6572 PetscCall(DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x)); 6573 if (X_t) { 6574 PetscCall(DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t)); 6575 for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i]; 6576 PetscCall(DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t)); 6577 } 6578 if (dmAux) { 6579 PetscInt subcell; 6580 PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell)); 6581 PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x)); 6582 for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i]; 6583 PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x)); 6584 } 6585 PetscCall(DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x)); 6586 for (i = 0; i < totDim; ++i) y[cind * totDim + i] = x[i]; 6587 PetscCall(DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x)); 6588 } 6589 PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim)); 6590 if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim)); 6591 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6592 PetscFE fe; 6593 PetscInt Nb; 6594 /* Conforming batches */ 6595 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 6596 /* Remainder */ 6597 PetscInt Nr, offset, Nq; 6598 PetscQuadrature qGeom = NULL; 6599 PetscInt maxDegree; 6600 PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL; 6601 6602 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 6603 PetscCall(PetscFEGetQuadrature(fe, &quad)); 6604 PetscCall(PetscFEGetDimension(fe, &Nb)); 6605 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 6606 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 6607 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom)); 6608 if (!qGeom) { 6609 PetscCall(PetscFEGetQuadrature(fe, &qGeom)); 6610 PetscCall(PetscObjectReference((PetscObject)qGeom)); 6611 } 6612 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 6613 PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FEGEOM_BASIC, &cgeomFEM)); 6614 blockSize = Nb; 6615 batchSize = numBlocks * blockSize; 6616 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 6617 numChunks = numCells / (numBatches * batchSize); 6618 Ne = numChunks * numBatches * batchSize; 6619 Nr = numCells % (numBatches * batchSize); 6620 offset = numCells - Nr; 6621 PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom)); 6622 PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom)); 6623 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 6624 key.field = fieldI * Nf + fieldJ; 6625 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat)); 6626 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, PetscSafePointerPlusOffset(a, offset * totDimAux), t, X_tShift, &elemMat[offset * totDim * totDim])); 6627 if (hasDyn) { 6628 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD)); 6629 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], PetscSafePointerPlusOffset(u_t, offset * totDim), probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim])); 6630 } 6631 } 6632 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom)); 6633 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom)); 6634 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 6635 PetscCall(PetscQuadratureDestroy(&qGeom)); 6636 } 6637 if (hasDyn) { 6638 for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c]; 6639 } 6640 for (c = cStart; c < cEnd; ++c) { 6641 const PetscInt cell = cells ? cells[c] : c; 6642 const PetscInt cind = c - cStart; 6643 const PetscBLASInt one = 1; 6644 PetscBLASInt M; 6645 const PetscScalar a = 1.0, b = 0.0; 6646 6647 PetscCall(PetscBLASIntCast(totDim, &M)); 6648 PetscCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind * totDim * totDim], &M, &y[cind * totDim], &one, &b, z, &one)); 6649 if (mesh->printFEM > 1) { 6650 PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind * totDim * totDim])); 6651 PetscCall(DMPrintCellVector(c, "Y", totDim, &y[cind * totDim])); 6652 PetscCall(DMPrintCellVector(c, "Z", totDim, z)); 6653 } 6654 PetscCall(DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES)); 6655 } 6656 PetscCall(PetscFree6(u, u_t, elemMat, elemMatD, y, z)); 6657 if (mesh->printFEM) { 6658 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n")); 6659 PetscCall(VecView(Z, NULL)); 6660 } 6661 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 6662 PetscCall(PetscFree(a)); 6663 PetscCall(DMDestroy(&plexAux)); 6664 PetscCall(DMDestroy(&plex)); 6665 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6666 PetscFunctionReturn(PETSC_SUCCESS); 6667 } 6668 6669 static void f0_1(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) 6670 { 6671 f0[0] = u[0]; 6672 } 6673 6674 static void f0_x(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) 6675 { 6676 f0[0] = x[(int)PetscRealPart(constants[0])] * u[0]; 6677 } 6678 6679 static void f0_x2(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) 6680 { 6681 PetscInt d; 6682 6683 f0[0] = 0.0; 6684 for (d = 0; d < dim; ++d) f0[0] += PetscSqr(x[d]) * u[0]; 6685 } 6686 6687 /*@ 6688 DMPlexComputeMoments - Compute the first three moments for a field 6689 6690 Noncollective 6691 6692 Input Parameters: 6693 + dm - the `DMPLEX` 6694 - u - the field 6695 6696 Output Parameter: 6697 . moments - the field moments 6698 6699 Level: intermediate 6700 6701 Note: 6702 The `moments` array should be of length cdim + 2, where cdim is the number of components for the coordinate field. 6703 6704 .seealso: `DM`, `DMPLEX`, `DMSwarmComputeMoments()` 6705 @*/ 6706 PetscErrorCode DMPlexComputeMoments(DM dm, Vec u, PetscReal moments[]) 6707 { 6708 PetscDS ds; 6709 PetscScalar mom, constants[1]; 6710 const PetscScalar *oldConstants; 6711 PetscInt cdim, Nf, field = 0, Ncon; 6712 MPI_Comm comm; 6713 void *user; 6714 6715 PetscFunctionBeginUser; 6716 PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); 6717 PetscCall(DMGetCoordinateDim(dm, &cdim)); 6718 PetscCall(DMGetApplicationContext(dm, &user)); 6719 PetscCall(DMGetDS(dm, &ds)); 6720 PetscCall(PetscDSGetNumFields(ds, &Nf)); 6721 PetscCall(PetscDSGetConstants(ds, &Ncon, &oldConstants)); 6722 PetscCheck(Nf == 1, comm, PETSC_ERR_ARG_WRONG, "We currently only support 1 field, not %" PetscInt_FMT, Nf); 6723 PetscCall(PetscDSSetObjective(ds, field, &f0_1)); 6724 PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user)); 6725 moments[0] = PetscRealPart(mom); 6726 for (PetscInt c = 0; c < cdim; ++c) { 6727 constants[0] = c; 6728 PetscCall(PetscDSSetConstants(ds, 1, constants)); 6729 PetscCall(PetscDSSetObjective(ds, field, &f0_x)); 6730 PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user)); 6731 moments[c + 1] = PetscRealPart(mom); 6732 } 6733 PetscCall(PetscDSSetObjective(ds, field, &f0_x2)); 6734 PetscCall(DMPlexComputeIntegralFEM(dm, u, &mom, user)); 6735 moments[cdim + 1] = PetscRealPart(mom); 6736 PetscCall(PetscDSSetConstants(ds, Ncon, (PetscScalar *)oldConstants)); 6737 PetscFunctionReturn(PETSC_SUCCESS); 6738 } 6739