1 #include <petsc/private/dmpleximpl.h> /*I "petscdmplex.h" I*/ 2 #include <petscsf.h> 3 4 #include <petscblaslapack.h> 5 #include <petsc/private/hashsetij.h> 6 #include <petsc/private/petscfeimpl.h> 7 #include <petsc/private/petscfvimpl.h> 8 9 PetscBool Clementcite = PETSC_FALSE; 10 const char ClementCitation[] = "@article{clement1975approximation,\n" 11 " title = {Approximation by finite element functions using local regularization},\n" 12 " author = {Philippe Cl{\\'e}ment},\n" 13 " journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n" 14 " volume = {9},\n" 15 " number = {R2},\n" 16 " pages = {77--84},\n" 17 " year = {1975}\n}\n"; 18 19 static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy) 20 { 21 PetscBool isPlex; 22 23 PetscFunctionBegin; 24 PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex)); 25 if (isPlex) { 26 *plex = dm; 27 PetscCall(PetscObjectReference((PetscObject)dm)); 28 } else { 29 PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex)); 30 if (!*plex) { 31 PetscCall(DMConvert(dm, DMPLEX, plex)); 32 PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex)); 33 if (copy) { 34 DMSubDomainHookLink link; 35 36 PetscCall(DMCopyAuxiliaryVec(dm, *plex)); 37 /* Run the subdomain hook (this will copy the DMSNES/DMTS) */ 38 for (link = dm->subdomainhook; link; link = link->next) { 39 if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx)); 40 } 41 } 42 } else { 43 PetscCall(PetscObjectReference((PetscObject)*plex)); 44 } 45 } 46 PetscFunctionReturn(PETSC_SUCCESS); 47 } 48 49 static PetscErrorCode PetscContainerUserDestroy_PetscFEGeom(void *ctx) 50 { 51 PetscFEGeom *geom = (PetscFEGeom *)ctx; 52 53 PetscFunctionBegin; 54 PetscCall(PetscFEGeomDestroy(&geom)); 55 PetscFunctionReturn(PETSC_SUCCESS); 56 } 57 58 static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) 59 { 60 char composeStr[33] = {0}; 61 PetscObjectId id; 62 PetscContainer container; 63 64 PetscFunctionBegin; 65 PetscCall(PetscObjectGetId((PetscObject)quad, &id)); 66 PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id)); 67 PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container)); 68 if (container) { 69 PetscCall(PetscContainerGetPointer(container, (void **)geom)); 70 } else { 71 PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom)); 72 PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container)); 73 PetscCall(PetscContainerSetPointer(container, (void *)*geom)); 74 PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom)); 75 PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container)); 76 PetscCall(PetscContainerDestroy(&container)); 77 } 78 PetscFunctionReturn(PETSC_SUCCESS); 79 } 80 81 static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) 82 { 83 PetscFunctionBegin; 84 *geom = NULL; 85 PetscFunctionReturn(PETSC_SUCCESS); 86 } 87 88 /*@ 89 DMPlexGetScale - Get the scale for the specified fundamental unit 90 91 Not Collective 92 93 Input Parameters: 94 + dm - the `DM` 95 - unit - The SI unit 96 97 Output Parameter: 98 . scale - The value used to scale all quantities with this unit 99 100 Level: advanced 101 102 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetScale()`, `PetscUnit` 103 @*/ 104 PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale) 105 { 106 DM_Plex *mesh = (DM_Plex *)dm->data; 107 108 PetscFunctionBegin; 109 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 110 PetscValidRealPointer(scale, 3); 111 *scale = mesh->scale[unit]; 112 PetscFunctionReturn(PETSC_SUCCESS); 113 } 114 115 /*@ 116 DMPlexSetScale - Set the scale for the specified fundamental unit 117 118 Not Collective 119 120 Input Parameters: 121 + dm - the `DM` 122 . unit - The SI unit 123 - scale - The value used to scale all quantities with this unit 124 125 Level: advanced 126 127 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetScale()`, `PetscUnit` 128 @*/ 129 PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale) 130 { 131 DM_Plex *mesh = (DM_Plex *)dm->data; 132 133 PetscFunctionBegin; 134 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 135 mesh->scale[unit] = scale; 136 PetscFunctionReturn(PETSC_SUCCESS); 137 } 138 139 static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx) 140 { 141 const PetscInt eps[3][3][3] = { 142 {{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, 143 {{0, 0, -1}, {0, 0, 0}, {1, 0, 0} }, 144 {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0} } 145 }; 146 PetscInt *ctxInt = (PetscInt *)ctx; 147 PetscInt dim2 = ctxInt[0]; 148 PetscInt d = ctxInt[1]; 149 PetscInt i, j, k = dim > 2 ? d - dim : d; 150 151 PetscFunctionBegin; 152 PetscCheck(dim == dim2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2); 153 for (i = 0; i < dim; i++) mode[i] = 0.; 154 if (d < dim) { 155 mode[d] = 1.; /* Translation along axis d */ 156 } else { 157 for (i = 0; i < dim; i++) { 158 for (j = 0; j < dim; j++) { mode[j] += eps[i][j][k] * X[i]; /* Rotation about axis d */ } 159 } 160 } 161 PetscFunctionReturn(PETSC_SUCCESS); 162 } 163 164 /*@ 165 DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation 166 167 Collective 168 169 Input Parameters: 170 + dm - the `DM` 171 - field - The field number for the rigid body space, or 0 for the default 172 173 Output Parameter: 174 . sp - the null space 175 176 Level: advanced 177 178 Note: 179 This is necessary to provide a suitable coarse space for algebraic multigrid 180 181 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()`, `PCGAMG` 182 @*/ 183 PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp) 184 { 185 PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *); 186 MPI_Comm comm; 187 Vec mode[6]; 188 PetscSection section, globalSection; 189 PetscInt dim, dimEmbed, Nf, n, m, mmin, d, i, j; 190 void **ctxs; 191 192 PetscFunctionBegin; 193 PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); 194 PetscCall(DMGetDimension(dm, &dim)); 195 PetscCall(DMGetCoordinateDim(dm, &dimEmbed)); 196 PetscCall(DMGetNumFields(dm, &Nf)); 197 PetscCheck(!Nf || !(field < 0 || field >= Nf), comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf); 198 if (dim == 1 && Nf < 2) { 199 PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp)); 200 PetscFunctionReturn(PETSC_SUCCESS); 201 } 202 PetscCall(DMGetLocalSection(dm, §ion)); 203 PetscCall(DMGetGlobalSection(dm, &globalSection)); 204 PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n)); 205 PetscCall(PetscCalloc2(Nf, &func, Nf, &ctxs)); 206 m = (dim * (dim + 1)) / 2; 207 PetscCall(VecCreate(comm, &mode[0])); 208 PetscCall(VecSetType(mode[0], dm->vectype)); 209 PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE)); 210 PetscCall(VecSetUp(mode[0])); 211 PetscCall(VecGetSize(mode[0], &n)); 212 mmin = PetscMin(m, n); 213 func[field] = DMPlexProjectRigidBody_Private; 214 for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i])); 215 for (d = 0; d < m; d++) { 216 PetscInt ctx[2]; 217 218 ctxs[field] = (void *)(&ctx[0]); 219 ctx[0] = dimEmbed; 220 ctx[1] = d; 221 PetscCall(DMProjectFunction(dm, 0.0, func, ctxs, INSERT_VALUES, mode[d])); 222 } 223 /* Orthonormalize system */ 224 for (i = 0; i < mmin; ++i) { 225 PetscScalar dots[6]; 226 227 PetscCall(VecNormalize(mode[i], NULL)); 228 PetscCall(VecMDot(mode[i], mmin - i - 1, mode + i + 1, dots + i + 1)); 229 for (j = i + 1; j < mmin; ++j) { 230 dots[j] *= -1.0; 231 PetscCall(VecAXPY(mode[j], dots[j], mode[i])); 232 } 233 } 234 PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp)); 235 for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i])); 236 PetscCall(PetscFree2(func, ctxs)); 237 PetscFunctionReturn(PETSC_SUCCESS); 238 } 239 240 /*@ 241 DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation 242 243 Collective 244 245 Input Parameters: 246 + dm - the `DM` 247 . nb - The number of bodies 248 . label - The `DMLabel` marking each domain 249 . nids - The number of ids per body 250 - ids - An array of the label ids in sequence for each domain 251 252 Output Parameter: 253 . sp - the null space 254 255 Level: advanced 256 257 Note: 258 This is necessary to provide a suitable coarse space for algebraic multigrid 259 260 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `MatNullSpaceCreate()` 261 @*/ 262 PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp) 263 { 264 MPI_Comm comm; 265 PetscSection section, globalSection; 266 Vec *mode; 267 PetscScalar *dots; 268 PetscInt dim, dimEmbed, n, m, b, d, i, j, off; 269 270 PetscFunctionBegin; 271 PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); 272 PetscCall(DMGetDimension(dm, &dim)); 273 PetscCall(DMGetCoordinateDim(dm, &dimEmbed)); 274 PetscCall(DMGetLocalSection(dm, §ion)); 275 PetscCall(DMGetGlobalSection(dm, &globalSection)); 276 PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n)); 277 m = nb * (dim * (dim + 1)) / 2; 278 PetscCall(PetscMalloc2(m, &mode, m, &dots)); 279 PetscCall(VecCreate(comm, &mode[0])); 280 PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE)); 281 PetscCall(VecSetUp(mode[0])); 282 for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i])); 283 for (b = 0, off = 0; b < nb; ++b) { 284 for (d = 0; d < m / nb; ++d) { 285 PetscInt ctx[2]; 286 PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private; 287 void *voidctx = (void *)(&ctx[0]); 288 289 ctx[0] = dimEmbed; 290 ctx[1] = d; 291 PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d])); 292 off += nids[b]; 293 } 294 } 295 /* Orthonormalize system */ 296 for (i = 0; i < m; ++i) { 297 PetscScalar dots[6]; 298 299 PetscCall(VecNormalize(mode[i], NULL)); 300 PetscCall(VecMDot(mode[i], m - i - 1, mode + i + 1, dots + i + 1)); 301 for (j = i + 1; j < m; ++j) { 302 dots[j] *= -1.0; 303 PetscCall(VecAXPY(mode[j], dots[j], mode[i])); 304 } 305 } 306 PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp)); 307 for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i])); 308 PetscCall(PetscFree2(mode, dots)); 309 PetscFunctionReturn(PETSC_SUCCESS); 310 } 311 312 /*@ 313 DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs 314 are computed by associating the basis function with one of the mesh points in its transitively-closed support, and 315 evaluating the dual space basis of that point. A basis function is associated with the point in its 316 transitively-closed support whose mesh height is highest (w.r.t. DAG height), but not greater than the maximum 317 projection height, which is set with this function. By default, the maximum projection height is zero, which means 318 that only mesh cells are used to project basis functions. A height of one, for example, evaluates a cell-interior 319 basis functions using its cells dual space basis, but all other basis functions with the dual space basis of a face. 320 321 Input Parameters: 322 + dm - the `DMPLEX` object 323 - height - the maximum projection height >= 0 324 325 Level: advanced 326 327 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()` 328 @*/ 329 PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height) 330 { 331 DM_Plex *plex = (DM_Plex *)dm->data; 332 333 PetscFunctionBegin; 334 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 335 plex->maxProjectionHeight = height; 336 PetscFunctionReturn(PETSC_SUCCESS); 337 } 338 339 /*@ 340 DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in 341 DMPlexProjectXXXLocal() functions. 342 343 Input Parameter: 344 . dm - the `DMPLEX` object 345 346 Output Parameter: 347 . height - the maximum projection height 348 349 Level: intermediate 350 351 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()` 352 @*/ 353 PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height) 354 { 355 DM_Plex *plex = (DM_Plex *)dm->data; 356 357 PetscFunctionBegin; 358 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 359 *height = plex->maxProjectionHeight; 360 PetscFunctionReturn(PETSC_SUCCESS); 361 } 362 363 typedef struct { 364 PetscReal alpha; /* The first Euler angle, and in 2D the only one */ 365 PetscReal beta; /* The second Euler angle */ 366 PetscReal gamma; /* The third Euler angle */ 367 PetscInt dim; /* The dimension of R */ 368 PetscScalar *R; /* The rotation matrix, transforming a vector in the local basis to the global basis */ 369 PetscScalar *RT; /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */ 370 } RotCtx; 371 372 /* 373 Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that 374 we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows: 375 $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis. 376 $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis. 377 $ The XYZ system rotates a third time about the z axis by gamma. 378 */ 379 static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx) 380 { 381 RotCtx *rc = (RotCtx *)ctx; 382 PetscInt dim = rc->dim; 383 PetscReal c1, s1, c2, s2, c3, s3; 384 385 PetscFunctionBegin; 386 PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT)); 387 switch (dim) { 388 case 2: 389 c1 = PetscCosReal(rc->alpha); 390 s1 = PetscSinReal(rc->alpha); 391 rc->R[0] = c1; 392 rc->R[1] = s1; 393 rc->R[2] = -s1; 394 rc->R[3] = c1; 395 PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim))); 396 DMPlex_Transpose2D_Internal(rc->RT); 397 break; 398 case 3: 399 c1 = PetscCosReal(rc->alpha); 400 s1 = PetscSinReal(rc->alpha); 401 c2 = PetscCosReal(rc->beta); 402 s2 = PetscSinReal(rc->beta); 403 c3 = PetscCosReal(rc->gamma); 404 s3 = PetscSinReal(rc->gamma); 405 rc->R[0] = c1 * c3 - c2 * s1 * s3; 406 rc->R[1] = c3 * s1 + c1 * c2 * s3; 407 rc->R[2] = s2 * s3; 408 rc->R[3] = -c1 * s3 - c2 * c3 * s1; 409 rc->R[4] = c1 * c2 * c3 - s1 * s3; 410 rc->R[5] = c3 * s2; 411 rc->R[6] = s1 * s2; 412 rc->R[7] = -c1 * s2; 413 rc->R[8] = c2; 414 PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim))); 415 DMPlex_Transpose3D_Internal(rc->RT); 416 break; 417 default: 418 SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim); 419 } 420 PetscFunctionReturn(PETSC_SUCCESS); 421 } 422 423 static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx) 424 { 425 RotCtx *rc = (RotCtx *)ctx; 426 427 PetscFunctionBegin; 428 PetscCall(PetscFree2(rc->R, rc->RT)); 429 PetscCall(PetscFree(rc)); 430 PetscFunctionReturn(PETSC_SUCCESS); 431 } 432 433 static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx) 434 { 435 RotCtx *rc = (RotCtx *)ctx; 436 437 PetscFunctionBeginHot; 438 PetscValidPointer(ctx, 5); 439 if (l2g) { 440 *A = rc->R; 441 } else { 442 *A = rc->RT; 443 } 444 PetscFunctionReturn(PETSC_SUCCESS); 445 } 446 447 PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx) 448 { 449 PetscFunctionBegin; 450 #if defined(PETSC_USE_COMPLEX) 451 switch (dim) { 452 case 2: { 453 PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0, 0.0}; 454 455 PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx)); 456 z[0] = PetscRealPart(zt[0]); 457 z[1] = PetscRealPart(zt[1]); 458 } break; 459 case 3: { 460 PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0, 0.0, 0.0}; 461 462 PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx)); 463 z[0] = PetscRealPart(zt[0]); 464 z[1] = PetscRealPart(zt[1]); 465 z[2] = PetscRealPart(zt[2]); 466 } break; 467 } 468 #else 469 PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx)); 470 #endif 471 PetscFunctionReturn(PETSC_SUCCESS); 472 } 473 474 PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx) 475 { 476 const PetscScalar *A; 477 478 PetscFunctionBeginHot; 479 PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx)); 480 switch (dim) { 481 case 2: 482 DMPlex_Mult2D_Internal(A, 1, y, z); 483 break; 484 case 3: 485 DMPlex_Mult3D_Internal(A, 1, y, z); 486 break; 487 } 488 PetscFunctionReturn(PETSC_SUCCESS); 489 } 490 491 static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a) 492 { 493 PetscSection ts; 494 const PetscScalar *ta, *tva; 495 PetscInt dof; 496 497 PetscFunctionBeginHot; 498 PetscCall(DMGetLocalSection(tdm, &ts)); 499 PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof)); 500 PetscCall(VecGetArrayRead(tv, &ta)); 501 PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva)); 502 if (l2g) { 503 switch (dof) { 504 case 4: 505 DMPlex_Mult2D_Internal(tva, 1, a, a); 506 break; 507 case 9: 508 DMPlex_Mult3D_Internal(tva, 1, a, a); 509 break; 510 } 511 } else { 512 switch (dof) { 513 case 4: 514 DMPlex_MultTranspose2D_Internal(tva, 1, a, a); 515 break; 516 case 9: 517 DMPlex_MultTranspose3D_Internal(tva, 1, a, a); 518 break; 519 } 520 } 521 PetscCall(VecRestoreArrayRead(tv, &ta)); 522 PetscFunctionReturn(PETSC_SUCCESS); 523 } 524 525 static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a) 526 { 527 PetscSection s, ts; 528 const PetscScalar *ta, *tvaf, *tvag; 529 PetscInt fdof, gdof, fpdof, gpdof; 530 531 PetscFunctionBeginHot; 532 PetscCall(DMGetLocalSection(dm, &s)); 533 PetscCall(DMGetLocalSection(tdm, &ts)); 534 PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof)); 535 PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof)); 536 PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof)); 537 PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof)); 538 PetscCall(VecGetArrayRead(tv, &ta)); 539 PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf)); 540 PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag)); 541 if (l2g) { 542 switch (fdof) { 543 case 4: 544 DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a); 545 break; 546 case 9: 547 DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a); 548 break; 549 } 550 switch (gdof) { 551 case 4: 552 DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a); 553 break; 554 case 9: 555 DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a); 556 break; 557 } 558 } else { 559 switch (fdof) { 560 case 4: 561 DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a); 562 break; 563 case 9: 564 DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a); 565 break; 566 } 567 switch (gdof) { 568 case 4: 569 DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a); 570 break; 571 case 9: 572 DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a); 573 break; 574 } 575 } 576 PetscCall(VecRestoreArrayRead(tv, &ta)); 577 PetscFunctionReturn(PETSC_SUCCESS); 578 } 579 580 PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a) 581 { 582 PetscSection s; 583 PetscSection clSection; 584 IS clPoints; 585 const PetscInt *clp; 586 PetscInt *points = NULL; 587 PetscInt Nf, f, Np, cp, dof, d = 0; 588 589 PetscFunctionBegin; 590 PetscCall(DMGetLocalSection(dm, &s)); 591 PetscCall(PetscSectionGetNumFields(s, &Nf)); 592 PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp)); 593 for (f = 0; f < Nf; ++f) { 594 for (cp = 0; cp < Np * 2; cp += 2) { 595 PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof)); 596 if (!dof) continue; 597 if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d])); 598 d += dof; 599 } 600 } 601 PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp)); 602 PetscFunctionReturn(PETSC_SUCCESS); 603 } 604 605 PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a) 606 { 607 PetscSection s; 608 PetscSection clSection; 609 IS clPoints; 610 const PetscInt *clp; 611 PetscInt *points = NULL; 612 PetscInt Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0; 613 614 PetscFunctionBegin; 615 PetscCall(DMGetLocalSection(dm, &s)); 616 PetscCall(PetscSectionGetNumFields(s, &Nf)); 617 PetscCall(DMPlexGetCompressedClosure(dm, s, p, 0, &Np, &points, &clSection, &clPoints, &clp)); 618 for (f = 0, r = 0; f < Nf; ++f) { 619 for (cpf = 0; cpf < Np * 2; cpf += 2) { 620 PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof)); 621 for (g = 0, c = 0; g < Nf; ++g) { 622 for (cpg = 0; cpg < Np * 2; cpg += 2) { 623 PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof)); 624 PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r * lda + c])); 625 c += gdof; 626 } 627 } 628 PetscCheck(c == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda); 629 r += fdof; 630 } 631 } 632 PetscCheck(r == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda); 633 PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp)); 634 PetscFunctionReturn(PETSC_SUCCESS); 635 } 636 637 static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g) 638 { 639 DM tdm; 640 Vec tv; 641 PetscSection ts, s; 642 const PetscScalar *ta; 643 PetscScalar *a, *va; 644 PetscInt pStart, pEnd, p, Nf, f; 645 646 PetscFunctionBegin; 647 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 648 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 649 PetscCall(DMGetLocalSection(tdm, &ts)); 650 PetscCall(DMGetLocalSection(dm, &s)); 651 PetscCall(PetscSectionGetChart(s, &pStart, &pEnd)); 652 PetscCall(PetscSectionGetNumFields(s, &Nf)); 653 PetscCall(VecGetArray(lv, &a)); 654 PetscCall(VecGetArrayRead(tv, &ta)); 655 for (p = pStart; p < pEnd; ++p) { 656 for (f = 0; f < Nf; ++f) { 657 PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va)); 658 PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va)); 659 } 660 } 661 PetscCall(VecRestoreArray(lv, &a)); 662 PetscCall(VecRestoreArrayRead(tv, &ta)); 663 PetscFunctionReturn(PETSC_SUCCESS); 664 } 665 666 /*@ 667 DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis 668 669 Input Parameters: 670 + dm - The `DM` 671 - lv - A local vector with values in the global basis 672 673 Output Parameter: 674 . lv - A local vector with values in the local basis 675 676 Level: developer 677 678 Note: 679 This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal. 680 681 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLocalToGlobalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()` 682 @*/ 683 PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv) 684 { 685 PetscFunctionBegin; 686 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 687 PetscValidHeaderSpecific(lv, VEC_CLASSID, 2); 688 PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE)); 689 PetscFunctionReturn(PETSC_SUCCESS); 690 } 691 692 /*@ 693 DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis 694 695 Input Parameters: 696 + dm - The `DM` 697 - lv - A local vector with values in the local basis 698 699 Output Parameter: 700 . lv - A local vector with values in the global basis 701 702 Level: developer 703 704 Note: 705 This method is only intended to be called inside `DMGlobalToLocal()`. It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal. 706 707 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()` 708 @*/ 709 PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv) 710 { 711 PetscFunctionBegin; 712 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 713 PetscValidHeaderSpecific(lv, VEC_CLASSID, 2); 714 PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE)); 715 PetscFunctionReturn(PETSC_SUCCESS); 716 } 717 718 /*@ 719 DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions 720 and global solutions, to a local basis, appropriate for discretization integrals and assembly. 721 722 Input Parameters: 723 + dm - The `DM` 724 . alpha - The first Euler angle, and in 2D the only one 725 . beta - The second Euler angle 726 - gamma - The third Euler angle 727 728 Level: developer 729 730 Note: 731 Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that 732 we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows 733 .vb 734 The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis. 735 The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis. 736 The XYZ system rotates a third time about the z axis by gamma. 737 .ve 738 739 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGlobalToLocalBasis()`, `DMPlexLocalToGlobalBasis()` 740 @*/ 741 PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma) 742 { 743 RotCtx *rc; 744 PetscInt cdim; 745 746 PetscFunctionBegin; 747 PetscCall(DMGetCoordinateDim(dm, &cdim)); 748 PetscCall(PetscMalloc1(1, &rc)); 749 dm->transformCtx = rc; 750 dm->transformSetUp = DMPlexBasisTransformSetUp_Rotation_Internal; 751 dm->transformDestroy = DMPlexBasisTransformDestroy_Rotation_Internal; 752 dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal; 753 rc->dim = cdim; 754 rc->alpha = alpha; 755 rc->beta = beta; 756 rc->gamma = gamma; 757 PetscCall((*dm->transformSetUp)(dm, dm->transformCtx)); 758 PetscCall(DMConstructBasisTransform_Internal(dm)); 759 PetscFunctionReturn(PETSC_SUCCESS); 760 } 761 762 /*@C 763 DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates 764 765 Input Parameters: 766 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 767 . time - The time 768 . field - The field to constrain 769 . Nc - The number of constrained field components, or 0 for all components 770 . comps - An array of constrained component numbers, or `NULL` for all components 771 . label - The `DMLabel` defining constrained points 772 . numids - The number of `DMLabel` ids for constrained points 773 . ids - An array of ids for constrained points 774 . func - A pointwise function giving boundary values 775 - ctx - An optional user context for bcFunc 776 777 Output Parameter: 778 . locX - A local vector to receives the boundary values 779 780 Level: developer 781 782 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabel`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()` 783 @*/ 784 PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX) 785 { 786 PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx); 787 void **ctxs; 788 PetscInt numFields; 789 790 PetscFunctionBegin; 791 PetscCall(DMGetNumFields(dm, &numFields)); 792 PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs)); 793 funcs[field] = func; 794 ctxs[field] = ctx; 795 PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX)); 796 PetscCall(PetscFree2(funcs, ctxs)); 797 PetscFunctionReturn(PETSC_SUCCESS); 798 } 799 800 /*@C 801 DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data 802 803 Input Parameters: 804 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 805 . time - The time 806 . locU - A local vector with the input solution values 807 . field - The field to constrain 808 . Nc - The number of constrained field components, or 0 for all components 809 . comps - An array of constrained component numbers, or `NULL` for all components 810 . label - The `DMLabel` defining constrained points 811 . numids - The number of `DMLabel` ids for constrained points 812 . ids - An array of ids for constrained points 813 . func - A pointwise function giving boundary values 814 - ctx - An optional user context for bcFunc 815 816 Output Parameter: 817 . locX - A local vector to receives the boundary values 818 819 Level: developer 820 821 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()` 822 @*/ 823 PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX) 824 { 825 void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]); 826 void **ctxs; 827 PetscInt numFields; 828 829 PetscFunctionBegin; 830 PetscCall(DMGetNumFields(dm, &numFields)); 831 PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs)); 832 funcs[field] = func; 833 ctxs[field] = ctx; 834 PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX)); 835 PetscCall(PetscFree2(funcs, ctxs)); 836 PetscFunctionReturn(PETSC_SUCCESS); 837 } 838 839 /*@C 840 DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coordinates and boundary field data 841 842 Collective 843 844 Input Parameters: 845 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 846 . time - The time 847 . locU - A local vector with the input solution values 848 . field - The field to constrain 849 . Nc - The number of constrained field components, or 0 for all components 850 . comps - An array of constrained component numbers, or `NULL` for all components 851 . label - The `DMLabel` defining constrained points 852 . numids - The number of `DMLabel` ids for constrained points 853 . ids - An array of ids for constrained points 854 . func - A pointwise function giving boundary values, the calling sequence is given in `DMProjectBdFieldLabelLocal()` 855 - ctx - An optional user context for `func` 856 857 Output Parameter: 858 . locX - A local vector to receive the boundary values 859 860 Level: developer 861 862 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectBdFieldLabelLocal()`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()` 863 @*/ 864 PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX) 865 { 866 void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]); 867 void **ctxs; 868 PetscInt numFields; 869 870 PetscFunctionBegin; 871 PetscCall(DMGetNumFields(dm, &numFields)); 872 PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs)); 873 funcs[field] = func; 874 ctxs[field] = ctx; 875 PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX)); 876 PetscCall(PetscFree2(funcs, ctxs)); 877 PetscFunctionReturn(PETSC_SUCCESS); 878 } 879 880 /*@C 881 DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector 882 883 Input Parameters: 884 + dm - The `DM`, with a `PetscDS` that matches the problem being constrained 885 . time - The time 886 . faceGeometry - A vector with the FVM face geometry information 887 . cellGeometry - A vector with the FVM cell geometry information 888 . Grad - A vector with the FVM cell gradient information 889 . field - The field to constrain 890 . Nc - The number of constrained field components, or 0 for all components 891 . comps - An array of constrained component numbers, or `NULL` for all components 892 . label - The `DMLabel` defining constrained points 893 . numids - The number of `DMLabel` ids for constrained points 894 . ids - An array of ids for constrained points 895 . func - A pointwise function giving boundary values 896 - ctx - An optional user context for bcFunc 897 898 Output Parameter: 899 . locX - A local vector to receives the boundary values 900 901 Level: developer 902 903 Note: 904 This implementation currently ignores the numcomps/comps argument from `DMAddBoundary()` 905 906 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()` 907 @*/ 908 PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *), void *ctx, Vec locX) 909 { 910 PetscDS prob; 911 PetscSF sf; 912 DM dmFace, dmCell, dmGrad; 913 const PetscScalar *facegeom, *cellgeom = NULL, *grad; 914 const PetscInt *leaves; 915 PetscScalar *x, *fx; 916 PetscInt dim, nleaves, loc, fStart, fEnd, pdim, i; 917 PetscErrorCode ierru = PETSC_SUCCESS; 918 919 PetscFunctionBegin; 920 PetscCall(DMGetPointSF(dm, &sf)); 921 PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL)); 922 nleaves = PetscMax(0, nleaves); 923 PetscCall(DMGetDimension(dm, &dim)); 924 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 925 PetscCall(DMGetDS(dm, &prob)); 926 PetscCall(VecGetDM(faceGeometry, &dmFace)); 927 PetscCall(VecGetArrayRead(faceGeometry, &facegeom)); 928 if (cellGeometry) { 929 PetscCall(VecGetDM(cellGeometry, &dmCell)); 930 PetscCall(VecGetArrayRead(cellGeometry, &cellgeom)); 931 } 932 if (Grad) { 933 PetscFV fv; 934 935 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fv)); 936 PetscCall(VecGetDM(Grad, &dmGrad)); 937 PetscCall(VecGetArrayRead(Grad, &grad)); 938 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 939 PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx)); 940 } 941 PetscCall(VecGetArray(locX, &x)); 942 for (i = 0; i < numids; ++i) { 943 IS faceIS; 944 const PetscInt *faces; 945 PetscInt numFaces, f; 946 947 PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS)); 948 if (!faceIS) continue; /* No points with that id on this process */ 949 PetscCall(ISGetLocalSize(faceIS, &numFaces)); 950 PetscCall(ISGetIndices(faceIS, &faces)); 951 for (f = 0; f < numFaces; ++f) { 952 const PetscInt face = faces[f], *cells; 953 PetscFVFaceGeom *fg; 954 955 if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */ 956 PetscCall(PetscFindInt(face, nleaves, (PetscInt *)leaves, &loc)); 957 if (loc >= 0) continue; 958 PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg)); 959 PetscCall(DMPlexGetSupport(dm, face, &cells)); 960 if (Grad) { 961 PetscFVCellGeom *cg; 962 PetscScalar *cx, *cgrad; 963 PetscScalar *xG; 964 PetscReal dx[3]; 965 PetscInt d; 966 967 PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg)); 968 PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx)); 969 PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad)); 970 PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG)); 971 DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx); 972 for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d * dim], dx); 973 PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx)); 974 } else { 975 PetscScalar *xI; 976 PetscScalar *xG; 977 978 PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI)); 979 PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG)); 980 ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx); 981 if (ierru) { 982 PetscCall(ISRestoreIndices(faceIS, &faces)); 983 PetscCall(ISDestroy(&faceIS)); 984 goto cleanup; 985 } 986 } 987 } 988 PetscCall(ISRestoreIndices(faceIS, &faces)); 989 PetscCall(ISDestroy(&faceIS)); 990 } 991 cleanup: 992 PetscCall(VecRestoreArray(locX, &x)); 993 if (Grad) { 994 PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx)); 995 PetscCall(VecRestoreArrayRead(Grad, &grad)); 996 } 997 if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom)); 998 PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom)); 999 PetscCall(ierru); 1000 PetscFunctionReturn(PETSC_SUCCESS); 1001 } 1002 1003 static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx) 1004 { 1005 PetscInt c; 1006 for (c = 0; c < Nc; ++c) u[c] = 0.0; 1007 return PETSC_SUCCESS; 1008 } 1009 1010 PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1011 { 1012 PetscObject isZero; 1013 PetscDS prob; 1014 PetscInt numBd, b; 1015 1016 PetscFunctionBegin; 1017 PetscCall(DMGetDS(dm, &prob)); 1018 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 1019 PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero)); 1020 for (b = 0; b < numBd; ++b) { 1021 PetscWeakForm wf; 1022 DMBoundaryConditionType type; 1023 const char *name; 1024 DMLabel label; 1025 PetscInt field, Nc; 1026 const PetscInt *comps; 1027 PetscObject obj; 1028 PetscClassId id; 1029 void (*bvfunc)(void); 1030 PetscInt numids; 1031 const PetscInt *ids; 1032 void *ctx; 1033 1034 PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx)); 1035 if (insertEssential != (type & DM_BC_ESSENTIAL)) continue; 1036 PetscCall(DMGetField(dm, field, NULL, &obj)); 1037 PetscCall(PetscObjectGetClassId(obj, &id)); 1038 if (id == PETSCFE_CLASSID) { 1039 switch (type) { 1040 /* for FEM, there is no insertion to be done for non-essential boundary conditions */ 1041 case DM_BC_ESSENTIAL: { 1042 PetscSimplePointFunc func = (PetscSimplePointFunc)bvfunc; 1043 1044 if (isZero) func = zero; 1045 PetscCall(DMPlexLabelAddCells(dm, label)); 1046 PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX)); 1047 PetscCall(DMPlexLabelClearCells(dm, label)); 1048 } break; 1049 case DM_BC_ESSENTIAL_FIELD: { 1050 PetscPointFunc func = (PetscPointFunc)bvfunc; 1051 1052 PetscCall(DMPlexLabelAddCells(dm, label)); 1053 PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX)); 1054 PetscCall(DMPlexLabelClearCells(dm, label)); 1055 } break; 1056 default: 1057 break; 1058 } 1059 } else if (id == PETSCFV_CLASSID) { 1060 { 1061 PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *) = (PetscErrorCode(*)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *))bvfunc; 1062 1063 if (!faceGeomFVM) continue; 1064 PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX)); 1065 } 1066 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1067 } 1068 PetscFunctionReturn(PETSC_SUCCESS); 1069 } 1070 1071 PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1072 { 1073 PetscObject isZero; 1074 PetscDS prob; 1075 PetscInt numBd, b; 1076 1077 PetscFunctionBegin; 1078 if (!locX) PetscFunctionReturn(PETSC_SUCCESS); 1079 PetscCall(DMGetDS(dm, &prob)); 1080 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 1081 PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero)); 1082 for (b = 0; b < numBd; ++b) { 1083 PetscWeakForm wf; 1084 DMBoundaryConditionType type; 1085 const char *name; 1086 DMLabel label; 1087 PetscInt field, Nc; 1088 const PetscInt *comps; 1089 PetscObject obj; 1090 PetscClassId id; 1091 PetscInt numids; 1092 const PetscInt *ids; 1093 void (*bvfunc)(void); 1094 void *ctx; 1095 1096 PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx)); 1097 if (insertEssential != (type & DM_BC_ESSENTIAL)) continue; 1098 PetscCall(DMGetField(dm, field, NULL, &obj)); 1099 PetscCall(PetscObjectGetClassId(obj, &id)); 1100 if (id == PETSCFE_CLASSID) { 1101 switch (type) { 1102 /* for FEM, there is no insertion to be done for non-essential boundary conditions */ 1103 case DM_BC_ESSENTIAL: { 1104 PetscSimplePointFunc func_t = (PetscSimplePointFunc)bvfunc; 1105 1106 if (isZero) func_t = zero; 1107 PetscCall(DMPlexLabelAddCells(dm, label)); 1108 PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX)); 1109 PetscCall(DMPlexLabelClearCells(dm, label)); 1110 } break; 1111 case DM_BC_ESSENTIAL_FIELD: { 1112 PetscPointFunc func_t = (PetscPointFunc)bvfunc; 1113 1114 PetscCall(DMPlexLabelAddCells(dm, label)); 1115 PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX)); 1116 PetscCall(DMPlexLabelClearCells(dm, label)); 1117 } break; 1118 default: 1119 break; 1120 } 1121 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1122 } 1123 PetscFunctionReturn(PETSC_SUCCESS); 1124 } 1125 1126 /*@ 1127 DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector 1128 1129 Not Collective 1130 1131 Input Parameters: 1132 + dm - The `DM` 1133 . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions 1134 . time - The time 1135 . faceGeomFVM - Face geometry data for FV discretizations 1136 . cellGeomFVM - Cell geometry data for FV discretizations 1137 - gradFVM - Gradient reconstruction data for FV discretizations 1138 1139 Output Parameter: 1140 . locX - Solution updated with boundary values 1141 1142 Level: intermediate 1143 1144 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()`, `DMAddBoundary()` 1145 @*/ 1146 PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1147 { 1148 PetscFunctionBegin; 1149 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1150 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 1151 if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5); 1152 if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6); 1153 if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7); 1154 PetscTryMethod(dm, "DMPlexInsertBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX, time, faceGeomFVM, cellGeomFVM, gradFVM)); 1155 PetscFunctionReturn(PETSC_SUCCESS); 1156 } 1157 1158 /*@ 1159 DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector 1160 1161 Input Parameters: 1162 + dm - The `DM` 1163 . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions 1164 . time - The time 1165 . faceGeomFVM - Face geometry data for FV discretizations 1166 . cellGeomFVM - Cell geometry data for FV discretizations 1167 - gradFVM - Gradient reconstruction data for FV discretizations 1168 1169 Output Parameter: 1170 . locX_t - Solution updated with boundary values 1171 1172 Level: developer 1173 1174 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunctionLabelLocal()` 1175 @*/ 1176 PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) 1177 { 1178 PetscFunctionBegin; 1179 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 1180 if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3); 1181 if (faceGeomFVM) PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5); 1182 if (cellGeomFVM) PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6); 1183 if (gradFVM) PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7); 1184 PetscTryMethod(dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX_t, time, faceGeomFVM, cellGeomFVM, gradFVM)); 1185 PetscFunctionReturn(PETSC_SUCCESS); 1186 } 1187 1188 PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff) 1189 { 1190 Vec localX; 1191 1192 PetscFunctionBegin; 1193 PetscCall(DMGetLocalVector(dm, &localX)); 1194 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL)); 1195 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1196 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1197 PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff)); 1198 PetscCall(DMRestoreLocalVector(dm, &localX)); 1199 PetscFunctionReturn(PETSC_SUCCESS); 1200 } 1201 1202 /*@C 1203 DMComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h. 1204 1205 Collective 1206 1207 Input Parameters: 1208 + dm - The `DM` 1209 . time - The time 1210 . funcs - The functions to evaluate for each field component 1211 . ctxs - Optional array of contexts to pass to each function, or `NULL`. 1212 - localX - The coefficient vector u_h, a local vector 1213 1214 Output Parameter: 1215 . diff - The diff ||u - u_h||_2 1216 1217 Level: developer 1218 1219 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1220 @*/ 1221 PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff) 1222 { 1223 const PetscInt debug = ((DM_Plex *)dm->data)->printL2; 1224 DM tdm; 1225 Vec tv; 1226 PetscSection section; 1227 PetscQuadrature quad; 1228 PetscFEGeom fegeom; 1229 PetscScalar *funcVal, *interpolant; 1230 PetscReal *coords, *gcoords; 1231 PetscReal localDiff = 0.0; 1232 const PetscReal *quadWeights; 1233 PetscInt dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset; 1234 PetscBool transform; 1235 1236 PetscFunctionBegin; 1237 PetscCall(DMGetDimension(dm, &dim)); 1238 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1239 fegeom.dimEmbed = coordDim; 1240 PetscCall(DMGetLocalSection(dm, §ion)); 1241 PetscCall(PetscSectionGetNumFields(section, &numFields)); 1242 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 1243 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 1244 PetscCall(DMHasBasisTransform(dm, &transform)); 1245 PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!"); 1246 for (field = 0; field < numFields; ++field) { 1247 PetscObject obj; 1248 PetscClassId id; 1249 PetscInt Nc; 1250 1251 PetscCall(DMGetField(dm, field, NULL, &obj)); 1252 PetscCall(PetscObjectGetClassId(obj, &id)); 1253 if (id == PETSCFE_CLASSID) { 1254 PetscFE fe = (PetscFE)obj; 1255 1256 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1257 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1258 } else if (id == PETSCFV_CLASSID) { 1259 PetscFV fv = (PetscFV)obj; 1260 1261 PetscCall(PetscFVGetQuadrature(fv, &quad)); 1262 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 1263 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1264 numComponents += Nc; 1265 } 1266 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights)); 1267 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1268 PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * (Nq + 1), &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ)); 1269 PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight)); 1270 PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd)); 1271 for (c = cStart; c < cEnd; ++c) { 1272 PetscScalar *x = NULL; 1273 PetscReal elemDiff = 0.0; 1274 PetscInt qc = 0; 1275 1276 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1277 PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x)); 1278 1279 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1280 PetscObject obj; 1281 PetscClassId id; 1282 void *const ctx = ctxs ? ctxs[field] : NULL; 1283 PetscInt Nb, Nc, q, fc; 1284 1285 PetscCall(DMGetField(dm, field, NULL, &obj)); 1286 PetscCall(PetscObjectGetClassId(obj, &id)); 1287 if (id == PETSCFE_CLASSID) { 1288 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1289 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1290 } else if (id == PETSCFV_CLASSID) { 1291 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1292 Nb = 1; 1293 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1294 if (debug) { 1295 char title[1024]; 1296 PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field)); 1297 PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset])); 1298 } 1299 for (q = 0; q < Nq; ++q) { 1300 PetscFEGeom qgeom; 1301 PetscErrorCode ierr; 1302 1303 qgeom.dimEmbed = fegeom.dimEmbed; 1304 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1305 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1306 qgeom.detJ = &fegeom.detJ[q]; 1307 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1308 if (transform) { 1309 gcoords = &coords[coordDim * Nq]; 1310 PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx)); 1311 } else { 1312 gcoords = &coords[coordDim * q]; 1313 } 1314 PetscCall(PetscArrayzero(funcVal, Nc)); 1315 ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx); 1316 if (ierr) { 1317 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1318 PetscCall(DMRestoreLocalVector(dm, &localX)); 1319 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1320 } 1321 if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx)); 1322 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant)); 1323 else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant)); 1324 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1325 for (fc = 0; fc < Nc; ++fc) { 1326 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1327 if (debug) 1328 PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim * q] : 0.), (double)(coordDim > 1 ? coords[coordDim * q + 1] : 0.), (double)(coordDim > 2 ? coords[coordDim * q + 2] : 0.), 1329 (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc]))); 1330 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1331 } 1332 } 1333 fieldOffset += Nb; 1334 qc += Nc; 1335 } 1336 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1337 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff)); 1338 localDiff += elemDiff; 1339 } 1340 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1341 PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 1342 *diff = PetscSqrtReal(*diff); 1343 PetscFunctionReturn(PETSC_SUCCESS); 1344 } 1345 1346 PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff) 1347 { 1348 const PetscInt debug = ((DM_Plex *)dm->data)->printL2; 1349 DM tdm; 1350 PetscSection section; 1351 PetscQuadrature quad; 1352 Vec localX, tv; 1353 PetscScalar *funcVal, *interpolant; 1354 const PetscReal *quadWeights; 1355 PetscFEGeom fegeom; 1356 PetscReal *coords, *gcoords; 1357 PetscReal localDiff = 0.0; 1358 PetscInt dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset; 1359 PetscBool transform; 1360 1361 PetscFunctionBegin; 1362 PetscCall(DMGetDimension(dm, &dim)); 1363 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1364 fegeom.dimEmbed = coordDim; 1365 PetscCall(DMGetLocalSection(dm, §ion)); 1366 PetscCall(PetscSectionGetNumFields(section, &numFields)); 1367 PetscCall(DMGetLocalVector(dm, &localX)); 1368 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1369 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1370 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 1371 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 1372 PetscCall(DMHasBasisTransform(dm, &transform)); 1373 for (field = 0; field < numFields; ++field) { 1374 PetscFE fe; 1375 PetscInt Nc; 1376 1377 PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe)); 1378 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1379 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1380 numComponents += Nc; 1381 } 1382 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights)); 1383 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1384 /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */ 1385 PetscCall(PetscMalloc6(numComponents, &funcVal, coordDim * (Nq + 1), &coords, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ, numComponents * coordDim, &interpolant, Nq, &fegeom.detJ)); 1386 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1387 for (c = cStart; c < cEnd; ++c) { 1388 PetscScalar *x = NULL; 1389 PetscReal elemDiff = 0.0; 1390 PetscInt qc = 0; 1391 1392 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1393 PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x)); 1394 1395 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1396 PetscFE fe; 1397 void *const ctx = ctxs ? ctxs[field] : NULL; 1398 PetscInt Nb, Nc, q, fc; 1399 1400 PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe)); 1401 PetscCall(PetscFEGetDimension(fe, &Nb)); 1402 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1403 if (debug) { 1404 char title[1024]; 1405 PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field)); 1406 PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset])); 1407 } 1408 for (q = 0; q < Nq; ++q) { 1409 PetscFEGeom qgeom; 1410 PetscErrorCode ierr; 1411 1412 qgeom.dimEmbed = fegeom.dimEmbed; 1413 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1414 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1415 qgeom.detJ = &fegeom.detJ[q]; 1416 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1417 if (transform) { 1418 gcoords = &coords[coordDim * Nq]; 1419 PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx)); 1420 } else { 1421 gcoords = &coords[coordDim * q]; 1422 } 1423 PetscCall(PetscArrayzero(funcVal, Nc)); 1424 ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx); 1425 if (ierr) { 1426 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1427 PetscCall(DMRestoreLocalVector(dm, &localX)); 1428 PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ)); 1429 } 1430 if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx)); 1431 PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant)); 1432 /* Overwrite with the dot product if the normal is given */ 1433 if (n) { 1434 for (fc = 0; fc < Nc; ++fc) { 1435 PetscScalar sum = 0.0; 1436 PetscInt d; 1437 for (d = 0; d < dim; ++d) sum += interpolant[fc * dim + d] * n[d]; 1438 interpolant[fc] = sum; 1439 } 1440 } 1441 for (fc = 0; fc < Nc; ++fc) { 1442 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1443 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]))); 1444 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1445 } 1446 } 1447 fieldOffset += Nb; 1448 qc += Nc; 1449 } 1450 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1451 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff)); 1452 localDiff += elemDiff; 1453 } 1454 PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ)); 1455 PetscCall(DMRestoreLocalVector(dm, &localX)); 1456 PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 1457 *diff = PetscSqrtReal(*diff); 1458 PetscFunctionReturn(PETSC_SUCCESS); 1459 } 1460 1461 PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff) 1462 { 1463 const PetscInt debug = ((DM_Plex *)dm->data)->printL2; 1464 DM tdm; 1465 DMLabel depthLabel; 1466 PetscSection section; 1467 Vec localX, tv; 1468 PetscReal *localDiff; 1469 PetscInt dim, depth, dE, Nf, f, Nds, s; 1470 PetscBool transform; 1471 1472 PetscFunctionBegin; 1473 PetscCall(DMGetDimension(dm, &dim)); 1474 PetscCall(DMGetCoordinateDim(dm, &dE)); 1475 PetscCall(DMGetLocalSection(dm, §ion)); 1476 PetscCall(DMGetLocalVector(dm, &localX)); 1477 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 1478 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 1479 PetscCall(DMHasBasisTransform(dm, &transform)); 1480 PetscCall(DMGetNumFields(dm, &Nf)); 1481 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 1482 PetscCall(DMLabelGetNumValues(depthLabel, &depth)); 1483 1484 PetscCall(VecSet(localX, 0.0)); 1485 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1486 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1487 PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX)); 1488 PetscCall(DMGetNumDS(dm, &Nds)); 1489 PetscCall(PetscCalloc1(Nf, &localDiff)); 1490 for (s = 0; s < Nds; ++s) { 1491 PetscDS ds; 1492 DMLabel label; 1493 IS fieldIS, pointIS; 1494 const PetscInt *fields, *points = NULL; 1495 PetscQuadrature quad; 1496 const PetscReal *quadPoints, *quadWeights; 1497 PetscFEGeom fegeom; 1498 PetscReal *coords, *gcoords; 1499 PetscScalar *funcVal, *interpolant; 1500 PetscBool isCohesive; 1501 PetscInt qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf; 1502 1503 PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds, NULL)); 1504 PetscCall(ISGetIndices(fieldIS, &fields)); 1505 PetscCall(PetscDSIsCohesive(ds, &isCohesive)); 1506 PetscCall(PetscDSGetNumFields(ds, &dsNf)); 1507 PetscCall(PetscDSGetTotalComponents(ds, &totNc)); 1508 PetscCall(PetscDSGetQuadrature(ds, &quad)); 1509 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 1510 PetscCheck(!(qNc != 1) || !(qNc != totNc), PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc); 1511 PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE * (Nq + 1), &coords, Nq, &fegeom.detJ, dE * dE * Nq, &fegeom.J, dE * dE * Nq, &fegeom.invJ)); 1512 if (!label) { 1513 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1514 } else { 1515 PetscCall(DMLabelGetStratumIS(label, 1, &pointIS)); 1516 PetscCall(ISGetLocalSize(pointIS, &cEnd)); 1517 PetscCall(ISGetIndices(pointIS, &points)); 1518 } 1519 for (c = cStart; c < cEnd; ++c) { 1520 const PetscInt cell = points ? points[c] : c; 1521 PetscScalar *x = NULL; 1522 const PetscInt *cone; 1523 PetscInt qc = 0, fOff = 0, dep; 1524 1525 PetscCall(DMLabelGetValue(depthLabel, cell, &dep)); 1526 if (dep != depth - 1) continue; 1527 if (isCohesive) { 1528 PetscCall(DMPlexGetCone(dm, cell, &cone)); 1529 PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1530 } else { 1531 PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1532 } 1533 PetscCall(DMPlexVecGetClosure(dm, NULL, localX, cell, NULL, &x)); 1534 for (f = 0; f < dsNf; ++f) { 1535 PetscObject obj; 1536 PetscClassId id; 1537 void *const ctx = ctxs ? ctxs[fields[f]] : NULL; 1538 PetscInt Nb, Nc, q, fc; 1539 PetscReal elemDiff = 0.0; 1540 PetscBool cohesive; 1541 1542 PetscCall(PetscDSGetCohesive(ds, f, &cohesive)); 1543 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 1544 PetscCall(PetscObjectGetClassId(obj, &id)); 1545 if (id == PETSCFE_CLASSID) { 1546 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1547 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1548 } else if (id == PETSCFV_CLASSID) { 1549 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1550 Nb = 1; 1551 } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]); 1552 if (isCohesive && !cohesive) { 1553 fOff += Nb * 2; 1554 qc += Nc; 1555 continue; 1556 } 1557 if (debug) { 1558 char title[1024]; 1559 PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f])); 1560 PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff])); 1561 } 1562 for (q = 0; q < Nq; ++q) { 1563 PetscFEGeom qgeom; 1564 PetscErrorCode ierr; 1565 1566 qgeom.dimEmbed = fegeom.dimEmbed; 1567 qgeom.J = &fegeom.J[q * dE * dE]; 1568 qgeom.invJ = &fegeom.invJ[q * dE * dE]; 1569 qgeom.detJ = &fegeom.detJ[q]; 1570 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q); 1571 if (transform) { 1572 gcoords = &coords[dE * Nq]; 1573 PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE * q], PETSC_TRUE, dE, &coords[dE * q], gcoords, dm->transformCtx)); 1574 } else { 1575 gcoords = &coords[dE * q]; 1576 } 1577 for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.; 1578 ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx); 1579 if (ierr) { 1580 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x)); 1581 PetscCall(DMRestoreLocalVector(dm, &localX)); 1582 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1583 } 1584 if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx)); 1585 /* Call once for each face, except for lagrange field */ 1586 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fOff], &qgeom, q, interpolant)); 1587 else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fOff], q, interpolant)); 1588 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]); 1589 for (fc = 0; fc < Nc; ++fc) { 1590 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1591 if (debug) 1592 PetscCall(PetscPrintf(PETSC_COMM_SELF, " cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE * q] : 0.), (double)(dE > 1 ? coords[dE * q + 1] : 0.), (double)(dE > 2 ? coords[dE * q + 2] : 0.), 1593 (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]))); 1594 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1595 } 1596 } 1597 fOff += Nb; 1598 qc += Nc; 1599 localDiff[fields[f]] += elemDiff; 1600 if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, " cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]])); 1601 } 1602 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x)); 1603 } 1604 if (label) { 1605 PetscCall(ISRestoreIndices(pointIS, &points)); 1606 PetscCall(ISDestroy(&pointIS)); 1607 } 1608 PetscCall(ISRestoreIndices(fieldIS, &fields)); 1609 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1610 } 1611 PetscCall(DMRestoreLocalVector(dm, &localX)); 1612 PetscCall(MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 1613 PetscCall(PetscFree(localDiff)); 1614 for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]); 1615 PetscFunctionReturn(PETSC_SUCCESS); 1616 } 1617 1618 /*@C 1619 DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec. 1620 1621 Collective 1622 1623 Input Parameters: 1624 + dm - The `DM` 1625 . time - The time 1626 . funcs - The functions to evaluate for each field component: `NULL` means that component does not contribute to error calculation 1627 . ctxs - Optional array of contexts to pass to each function, or `NULL`. 1628 - X - The coefficient vector u_h 1629 1630 Output Parameter: 1631 . D - A `Vec` which holds the difference ||u - u_h||_2 for each cell 1632 1633 Level: developer 1634 1635 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1636 @*/ 1637 PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D) 1638 { 1639 PetscSection section; 1640 PetscQuadrature quad; 1641 Vec localX; 1642 PetscFEGeom fegeom; 1643 PetscScalar *funcVal, *interpolant; 1644 PetscReal *coords; 1645 const PetscReal *quadPoints, *quadWeights; 1646 PetscInt dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset; 1647 1648 PetscFunctionBegin; 1649 PetscCall(VecSet(D, 0.0)); 1650 PetscCall(DMGetDimension(dm, &dim)); 1651 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1652 PetscCall(DMGetLocalSection(dm, §ion)); 1653 PetscCall(PetscSectionGetNumFields(section, &numFields)); 1654 PetscCall(DMGetLocalVector(dm, &localX)); 1655 PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX)); 1656 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX)); 1657 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX)); 1658 for (field = 0; field < numFields; ++field) { 1659 PetscObject obj; 1660 PetscClassId id; 1661 PetscInt Nc; 1662 1663 PetscCall(DMGetField(dm, field, NULL, &obj)); 1664 PetscCall(PetscObjectGetClassId(obj, &id)); 1665 if (id == PETSCFE_CLASSID) { 1666 PetscFE fe = (PetscFE)obj; 1667 1668 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1669 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1670 } else if (id == PETSCFV_CLASSID) { 1671 PetscFV fv = (PetscFV)obj; 1672 1673 PetscCall(PetscFVGetQuadrature(fv, &quad)); 1674 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 1675 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1676 numComponents += Nc; 1677 } 1678 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 1679 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1680 PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ)); 1681 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1682 for (c = cStart; c < cEnd; ++c) { 1683 PetscScalar *x = NULL; 1684 PetscScalar elemDiff = 0.0; 1685 PetscInt qc = 0; 1686 1687 PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1688 PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x)); 1689 1690 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1691 PetscObject obj; 1692 PetscClassId id; 1693 void *const ctx = ctxs ? ctxs[field] : NULL; 1694 PetscInt Nb, Nc, q, fc; 1695 1696 PetscCall(DMGetField(dm, field, NULL, &obj)); 1697 PetscCall(PetscObjectGetClassId(obj, &id)); 1698 if (id == PETSCFE_CLASSID) { 1699 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1700 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1701 } else if (id == PETSCFV_CLASSID) { 1702 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1703 Nb = 1; 1704 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1705 if (funcs[field]) { 1706 for (q = 0; q < Nq; ++q) { 1707 PetscFEGeom qgeom; 1708 1709 qgeom.dimEmbed = fegeom.dimEmbed; 1710 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1711 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1712 qgeom.detJ = &fegeom.detJ[q]; 1713 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q); 1714 PetscCall((*funcs[field])(coordDim, time, &coords[q * coordDim], Nc, funcVal, ctx)); 1715 #if defined(needs_fix_with_return_code_argument) 1716 if (ierr) { 1717 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1718 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1719 PetscCall(DMRestoreLocalVector(dm, &localX)); 1720 } 1721 #endif 1722 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant)); 1723 else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant)); 1724 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1725 for (fc = 0; fc < Nc; ++fc) { 1726 const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)]; 1727 elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]; 1728 } 1729 } 1730 } 1731 fieldOffset += Nb; 1732 qc += Nc; 1733 } 1734 PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x)); 1735 PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES)); 1736 } 1737 PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1738 PetscCall(DMRestoreLocalVector(dm, &localX)); 1739 PetscCall(VecSqrtAbs(D)); 1740 PetscFunctionReturn(PETSC_SUCCESS); 1741 } 1742 1743 /*@ 1744 DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1 1745 1746 Collective 1747 1748 Input Parameters: 1749 + dm - The `DM` 1750 - locX - The coefficient vector u_h 1751 1752 Output Parameter: 1753 . locC - A `Vec` which holds the Clement interpolant of the function 1754 1755 Level: developer 1756 1757 Note: 1758 $ u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume 1759 1760 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1761 @*/ 1762 PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC) 1763 { 1764 PetscInt debug = ((DM_Plex *)dm->data)->printFEM; 1765 DM dmc; 1766 PetscQuadrature quad; 1767 PetscScalar *interpolant, *valsum; 1768 PetscFEGeom fegeom; 1769 PetscReal *coords; 1770 const PetscReal *quadPoints, *quadWeights; 1771 PetscInt dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v; 1772 1773 PetscFunctionBegin; 1774 PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite)); 1775 PetscCall(VecGetDM(locC, &dmc)); 1776 PetscCall(VecSet(locC, 0.0)); 1777 PetscCall(DMGetDimension(dm, &dim)); 1778 PetscCall(DMGetCoordinateDim(dm, &cdim)); 1779 fegeom.dimEmbed = cdim; 1780 PetscCall(DMGetNumFields(dm, &Nf)); 1781 PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!"); 1782 for (f = 0; f < Nf; ++f) { 1783 PetscObject obj; 1784 PetscClassId id; 1785 PetscInt fNc; 1786 1787 PetscCall(DMGetField(dm, f, NULL, &obj)); 1788 PetscCall(PetscObjectGetClassId(obj, &id)); 1789 if (id == PETSCFE_CLASSID) { 1790 PetscFE fe = (PetscFE)obj; 1791 1792 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1793 PetscCall(PetscFEGetNumComponents(fe, &fNc)); 1794 } else if (id == PETSCFV_CLASSID) { 1795 PetscFV fv = (PetscFV)obj; 1796 1797 PetscCall(PetscFVGetQuadrature(fv, &quad)); 1798 PetscCall(PetscFVGetNumComponents(fv, &fNc)); 1799 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 1800 Nc += fNc; 1801 } 1802 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 1803 PetscCheck(qNc == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc); 1804 PetscCall(PetscMalloc6(Nc * 2, &valsum, Nc, &interpolant, cdim * Nq, &coords, Nq, &fegeom.detJ, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ)); 1805 PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd)); 1806 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1807 for (v = vStart; v < vEnd; ++v) { 1808 PetscScalar volsum = 0.0; 1809 PetscInt *star = NULL; 1810 PetscInt starSize, st, fc; 1811 1812 PetscCall(PetscArrayzero(valsum, Nc)); 1813 PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 1814 for (st = 0; st < starSize * 2; st += 2) { 1815 const PetscInt cell = star[st]; 1816 PetscScalar *val = &valsum[Nc]; 1817 PetscScalar *x = NULL; 1818 PetscReal vol = 0.0; 1819 PetscInt foff = 0; 1820 1821 if ((cell < cStart) || (cell >= cEnd)) continue; 1822 PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1823 PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x)); 1824 for (f = 0; f < Nf; ++f) { 1825 PetscObject obj; 1826 PetscClassId id; 1827 PetscInt Nb, fNc, q; 1828 1829 PetscCall(PetscArrayzero(val, Nc)); 1830 PetscCall(DMGetField(dm, f, NULL, &obj)); 1831 PetscCall(PetscObjectGetClassId(obj, &id)); 1832 if (id == PETSCFE_CLASSID) { 1833 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &fNc)); 1834 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1835 } else if (id == PETSCFV_CLASSID) { 1836 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &fNc)); 1837 Nb = 1; 1838 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 1839 for (q = 0; q < Nq; ++q) { 1840 const PetscReal wt = quadWeights[q] * fegeom.detJ[q]; 1841 PetscFEGeom qgeom; 1842 1843 qgeom.dimEmbed = fegeom.dimEmbed; 1844 qgeom.J = &fegeom.J[q * cdim * cdim]; 1845 qgeom.invJ = &fegeom.invJ[q * cdim * cdim]; 1846 qgeom.detJ = &fegeom.detJ[q]; 1847 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q); 1848 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[foff], &qgeom, q, interpolant)); 1849 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 1850 for (fc = 0; fc < fNc; ++fc) val[foff + fc] += interpolant[fc] * wt; 1851 vol += wt; 1852 } 1853 foff += Nb; 1854 } 1855 PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x)); 1856 for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc]; 1857 volsum += vol; 1858 if (debug) { 1859 PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell)); 1860 for (fc = 0; fc < Nc; ++fc) { 1861 if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", ")); 1862 PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(val[fc]))); 1863 } 1864 PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n")); 1865 } 1866 } 1867 for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum; 1868 PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 1869 PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES)); 1870 } 1871 PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 1872 PetscFunctionReturn(PETSC_SUCCESS); 1873 } 1874 1875 /*@ 1876 DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1 1877 1878 Collective 1879 1880 Input Parameters: 1881 + dm - The `DM` 1882 - locX - The coefficient vector u_h 1883 1884 Output Parameter: 1885 . locC - A `Vec` which holds the Clement interpolant of the gradient 1886 1887 Level: developer 1888 1889 Note: 1890 $\nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| $ where $ |T_i| $ is the cell volume 1891 1892 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()` 1893 @*/ 1894 PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC) 1895 { 1896 DM_Plex *mesh = (DM_Plex *)dm->data; 1897 PetscInt debug = mesh->printFEM; 1898 DM dmC; 1899 PetscQuadrature quad; 1900 PetscScalar *interpolant, *gradsum; 1901 PetscFEGeom fegeom; 1902 PetscReal *coords; 1903 const PetscReal *quadPoints, *quadWeights; 1904 PetscInt dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset; 1905 1906 PetscFunctionBegin; 1907 PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite)); 1908 PetscCall(VecGetDM(locC, &dmC)); 1909 PetscCall(VecSet(locC, 0.0)); 1910 PetscCall(DMGetDimension(dm, &dim)); 1911 PetscCall(DMGetCoordinateDim(dm, &coordDim)); 1912 fegeom.dimEmbed = coordDim; 1913 PetscCall(DMGetNumFields(dm, &numFields)); 1914 PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!"); 1915 for (field = 0; field < numFields; ++field) { 1916 PetscObject obj; 1917 PetscClassId id; 1918 PetscInt Nc; 1919 1920 PetscCall(DMGetField(dm, field, NULL, &obj)); 1921 PetscCall(PetscObjectGetClassId(obj, &id)); 1922 if (id == PETSCFE_CLASSID) { 1923 PetscFE fe = (PetscFE)obj; 1924 1925 PetscCall(PetscFEGetQuadrature(fe, &quad)); 1926 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 1927 } else if (id == PETSCFV_CLASSID) { 1928 PetscFV fv = (PetscFV)obj; 1929 1930 PetscCall(PetscFVGetQuadrature(fv, &quad)); 1931 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 1932 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1933 numComponents += Nc; 1934 } 1935 PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights)); 1936 PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents); 1937 PetscCall(PetscMalloc6(coordDim * numComponents * 2, &gradsum, coordDim * numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ)); 1938 PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd)); 1939 PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd)); 1940 for (v = vStart; v < vEnd; ++v) { 1941 PetscScalar volsum = 0.0; 1942 PetscInt *star = NULL; 1943 PetscInt starSize, st, d, fc; 1944 1945 PetscCall(PetscArrayzero(gradsum, coordDim * numComponents)); 1946 PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 1947 for (st = 0; st < starSize * 2; st += 2) { 1948 const PetscInt cell = star[st]; 1949 PetscScalar *grad = &gradsum[coordDim * numComponents]; 1950 PetscScalar *x = NULL; 1951 PetscReal vol = 0.0; 1952 1953 if ((cell < cStart) || (cell >= cEnd)) continue; 1954 PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ)); 1955 PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x)); 1956 for (field = 0, fieldOffset = 0; field < numFields; ++field) { 1957 PetscObject obj; 1958 PetscClassId id; 1959 PetscInt Nb, Nc, q, qc = 0; 1960 1961 PetscCall(PetscArrayzero(grad, coordDim * numComponents)); 1962 PetscCall(DMGetField(dm, field, NULL, &obj)); 1963 PetscCall(PetscObjectGetClassId(obj, &id)); 1964 if (id == PETSCFE_CLASSID) { 1965 PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc)); 1966 PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb)); 1967 } else if (id == PETSCFV_CLASSID) { 1968 PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc)); 1969 Nb = 1; 1970 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1971 for (q = 0; q < Nq; ++q) { 1972 PetscFEGeom qgeom; 1973 1974 qgeom.dimEmbed = fegeom.dimEmbed; 1975 qgeom.J = &fegeom.J[q * coordDim * coordDim]; 1976 qgeom.invJ = &fegeom.invJ[q * coordDim * coordDim]; 1977 qgeom.detJ = &fegeom.detJ[q]; 1978 PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q); 1979 if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolateGradient_Static((PetscFE)obj, 1, &x[fieldOffset], &qgeom, q, interpolant)); 1980 else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 1981 for (fc = 0; fc < Nc; ++fc) { 1982 const PetscReal wt = quadWeights[q * qNc + qc]; 1983 1984 for (d = 0; d < coordDim; ++d) grad[fc * coordDim + d] += interpolant[fc * dim + d] * wt * fegeom.detJ[q]; 1985 } 1986 vol += quadWeights[q * qNc] * fegeom.detJ[q]; 1987 } 1988 fieldOffset += Nb; 1989 qc += Nc; 1990 } 1991 PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x)); 1992 for (fc = 0; fc < numComponents; ++fc) { 1993 for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] += grad[fc * coordDim + d]; 1994 } 1995 volsum += vol; 1996 if (debug) { 1997 PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell)); 1998 for (fc = 0; fc < numComponents; ++fc) { 1999 for (d = 0; d < coordDim; ++d) { 2000 if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", ")); 2001 PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc * coordDim + d]))); 2002 } 2003 } 2004 PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n")); 2005 } 2006 } 2007 for (fc = 0; fc < numComponents; ++fc) { 2008 for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] /= volsum; 2009 } 2010 PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star)); 2011 PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES)); 2012 } 2013 PetscCall(PetscFree6(gradsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ)); 2014 PetscFunctionReturn(PETSC_SUCCESS); 2015 } 2016 2017 static PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec X, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user) 2018 { 2019 DM dmAux = NULL; 2020 PetscDS prob, probAux = NULL; 2021 PetscSection section, sectionAux; 2022 Vec locX, locA; 2023 PetscInt dim, numCells = cEnd - cStart, c, f; 2024 PetscBool useFVM = PETSC_FALSE; 2025 /* DS */ 2026 PetscInt Nf, totDim, *uOff, *uOff_x, numConstants; 2027 PetscInt NfAux, totDimAux, *aOff; 2028 PetscScalar *u, *a; 2029 const PetscScalar *constants; 2030 /* Geometry */ 2031 PetscFEGeom *cgeomFEM; 2032 DM dmGrad; 2033 PetscQuadrature affineQuad = NULL; 2034 Vec cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL; 2035 PetscFVCellGeom *cgeomFVM; 2036 const PetscScalar *lgrad; 2037 PetscInt maxDegree; 2038 DMField coordField; 2039 IS cellIS; 2040 2041 PetscFunctionBegin; 2042 PetscCall(DMGetDS(dm, &prob)); 2043 PetscCall(DMGetDimension(dm, &dim)); 2044 PetscCall(DMGetLocalSection(dm, §ion)); 2045 PetscCall(DMGetNumFields(dm, &Nf)); 2046 /* Determine which discretizations we have */ 2047 for (f = 0; f < Nf; ++f) { 2048 PetscObject obj; 2049 PetscClassId id; 2050 2051 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2052 PetscCall(PetscObjectGetClassId(obj, &id)); 2053 if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE; 2054 } 2055 /* Get local solution with boundary values */ 2056 PetscCall(DMGetLocalVector(dm, &locX)); 2057 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL)); 2058 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX)); 2059 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX)); 2060 /* Read DS information */ 2061 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 2062 PetscCall(PetscDSGetComponentOffsets(prob, &uOff)); 2063 PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x)); 2064 PetscCall(ISCreateStride(PETSC_COMM_SELF, numCells, cStart, 1, &cellIS)); 2065 PetscCall(PetscDSGetConstants(prob, &numConstants, &constants)); 2066 /* Read Auxiliary DS information */ 2067 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA)); 2068 if (locA) { 2069 PetscCall(VecGetDM(locA, &dmAux)); 2070 PetscCall(DMGetDS(dmAux, &probAux)); 2071 PetscCall(PetscDSGetNumFields(probAux, &NfAux)); 2072 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 2073 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 2074 PetscCall(PetscDSGetComponentOffsets(probAux, &aOff)); 2075 } 2076 /* Allocate data arrays */ 2077 PetscCall(PetscCalloc1(numCells * totDim, &u)); 2078 if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a)); 2079 /* Read out geometry */ 2080 PetscCall(DMGetCoordinateField(dm, &coordField)); 2081 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 2082 if (maxDegree <= 1) { 2083 PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad)); 2084 if (affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &cgeomFEM)); 2085 } 2086 if (useFVM) { 2087 PetscFV fv = NULL; 2088 Vec grad; 2089 PetscInt fStart, fEnd; 2090 PetscBool compGrad; 2091 2092 for (f = 0; f < Nf; ++f) { 2093 PetscObject obj; 2094 PetscClassId id; 2095 2096 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2097 PetscCall(PetscObjectGetClassId(obj, &id)); 2098 if (id == PETSCFV_CLASSID) { 2099 fv = (PetscFV)obj; 2100 break; 2101 } 2102 } 2103 PetscCall(PetscFVGetComputeGradients(fv, &compGrad)); 2104 PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE)); 2105 PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM)); 2106 PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad)); 2107 PetscCall(PetscFVSetComputeGradients(fv, compGrad)); 2108 PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM)); 2109 /* Reconstruct and limit cell gradients */ 2110 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 2111 PetscCall(DMGetGlobalVector(dmGrad, &grad)); 2112 PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad)); 2113 /* Communicate gradient values */ 2114 PetscCall(DMGetLocalVector(dmGrad, &locGrad)); 2115 PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad)); 2116 PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad)); 2117 PetscCall(DMRestoreGlobalVector(dmGrad, &grad)); 2118 /* Handle non-essential (e.g. outflow) boundary values */ 2119 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad)); 2120 PetscCall(VecGetArrayRead(locGrad, &lgrad)); 2121 } 2122 /* Read out data from inputs */ 2123 for (c = cStart; c < cEnd; ++c) { 2124 PetscScalar *x = NULL; 2125 PetscInt i; 2126 2127 PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x)); 2128 for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i]; 2129 PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x)); 2130 if (dmAux) { 2131 PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, locA, c, NULL, &x)); 2132 for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i]; 2133 PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, locA, c, NULL, &x)); 2134 } 2135 } 2136 /* Do integration for each field */ 2137 for (f = 0; f < Nf; ++f) { 2138 PetscObject obj; 2139 PetscClassId id; 2140 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset; 2141 2142 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2143 PetscCall(PetscObjectGetClassId(obj, &id)); 2144 if (id == PETSCFE_CLASSID) { 2145 PetscFE fe = (PetscFE)obj; 2146 PetscQuadrature q; 2147 PetscFEGeom *chunkGeom = NULL; 2148 PetscInt Nq, Nb; 2149 2150 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 2151 PetscCall(PetscFEGetQuadrature(fe, &q)); 2152 PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL)); 2153 PetscCall(PetscFEGetDimension(fe, &Nb)); 2154 blockSize = Nb * Nq; 2155 batchSize = numBlocks * blockSize; 2156 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 2157 numChunks = numCells / (numBatches * batchSize); 2158 Ne = numChunks * numBatches * batchSize; 2159 Nr = numCells % (numBatches * batchSize); 2160 offset = numCells - Nr; 2161 if (!affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, q, PETSC_FALSE, &cgeomFEM)); 2162 PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom)); 2163 PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral)); 2164 PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &chunkGeom)); 2165 PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset * totDim], probAux, &a[offset * totDimAux], &cintegral[offset * Nf])); 2166 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &chunkGeom)); 2167 if (!affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM)); 2168 } else if (id == PETSCFV_CLASSID) { 2169 PetscInt foff; 2170 PetscPointFunc obj_func; 2171 PetscScalar lint; 2172 2173 PetscCall(PetscDSGetObjective(prob, f, &obj_func)); 2174 PetscCall(PetscDSGetFieldOffset(prob, f, &foff)); 2175 if (obj_func) { 2176 for (c = 0; c < numCells; ++c) { 2177 PetscScalar *u_x; 2178 2179 PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x)); 2180 obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim * c + foff], NULL, u_x, aOff, NULL, &a[totDimAux * c], NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint); 2181 cintegral[c * Nf + f] += PetscRealPart(lint) * cgeomFVM[c].volume; 2182 } 2183 } 2184 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 2185 } 2186 /* Cleanup data arrays */ 2187 if (useFVM) { 2188 PetscCall(VecRestoreArrayRead(locGrad, &lgrad)); 2189 PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM)); 2190 PetscCall(DMRestoreLocalVector(dmGrad, &locGrad)); 2191 PetscCall(VecDestroy(&faceGeometryFVM)); 2192 PetscCall(VecDestroy(&cellGeometryFVM)); 2193 PetscCall(DMDestroy(&dmGrad)); 2194 } 2195 if (dmAux) PetscCall(PetscFree(a)); 2196 PetscCall(PetscFree(u)); 2197 /* Cleanup */ 2198 if (affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM)); 2199 PetscCall(PetscQuadratureDestroy(&affineQuad)); 2200 PetscCall(ISDestroy(&cellIS)); 2201 PetscCall(DMRestoreLocalVector(dm, &locX)); 2202 PetscFunctionReturn(PETSC_SUCCESS); 2203 } 2204 2205 /*@ 2206 DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user 2207 2208 Input Parameters: 2209 + dm - The mesh 2210 . X - Global input vector 2211 - user - The user context 2212 2213 Output Parameter: 2214 . integral - Integral for each field 2215 2216 Level: developer 2217 2218 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()` 2219 @*/ 2220 PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user) 2221 { 2222 DM_Plex *mesh = (DM_Plex *)dm->data; 2223 PetscScalar *cintegral, *lintegral; 2224 PetscInt Nf, f, cellHeight, cStart, cEnd, cell; 2225 2226 PetscFunctionBegin; 2227 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2228 PetscValidHeaderSpecific(X, VEC_CLASSID, 2); 2229 PetscValidScalarPointer(integral, 3); 2230 PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2231 PetscCall(DMGetNumFields(dm, &Nf)); 2232 PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight)); 2233 PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd)); 2234 /* TODO Introduce a loop over large chunks (right now this is a single chunk) */ 2235 PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd - cStart) * Nf, &cintegral)); 2236 PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user)); 2237 /* Sum up values */ 2238 for (cell = cStart; cell < cEnd; ++cell) { 2239 const PetscInt c = cell - cStart; 2240 2241 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf])); 2242 for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c * Nf + f]; 2243 } 2244 PetscCall(MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject)dm))); 2245 if (mesh->printFEM) { 2246 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "Integral:")); 2247 for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), " %g", (double)PetscRealPart(integral[f]))); 2248 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "\n")); 2249 } 2250 PetscCall(PetscFree2(lintegral, cintegral)); 2251 PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2252 PetscFunctionReturn(PETSC_SUCCESS); 2253 } 2254 2255 /*@ 2256 DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user 2257 2258 Input Parameters: 2259 + dm - The mesh 2260 . X - Global input vector 2261 - user - The user context 2262 2263 Output Parameter: 2264 . integral - Cellwise integrals for each field 2265 2266 Level: developer 2267 2268 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSNESComputeResidualFEM()` 2269 @*/ 2270 PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user) 2271 { 2272 DM_Plex *mesh = (DM_Plex *)dm->data; 2273 DM dmF; 2274 PetscSection sectionF; 2275 PetscScalar *cintegral, *af; 2276 PetscInt Nf, f, cellHeight, cStart, cEnd, cell; 2277 2278 PetscFunctionBegin; 2279 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2280 PetscValidHeaderSpecific(X, VEC_CLASSID, 2); 2281 PetscValidHeaderSpecific(F, VEC_CLASSID, 3); 2282 PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2283 PetscCall(DMGetNumFields(dm, &Nf)); 2284 PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight)); 2285 PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd)); 2286 /* TODO Introduce a loop over large chunks (right now this is a single chunk) */ 2287 PetscCall(PetscCalloc1((cEnd - cStart) * Nf, &cintegral)); 2288 PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user)); 2289 /* Put values in F*/ 2290 PetscCall(VecGetDM(F, &dmF)); 2291 PetscCall(DMGetLocalSection(dmF, §ionF)); 2292 PetscCall(VecGetArray(F, &af)); 2293 for (cell = cStart; cell < cEnd; ++cell) { 2294 const PetscInt c = cell - cStart; 2295 PetscInt dof, off; 2296 2297 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf])); 2298 PetscCall(PetscSectionGetDof(sectionF, cell, &dof)); 2299 PetscCall(PetscSectionGetOffset(sectionF, cell, &off)); 2300 PetscCheck(dof == Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf); 2301 for (f = 0; f < Nf; ++f) af[off + f] = cintegral[c * Nf + f]; 2302 } 2303 PetscCall(VecRestoreArray(F, &af)); 2304 PetscCall(PetscFree(cintegral)); 2305 PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2306 PetscFunctionReturn(PETSC_SUCCESS); 2307 } 2308 2309 static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS, void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *fintegral, void *user) 2310 { 2311 DM plex = NULL, plexA = NULL; 2312 DMEnclosureType encAux; 2313 PetscDS prob, probAux = NULL; 2314 PetscSection section, sectionAux = NULL; 2315 Vec locA = NULL; 2316 DMField coordField; 2317 PetscInt Nf, totDim, *uOff, *uOff_x; 2318 PetscInt NfAux = 0, totDimAux = 0, *aOff = NULL; 2319 PetscScalar *u, *a = NULL; 2320 const PetscScalar *constants; 2321 PetscInt numConstants, f; 2322 2323 PetscFunctionBegin; 2324 PetscCall(DMGetCoordinateField(dm, &coordField)); 2325 PetscCall(DMConvert(dm, DMPLEX, &plex)); 2326 PetscCall(DMGetDS(dm, &prob)); 2327 PetscCall(DMGetLocalSection(dm, §ion)); 2328 PetscCall(PetscSectionGetNumFields(section, &Nf)); 2329 /* Determine which discretizations we have */ 2330 for (f = 0; f < Nf; ++f) { 2331 PetscObject obj; 2332 PetscClassId id; 2333 2334 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 2335 PetscCall(PetscObjectGetClassId(obj, &id)); 2336 PetscCheck(id != PETSCFV_CLASSID, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f); 2337 } 2338 /* Read DS information */ 2339 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 2340 PetscCall(PetscDSGetComponentOffsets(prob, &uOff)); 2341 PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x)); 2342 PetscCall(PetscDSGetConstants(prob, &numConstants, &constants)); 2343 /* Read Auxiliary DS information */ 2344 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA)); 2345 if (locA) { 2346 DM dmAux; 2347 2348 PetscCall(VecGetDM(locA, &dmAux)); 2349 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 2350 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 2351 PetscCall(DMGetDS(dmAux, &probAux)); 2352 PetscCall(PetscDSGetNumFields(probAux, &NfAux)); 2353 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 2354 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 2355 PetscCall(PetscDSGetComponentOffsets(probAux, &aOff)); 2356 } 2357 /* Integrate over points */ 2358 { 2359 PetscFEGeom *fgeom, *chunkGeom = NULL; 2360 PetscInt maxDegree; 2361 PetscQuadrature qGeom = NULL; 2362 const PetscInt *points; 2363 PetscInt numFaces, face, Nq, field; 2364 PetscInt numChunks, chunkSize, chunk, Nr, offset; 2365 2366 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 2367 PetscCall(ISGetIndices(pointIS, &points)); 2368 PetscCall(PetscCalloc2(numFaces * totDim, &u, locA ? numFaces * totDimAux : 0, &a)); 2369 PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree)); 2370 for (field = 0; field < Nf; ++field) { 2371 PetscFE fe; 2372 2373 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fe)); 2374 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom)); 2375 if (!qGeom) { 2376 PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom)); 2377 PetscCall(PetscObjectReference((PetscObject)qGeom)); 2378 } 2379 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 2380 PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 2381 for (face = 0; face < numFaces; ++face) { 2382 const PetscInt point = points[face], *support; 2383 PetscScalar *x = NULL; 2384 PetscInt i; 2385 2386 PetscCall(DMPlexGetSupport(dm, point, &support)); 2387 PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x)); 2388 for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i]; 2389 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x)); 2390 if (locA) { 2391 PetscInt subp; 2392 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp)); 2393 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x)); 2394 for (i = 0; i < totDimAux; ++i) a[f * totDimAux + i] = x[i]; 2395 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x)); 2396 } 2397 } 2398 /* Get blocking */ 2399 { 2400 PetscQuadrature q; 2401 PetscInt numBatches, batchSize, numBlocks, blockSize; 2402 PetscInt Nq, Nb; 2403 2404 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 2405 PetscCall(PetscFEGetQuadrature(fe, &q)); 2406 PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL)); 2407 PetscCall(PetscFEGetDimension(fe, &Nb)); 2408 blockSize = Nb * Nq; 2409 batchSize = numBlocks * blockSize; 2410 chunkSize = numBatches * batchSize; 2411 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 2412 numChunks = numFaces / chunkSize; 2413 Nr = numFaces % chunkSize; 2414 offset = numFaces - Nr; 2415 } 2416 /* Do integration for each field */ 2417 for (chunk = 0; chunk < numChunks; ++chunk) { 2418 PetscCall(PetscFEGeomGetChunk(fgeom, chunk * chunkSize, (chunk + 1) * chunkSize, &chunkGeom)); 2419 PetscCall(PetscFEIntegrateBd(prob, field, func, chunkSize, chunkGeom, u, probAux, a, fintegral)); 2420 PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom)); 2421 } 2422 PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom)); 2423 PetscCall(PetscFEIntegrateBd(prob, field, func, Nr, chunkGeom, &u[offset * totDim], probAux, a ? &a[offset * totDimAux] : NULL, &fintegral[offset * Nf])); 2424 PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom)); 2425 /* Cleanup data arrays */ 2426 PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 2427 PetscCall(PetscQuadratureDestroy(&qGeom)); 2428 PetscCall(PetscFree2(u, a)); 2429 PetscCall(ISRestoreIndices(pointIS, &points)); 2430 } 2431 } 2432 if (plex) PetscCall(DMDestroy(&plex)); 2433 if (plexA) PetscCall(DMDestroy(&plexA)); 2434 PetscFunctionReturn(PETSC_SUCCESS); 2435 } 2436 2437 /*@ 2438 DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user 2439 2440 Input Parameters: 2441 + dm - The mesh 2442 . X - Global input vector 2443 . label - The boundary `DMLabel` 2444 . numVals - The number of label values to use, or `PETSC_DETERMINE` for all values 2445 . vals - The label values to use, or NULL for all values 2446 . func - The function to integrate along the boundary 2447 - user - The user context 2448 2449 Output Parameter: 2450 . integral - Integral for each field 2451 2452 Level: developer 2453 2454 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeIntegralFEM()`, `DMPlexComputeBdResidualFEM()` 2455 @*/ 2456 PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *integral, void *user) 2457 { 2458 Vec locX; 2459 PetscSection section; 2460 DMLabel depthLabel; 2461 IS facetIS; 2462 PetscInt dim, Nf, f, v; 2463 2464 PetscFunctionBegin; 2465 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 2466 PetscValidHeaderSpecific(X, VEC_CLASSID, 2); 2467 PetscValidPointer(label, 3); 2468 if (vals) PetscValidIntPointer(vals, 5); 2469 PetscValidScalarPointer(integral, 7); 2470 PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2471 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 2472 PetscCall(DMGetDimension(dm, &dim)); 2473 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 2474 PetscCall(DMGetLocalSection(dm, §ion)); 2475 PetscCall(PetscSectionGetNumFields(section, &Nf)); 2476 /* Get local solution with boundary values */ 2477 PetscCall(DMGetLocalVector(dm, &locX)); 2478 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL)); 2479 PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX)); 2480 PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX)); 2481 /* Loop over label values */ 2482 PetscCall(PetscArrayzero(integral, Nf)); 2483 for (v = 0; v < numVals; ++v) { 2484 IS pointIS; 2485 PetscInt numFaces, face; 2486 PetscScalar *fintegral; 2487 2488 PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS)); 2489 if (!pointIS) continue; /* No points with that id on this process */ 2490 { 2491 IS isectIS; 2492 2493 /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */ 2494 PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS)); 2495 PetscCall(ISDestroy(&pointIS)); 2496 pointIS = isectIS; 2497 } 2498 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 2499 PetscCall(PetscCalloc1(numFaces * Nf, &fintegral)); 2500 PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, func, fintegral, user)); 2501 /* Sum point contributions into integral */ 2502 for (f = 0; f < Nf; ++f) 2503 for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face * Nf + f]; 2504 PetscCall(PetscFree(fintegral)); 2505 PetscCall(ISDestroy(&pointIS)); 2506 } 2507 PetscCall(DMRestoreLocalVector(dm, &locX)); 2508 PetscCall(ISDestroy(&facetIS)); 2509 PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0)); 2510 PetscFunctionReturn(PETSC_SUCCESS); 2511 } 2512 2513 /*@ 2514 DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix I from the coarse `DM` to a uniformly refined `DM`. 2515 2516 Input Parameters: 2517 + dmc - The coarse mesh 2518 . dmf - The fine mesh 2519 . isRefined - Flag indicating regular refinement, rather than the same topology 2520 - user - The user context 2521 2522 Output Parameter: 2523 . In - The interpolation matrix 2524 2525 Level: developer 2526 2527 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorGeneral()`, `DMPlexComputeJacobianFEM()` 2528 @*/ 2529 PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user) 2530 { 2531 DM_Plex *mesh = (DM_Plex *)dmc->data; 2532 const char *name = "Interpolator"; 2533 PetscFE *feRef; 2534 PetscFV *fvRef; 2535 PetscSection fsection, fglobalSection; 2536 PetscSection csection, cglobalSection; 2537 PetscScalar *elemMat; 2538 PetscInt dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c; 2539 PetscInt cTotDim = 0, rTotDim = 0; 2540 2541 PetscFunctionBegin; 2542 PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 2543 PetscCall(DMGetDimension(dmf, &dim)); 2544 PetscCall(DMGetLocalSection(dmf, &fsection)); 2545 PetscCall(DMGetGlobalSection(dmf, &fglobalSection)); 2546 PetscCall(DMGetLocalSection(dmc, &csection)); 2547 PetscCall(DMGetGlobalSection(dmc, &cglobalSection)); 2548 PetscCall(PetscSectionGetNumFields(fsection, &Nf)); 2549 PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd)); 2550 PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef)); 2551 for (f = 0; f < Nf; ++f) { 2552 PetscObject obj, objc; 2553 PetscClassId id, idc; 2554 PetscInt rNb = 0, Nc = 0, cNb = 0; 2555 2556 PetscCall(DMGetField(dmf, f, NULL, &obj)); 2557 PetscCall(PetscObjectGetClassId(obj, &id)); 2558 if (id == PETSCFE_CLASSID) { 2559 PetscFE fe = (PetscFE)obj; 2560 2561 if (isRefined) { 2562 PetscCall(PetscFERefine(fe, &feRef[f])); 2563 } else { 2564 PetscCall(PetscObjectReference((PetscObject)fe)); 2565 feRef[f] = fe; 2566 } 2567 PetscCall(PetscFEGetDimension(feRef[f], &rNb)); 2568 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 2569 } else if (id == PETSCFV_CLASSID) { 2570 PetscFV fv = (PetscFV)obj; 2571 PetscDualSpace Q; 2572 2573 if (isRefined) { 2574 PetscCall(PetscFVRefine(fv, &fvRef[f])); 2575 } else { 2576 PetscCall(PetscObjectReference((PetscObject)fv)); 2577 fvRef[f] = fv; 2578 } 2579 PetscCall(PetscFVGetDualSpace(fvRef[f], &Q)); 2580 PetscCall(PetscDualSpaceGetDimension(Q, &rNb)); 2581 PetscCall(PetscFVGetDualSpace(fv, &Q)); 2582 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 2583 } 2584 PetscCall(DMGetField(dmc, f, NULL, &objc)); 2585 PetscCall(PetscObjectGetClassId(objc, &idc)); 2586 if (idc == PETSCFE_CLASSID) { 2587 PetscFE fe = (PetscFE)objc; 2588 2589 PetscCall(PetscFEGetDimension(fe, &cNb)); 2590 } else if (id == PETSCFV_CLASSID) { 2591 PetscFV fv = (PetscFV)obj; 2592 PetscDualSpace Q; 2593 2594 PetscCall(PetscFVGetDualSpace(fv, &Q)); 2595 PetscCall(PetscDualSpaceGetDimension(Q, &cNb)); 2596 } 2597 rTotDim += rNb; 2598 cTotDim += cNb; 2599 } 2600 PetscCall(PetscMalloc1(rTotDim * cTotDim, &elemMat)); 2601 PetscCall(PetscArrayzero(elemMat, rTotDim * cTotDim)); 2602 for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) { 2603 PetscDualSpace Qref; 2604 PetscQuadrature f; 2605 const PetscReal *qpoints, *qweights; 2606 PetscReal *points; 2607 PetscInt npoints = 0, Nc, Np, fpdim, i, k, p, d; 2608 2609 /* Compose points from all dual basis functionals */ 2610 if (feRef[fieldI]) { 2611 PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref)); 2612 PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc)); 2613 } else { 2614 PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref)); 2615 PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc)); 2616 } 2617 PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim)); 2618 for (i = 0; i < fpdim; ++i) { 2619 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2620 PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL)); 2621 npoints += Np; 2622 } 2623 PetscCall(PetscMalloc1(npoints * dim, &points)); 2624 for (i = 0, k = 0; i < fpdim; ++i) { 2625 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2626 PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL)); 2627 for (p = 0; p < Np; ++p, ++k) 2628 for (d = 0; d < dim; ++d) points[k * dim + d] = qpoints[p * dim + d]; 2629 } 2630 2631 for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) { 2632 PetscObject obj; 2633 PetscClassId id; 2634 PetscInt NcJ = 0, cpdim = 0, j, qNc; 2635 2636 PetscCall(DMGetField(dmc, fieldJ, NULL, &obj)); 2637 PetscCall(PetscObjectGetClassId(obj, &id)); 2638 if (id == PETSCFE_CLASSID) { 2639 PetscFE fe = (PetscFE)obj; 2640 PetscTabulation T = NULL; 2641 2642 /* Evaluate basis at points */ 2643 PetscCall(PetscFEGetNumComponents(fe, &NcJ)); 2644 PetscCall(PetscFEGetDimension(fe, &cpdim)); 2645 /* For now, fields only interpolate themselves */ 2646 if (fieldI == fieldJ) { 2647 PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ); 2648 PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T)); 2649 for (i = 0, k = 0; i < fpdim; ++i) { 2650 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2651 PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights)); 2652 PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ); 2653 for (p = 0; p < Np; ++p, ++k) { 2654 for (j = 0; j < cpdim; ++j) { 2655 /* 2656 cTotDim: Total columns in element interpolation matrix, sum of number of dual basis functionals in each field 2657 offsetI, offsetJ: Offsets into the larger element interpolation matrix for different fields 2658 fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals 2659 qNC, Nc, Ncj, c: Number of components in this field 2660 Np, p: Number of quad points in the fine grid functional i 2661 k: i*Np + p, overall point number for the interpolation 2662 */ 2663 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += T->T[0][k * cpdim * NcJ + j * Nc + c] * qweights[p * qNc + c]; 2664 } 2665 } 2666 } 2667 PetscCall(PetscTabulationDestroy(&T)); 2668 } 2669 } else if (id == PETSCFV_CLASSID) { 2670 PetscFV fv = (PetscFV)obj; 2671 2672 /* Evaluate constant function at points */ 2673 PetscCall(PetscFVGetNumComponents(fv, &NcJ)); 2674 cpdim = 1; 2675 /* For now, fields only interpolate themselves */ 2676 if (fieldI == fieldJ) { 2677 PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ); 2678 for (i = 0, k = 0; i < fpdim; ++i) { 2679 PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f)); 2680 PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights)); 2681 PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ); 2682 for (p = 0; p < Np; ++p, ++k) { 2683 for (j = 0; j < cpdim; ++j) { 2684 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += 1.0 * qweights[p * qNc + c]; 2685 } 2686 } 2687 } 2688 } 2689 } 2690 offsetJ += cpdim; 2691 } 2692 offsetI += fpdim; 2693 PetscCall(PetscFree(points)); 2694 } 2695 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat)); 2696 /* Preallocate matrix */ 2697 { 2698 Mat preallocator; 2699 PetscScalar *vals; 2700 PetscInt *cellCIndices, *cellFIndices; 2701 PetscInt locRows, locCols, cell; 2702 2703 PetscCall(MatGetLocalSize(In, &locRows, &locCols)); 2704 PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &preallocator)); 2705 PetscCall(MatSetType(preallocator, MATPREALLOCATOR)); 2706 PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE)); 2707 PetscCall(MatSetUp(preallocator)); 2708 PetscCall(PetscCalloc3(rTotDim * cTotDim, &vals, cTotDim, &cellCIndices, rTotDim, &cellFIndices)); 2709 for (cell = cStart; cell < cEnd; ++cell) { 2710 if (isRefined) { 2711 PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices)); 2712 PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES)); 2713 } else { 2714 PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, preallocator, cell, vals, INSERT_VALUES)); 2715 } 2716 } 2717 PetscCall(PetscFree3(vals, cellCIndices, cellFIndices)); 2718 PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY)); 2719 PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY)); 2720 PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In)); 2721 PetscCall(MatDestroy(&preallocator)); 2722 } 2723 /* Fill matrix */ 2724 PetscCall(MatZeroEntries(In)); 2725 for (c = cStart; c < cEnd; ++c) { 2726 if (isRefined) { 2727 PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES)); 2728 } else { 2729 PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES)); 2730 } 2731 } 2732 for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f])); 2733 PetscCall(PetscFree2(feRef, fvRef)); 2734 PetscCall(PetscFree(elemMat)); 2735 PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY)); 2736 PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY)); 2737 if (mesh->printFEM > 1) { 2738 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name)); 2739 PetscCall(MatChop(In, 1.0e-10)); 2740 PetscCall(MatView(In, NULL)); 2741 } 2742 PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 2743 PetscFunctionReturn(PETSC_SUCCESS); 2744 } 2745 2746 PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user) 2747 { 2748 SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_SUP, "Laziness"); 2749 } 2750 2751 /*@ 2752 DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix I from the coarse `DM` to a non-nested fine `DM`. 2753 2754 Input Parameters: 2755 + dmf - The fine mesh 2756 . dmc - The coarse mesh 2757 - user - The user context 2758 2759 Output Parameter: 2760 . In - The interpolation matrix 2761 2762 Level: developer 2763 2764 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeJacobianFEM()` 2765 @*/ 2766 PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user) 2767 { 2768 DM_Plex *mesh = (DM_Plex *)dmf->data; 2769 const char *name = "Interpolator"; 2770 PetscDS prob; 2771 Mat interp; 2772 PetscSection fsection, globalFSection; 2773 PetscSection csection, globalCSection; 2774 PetscInt locRows, locCols; 2775 PetscReal *x, *v0, *J, *invJ, detJ; 2776 PetscReal *v0c, *Jc, *invJc, detJc; 2777 PetscScalar *elemMat; 2778 PetscInt dim, Nf, field, totDim, cStart, cEnd, cell, ccell, s; 2779 2780 PetscFunctionBegin; 2781 PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 2782 PetscCall(DMGetCoordinateDim(dmc, &dim)); 2783 PetscCall(DMGetDS(dmc, &prob)); 2784 PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL)); 2785 PetscCall(PetscDSGetNumFields(prob, &Nf)); 2786 PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ)); 2787 PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc)); 2788 PetscCall(DMGetLocalSection(dmf, &fsection)); 2789 PetscCall(DMGetGlobalSection(dmf, &globalFSection)); 2790 PetscCall(DMGetLocalSection(dmc, &csection)); 2791 PetscCall(DMGetGlobalSection(dmc, &globalCSection)); 2792 PetscCall(DMPlexGetSimplexOrBoxCells(dmf, 0, &cStart, &cEnd)); 2793 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 2794 PetscCall(PetscMalloc1(totDim, &elemMat)); 2795 2796 PetscCall(MatGetLocalSize(In, &locRows, &locCols)); 2797 PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &interp)); 2798 PetscCall(MatSetType(interp, MATPREALLOCATOR)); 2799 PetscCall(MatSetSizes(interp, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE)); 2800 PetscCall(MatSetUp(interp)); 2801 for (s = 0; s < 2; ++s) { 2802 for (field = 0; field < Nf; ++field) { 2803 PetscObject obj; 2804 PetscClassId id; 2805 PetscDualSpace Q = NULL; 2806 PetscTabulation T = NULL; 2807 PetscQuadrature f; 2808 const PetscReal *qpoints, *qweights; 2809 PetscInt Nc, qNc, Np, fpdim, off, i, d; 2810 2811 PetscCall(PetscDSGetFieldOffset(prob, field, &off)); 2812 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 2813 PetscCall(PetscObjectGetClassId(obj, &id)); 2814 if (id == PETSCFE_CLASSID) { 2815 PetscFE fe = (PetscFE)obj; 2816 2817 PetscCall(PetscFEGetDualSpace(fe, &Q)); 2818 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 2819 if (s) PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T)); 2820 } else if (id == PETSCFV_CLASSID) { 2821 PetscFV fv = (PetscFV)obj; 2822 2823 PetscCall(PetscFVGetDualSpace(fv, &Q)); 2824 Nc = 1; 2825 } else SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field); 2826 PetscCall(PetscDualSpaceGetDimension(Q, &fpdim)); 2827 /* For each fine grid cell */ 2828 for (cell = cStart; cell < cEnd; ++cell) { 2829 PetscInt *findices, *cindices; 2830 PetscInt numFIndices, numCIndices; 2831 2832 PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 2833 PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ)); 2834 PetscCheck(numFIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim); 2835 for (i = 0; i < fpdim; ++i) { 2836 Vec pointVec; 2837 PetscScalar *pV; 2838 PetscSF coarseCellSF = NULL; 2839 const PetscSFNode *coarseCells; 2840 PetscInt numCoarseCells, cpdim, row = findices[i + off], q, c, j; 2841 2842 /* Get points from the dual basis functional quadrature */ 2843 PetscCall(PetscDualSpaceGetFunctional(Q, i, &f)); 2844 PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights)); 2845 PetscCheck(qNc == Nc, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc); 2846 PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np * dim, &pointVec)); 2847 PetscCall(VecSetBlockSize(pointVec, dim)); 2848 PetscCall(VecGetArray(pointVec, &pV)); 2849 for (q = 0; q < Np; ++q) { 2850 const PetscReal xi0[3] = {-1., -1., -1.}; 2851 2852 /* Transform point to real space */ 2853 CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x); 2854 for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d]; 2855 } 2856 PetscCall(VecRestoreArray(pointVec, &pV)); 2857 /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */ 2858 /* OPT: Read this out from preallocation information */ 2859 PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF)); 2860 /* Update preallocation info */ 2861 PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells)); 2862 PetscCheck(numCoarseCells == Np, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located"); 2863 PetscCall(VecGetArray(pointVec, &pV)); 2864 for (ccell = 0; ccell < numCoarseCells; ++ccell) { 2865 PetscReal pVReal[3]; 2866 const PetscReal xi0[3] = {-1., -1., -1.}; 2867 2868 PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 2869 if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetDimension((PetscFE)obj, &cpdim)); 2870 else cpdim = 1; 2871 2872 if (s) { 2873 /* Transform points from real space to coarse reference space */ 2874 PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc)); 2875 for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]); 2876 CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x); 2877 2878 if (id == PETSCFE_CLASSID) { 2879 /* Evaluate coarse basis on contained point */ 2880 PetscCall(PetscFEComputeTabulation((PetscFE)obj, 1, x, 0, T)); 2881 PetscCall(PetscArrayzero(elemMat, cpdim)); 2882 /* Get elemMat entries by multiplying by weight */ 2883 for (j = 0; j < cpdim; ++j) { 2884 for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * qweights[ccell * qNc + c]; 2885 } 2886 } else { 2887 for (j = 0; j < cpdim; ++j) { 2888 for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * qweights[ccell * qNc + c]; 2889 } 2890 } 2891 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat)); 2892 } 2893 /* Update interpolator */ 2894 PetscCheck(numCIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, totDim); 2895 PetscCall(MatSetValues(interp, 1, &row, cpdim, &cindices[off], elemMat, INSERT_VALUES)); 2896 PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 2897 } 2898 PetscCall(VecRestoreArray(pointVec, &pV)); 2899 PetscCall(PetscSFDestroy(&coarseCellSF)); 2900 PetscCall(VecDestroy(&pointVec)); 2901 } 2902 PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 2903 } 2904 if (s && id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T)); 2905 } 2906 if (!s) { 2907 PetscCall(MatAssemblyBegin(interp, MAT_FINAL_ASSEMBLY)); 2908 PetscCall(MatAssemblyEnd(interp, MAT_FINAL_ASSEMBLY)); 2909 PetscCall(MatPreallocatorPreallocate(interp, PETSC_TRUE, In)); 2910 PetscCall(MatDestroy(&interp)); 2911 interp = In; 2912 } 2913 } 2914 PetscCall(PetscFree3(v0, J, invJ)); 2915 PetscCall(PetscFree3(v0c, Jc, invJc)); 2916 PetscCall(PetscFree(elemMat)); 2917 PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY)); 2918 PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY)); 2919 PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0)); 2920 PetscFunctionReturn(PETSC_SUCCESS); 2921 } 2922 2923 /*@ 2924 DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix M from the coarse `DM` to a non-nested fine `DM`. 2925 2926 Input Parameters: 2927 + dmf - The fine mesh 2928 . dmc - The coarse mesh 2929 - user - The user context 2930 2931 Output Parameter: 2932 . mass - The mass matrix 2933 2934 Level: developer 2935 2936 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeMassMatrixNested()`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeInterpolatorGeneral()`, `DMPlexComputeJacobianFEM()` 2937 @*/ 2938 PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user) 2939 { 2940 DM_Plex *mesh = (DM_Plex *)dmf->data; 2941 const char *name = "Mass Matrix"; 2942 PetscDS prob; 2943 PetscSection fsection, csection, globalFSection, globalCSection; 2944 PetscHSetIJ ht; 2945 PetscLayout rLayout; 2946 PetscInt *dnz, *onz; 2947 PetscInt locRows, rStart, rEnd; 2948 PetscReal *x, *v0, *J, *invJ, detJ; 2949 PetscReal *v0c, *Jc, *invJc, detJc; 2950 PetscScalar *elemMat; 2951 PetscInt dim, Nf, field, totDim, cStart, cEnd, cell, ccell; 2952 2953 PetscFunctionBegin; 2954 PetscCall(DMGetCoordinateDim(dmc, &dim)); 2955 PetscCall(DMGetDS(dmc, &prob)); 2956 PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL)); 2957 PetscCall(PetscDSGetNumFields(prob, &Nf)); 2958 PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ)); 2959 PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc)); 2960 PetscCall(DMGetLocalSection(dmf, &fsection)); 2961 PetscCall(DMGetGlobalSection(dmf, &globalFSection)); 2962 PetscCall(DMGetLocalSection(dmc, &csection)); 2963 PetscCall(DMGetGlobalSection(dmc, &globalCSection)); 2964 PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd)); 2965 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 2966 PetscCall(PetscMalloc1(totDim, &elemMat)); 2967 2968 PetscCall(MatGetLocalSize(mass, &locRows, NULL)); 2969 PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)mass), &rLayout)); 2970 PetscCall(PetscLayoutSetLocalSize(rLayout, locRows)); 2971 PetscCall(PetscLayoutSetBlockSize(rLayout, 1)); 2972 PetscCall(PetscLayoutSetUp(rLayout)); 2973 PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd)); 2974 PetscCall(PetscLayoutDestroy(&rLayout)); 2975 PetscCall(PetscCalloc2(locRows, &dnz, locRows, &onz)); 2976 PetscCall(PetscHSetIJCreate(&ht)); 2977 for (field = 0; field < Nf; ++field) { 2978 PetscObject obj; 2979 PetscClassId id; 2980 PetscQuadrature quad; 2981 const PetscReal *qpoints; 2982 PetscInt Nq, Nc, i, d; 2983 2984 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 2985 PetscCall(PetscObjectGetClassId(obj, &id)); 2986 if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad)); 2987 else PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad)); 2988 PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL)); 2989 /* For each fine grid cell */ 2990 for (cell = cStart; cell < cEnd; ++cell) { 2991 Vec pointVec; 2992 PetscScalar *pV; 2993 PetscSF coarseCellSF = NULL; 2994 const PetscSFNode *coarseCells; 2995 PetscInt numCoarseCells, q, c; 2996 PetscInt *findices, *cindices; 2997 PetscInt numFIndices, numCIndices; 2998 2999 PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3000 PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ)); 3001 /* Get points from the quadrature */ 3002 PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec)); 3003 PetscCall(VecSetBlockSize(pointVec, dim)); 3004 PetscCall(VecGetArray(pointVec, &pV)); 3005 for (q = 0; q < Nq; ++q) { 3006 const PetscReal xi0[3] = {-1., -1., -1.}; 3007 3008 /* Transform point to real space */ 3009 CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x); 3010 for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d]; 3011 } 3012 PetscCall(VecRestoreArray(pointVec, &pV)); 3013 /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */ 3014 PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF)); 3015 PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view")); 3016 /* Update preallocation info */ 3017 PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells)); 3018 PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located"); 3019 { 3020 PetscHashIJKey key; 3021 PetscBool missing; 3022 3023 for (i = 0; i < numFIndices; ++i) { 3024 key.i = findices[i]; 3025 if (key.i >= 0) { 3026 /* Get indices for coarse elements */ 3027 for (ccell = 0; ccell < numCoarseCells; ++ccell) { 3028 PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3029 for (c = 0; c < numCIndices; ++c) { 3030 key.j = cindices[c]; 3031 if (key.j < 0) continue; 3032 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing)); 3033 if (missing) { 3034 if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i - rStart]; 3035 else ++onz[key.i - rStart]; 3036 } 3037 } 3038 PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3039 } 3040 } 3041 } 3042 } 3043 PetscCall(PetscSFDestroy(&coarseCellSF)); 3044 PetscCall(VecDestroy(&pointVec)); 3045 PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3046 } 3047 } 3048 PetscCall(PetscHSetIJDestroy(&ht)); 3049 PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL)); 3050 PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE)); 3051 PetscCall(PetscFree2(dnz, onz)); 3052 for (field = 0; field < Nf; ++field) { 3053 PetscObject obj; 3054 PetscClassId id; 3055 PetscTabulation T, Tfine; 3056 PetscQuadrature quad; 3057 const PetscReal *qpoints, *qweights; 3058 PetscInt Nq, Nc, i, d; 3059 3060 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 3061 PetscCall(PetscObjectGetClassId(obj, &id)); 3062 if (id == PETSCFE_CLASSID) { 3063 PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad)); 3064 PetscCall(PetscFEGetCellTabulation((PetscFE)obj, 1, &Tfine)); 3065 PetscCall(PetscFECreateTabulation((PetscFE)obj, 1, 1, x, 0, &T)); 3066 } else { 3067 PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad)); 3068 } 3069 PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights)); 3070 /* For each fine grid cell */ 3071 for (cell = cStart; cell < cEnd; ++cell) { 3072 Vec pointVec; 3073 PetscScalar *pV; 3074 PetscSF coarseCellSF = NULL; 3075 const PetscSFNode *coarseCells; 3076 PetscInt numCoarseCells, cpdim, q, c, j; 3077 PetscInt *findices, *cindices; 3078 PetscInt numFIndices, numCIndices; 3079 3080 PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3081 PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ)); 3082 /* Get points from the quadrature */ 3083 PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec)); 3084 PetscCall(VecSetBlockSize(pointVec, dim)); 3085 PetscCall(VecGetArray(pointVec, &pV)); 3086 for (q = 0; q < Nq; ++q) { 3087 const PetscReal xi0[3] = {-1., -1., -1.}; 3088 3089 /* Transform point to real space */ 3090 CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x); 3091 for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d]; 3092 } 3093 PetscCall(VecRestoreArray(pointVec, &pV)); 3094 /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */ 3095 PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF)); 3096 /* Update matrix */ 3097 PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells)); 3098 PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located"); 3099 PetscCall(VecGetArray(pointVec, &pV)); 3100 for (ccell = 0; ccell < numCoarseCells; ++ccell) { 3101 PetscReal pVReal[3]; 3102 const PetscReal xi0[3] = {-1., -1., -1.}; 3103 3104 PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3105 /* Transform points from real space to coarse reference space */ 3106 PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc)); 3107 for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]); 3108 CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x); 3109 3110 if (id == PETSCFE_CLASSID) { 3111 PetscFE fe = (PetscFE)obj; 3112 3113 /* Evaluate coarse basis on contained point */ 3114 PetscCall(PetscFEGetDimension(fe, &cpdim)); 3115 PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T)); 3116 /* Get elemMat entries by multiplying by weight */ 3117 for (i = 0; i < numFIndices; ++i) { 3118 PetscCall(PetscArrayzero(elemMat, cpdim)); 3119 for (j = 0; j < cpdim; ++j) { 3120 for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * Tfine->T[0][(ccell * numFIndices + i) * Nc + c] * qweights[ccell * Nc + c] * detJ; 3121 } 3122 /* Update interpolator */ 3123 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat)); 3124 PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim); 3125 PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES)); 3126 } 3127 } else { 3128 cpdim = 1; 3129 for (i = 0; i < numFIndices; ++i) { 3130 PetscCall(PetscArrayzero(elemMat, cpdim)); 3131 for (j = 0; j < cpdim; ++j) { 3132 for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * 1.0 * qweights[ccell * Nc + c] * detJ; 3133 } 3134 /* Update interpolator */ 3135 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat)); 3136 PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices)); 3137 PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim); 3138 PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES)); 3139 } 3140 } 3141 PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL)); 3142 } 3143 PetscCall(VecRestoreArray(pointVec, &pV)); 3144 PetscCall(PetscSFDestroy(&coarseCellSF)); 3145 PetscCall(VecDestroy(&pointVec)); 3146 PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL)); 3147 } 3148 if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T)); 3149 } 3150 PetscCall(PetscFree3(v0, J, invJ)); 3151 PetscCall(PetscFree3(v0c, Jc, invJc)); 3152 PetscCall(PetscFree(elemMat)); 3153 PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY)); 3154 PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY)); 3155 PetscFunctionReturn(PETSC_SUCCESS); 3156 } 3157 3158 /*@ 3159 DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns 3160 3161 Input Parameters: 3162 + dmc - The coarse mesh 3163 - dmf - The fine mesh 3164 - user - The user context 3165 3166 Output Parameter: 3167 . sc - The mapping 3168 3169 Level: developer 3170 3171 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeJacobianFEM()` 3172 @*/ 3173 PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user) 3174 { 3175 PetscDS prob; 3176 PetscFE *feRef; 3177 PetscFV *fvRef; 3178 Vec fv, cv; 3179 IS fis, cis; 3180 PetscSection fsection, fglobalSection, csection, cglobalSection; 3181 PetscInt *cmap, *cellCIndices, *cellFIndices, *cindices, *findices; 3182 PetscInt cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m; 3183 PetscBool *needAvg; 3184 3185 PetscFunctionBegin; 3186 PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM, dmc, dmf, 0, 0)); 3187 PetscCall(DMGetDimension(dmf, &dim)); 3188 PetscCall(DMGetLocalSection(dmf, &fsection)); 3189 PetscCall(DMGetGlobalSection(dmf, &fglobalSection)); 3190 PetscCall(DMGetLocalSection(dmc, &csection)); 3191 PetscCall(DMGetGlobalSection(dmc, &cglobalSection)); 3192 PetscCall(PetscSectionGetNumFields(fsection, &Nf)); 3193 PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd)); 3194 PetscCall(DMGetDS(dmc, &prob)); 3195 PetscCall(PetscCalloc3(Nf, &feRef, Nf, &fvRef, Nf, &needAvg)); 3196 for (f = 0; f < Nf; ++f) { 3197 PetscObject obj; 3198 PetscClassId id; 3199 PetscInt fNb = 0, Nc = 0; 3200 3201 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 3202 PetscCall(PetscObjectGetClassId(obj, &id)); 3203 if (id == PETSCFE_CLASSID) { 3204 PetscFE fe = (PetscFE)obj; 3205 PetscSpace sp; 3206 PetscInt maxDegree; 3207 3208 PetscCall(PetscFERefine(fe, &feRef[f])); 3209 PetscCall(PetscFEGetDimension(feRef[f], &fNb)); 3210 PetscCall(PetscFEGetNumComponents(fe, &Nc)); 3211 PetscCall(PetscFEGetBasisSpace(fe, &sp)); 3212 PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree)); 3213 if (!maxDegree) needAvg[f] = PETSC_TRUE; 3214 } else if (id == PETSCFV_CLASSID) { 3215 PetscFV fv = (PetscFV)obj; 3216 PetscDualSpace Q; 3217 3218 PetscCall(PetscFVRefine(fv, &fvRef[f])); 3219 PetscCall(PetscFVGetDualSpace(fvRef[f], &Q)); 3220 PetscCall(PetscDualSpaceGetDimension(Q, &fNb)); 3221 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 3222 needAvg[f] = PETSC_TRUE; 3223 } 3224 fTotDim += fNb; 3225 } 3226 PetscCall(PetscDSGetTotalDimension(prob, &cTotDim)); 3227 PetscCall(PetscMalloc1(cTotDim, &cmap)); 3228 for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) { 3229 PetscFE feC; 3230 PetscFV fvC; 3231 PetscDualSpace QF, QC; 3232 PetscInt order = -1, NcF, NcC, fpdim, cpdim; 3233 3234 if (feRef[field]) { 3235 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&feC)); 3236 PetscCall(PetscFEGetNumComponents(feC, &NcC)); 3237 PetscCall(PetscFEGetNumComponents(feRef[field], &NcF)); 3238 PetscCall(PetscFEGetDualSpace(feRef[field], &QF)); 3239 PetscCall(PetscDualSpaceGetOrder(QF, &order)); 3240 PetscCall(PetscDualSpaceGetDimension(QF, &fpdim)); 3241 PetscCall(PetscFEGetDualSpace(feC, &QC)); 3242 PetscCall(PetscDualSpaceGetDimension(QC, &cpdim)); 3243 } else { 3244 PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fvC)); 3245 PetscCall(PetscFVGetNumComponents(fvC, &NcC)); 3246 PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF)); 3247 PetscCall(PetscFVGetDualSpace(fvRef[field], &QF)); 3248 PetscCall(PetscDualSpaceGetDimension(QF, &fpdim)); 3249 PetscCall(PetscFVGetDualSpace(fvC, &QC)); 3250 PetscCall(PetscDualSpaceGetDimension(QC, &cpdim)); 3251 } 3252 PetscCheck(NcF == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC); 3253 for (c = 0; c < cpdim; ++c) { 3254 PetscQuadrature cfunc; 3255 const PetscReal *cqpoints, *cqweights; 3256 PetscInt NqcC, NpC; 3257 PetscBool found = PETSC_FALSE; 3258 3259 PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc)); 3260 PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights)); 3261 PetscCheck(NqcC == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC); 3262 PetscCheck(NpC == 1 || !feRef[field], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments"); 3263 for (f = 0; f < fpdim; ++f) { 3264 PetscQuadrature ffunc; 3265 const PetscReal *fqpoints, *fqweights; 3266 PetscReal sum = 0.0; 3267 PetscInt NqcF, NpF; 3268 3269 PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc)); 3270 PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights)); 3271 PetscCheck(NqcF == NcF, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF); 3272 if (NpC != NpF) continue; 3273 for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]); 3274 if (sum > 1.0e-9) continue; 3275 for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d] * fqweights[d]); 3276 if (sum < 1.0e-9) continue; 3277 cmap[offsetC + c] = offsetF + f; 3278 found = PETSC_TRUE; 3279 break; 3280 } 3281 if (!found) { 3282 /* TODO We really want the average here, but some asshole put VecScatter in the interface */ 3283 if (fvRef[field] || (feRef[field] && order == 0)) { 3284 cmap[offsetC + c] = offsetF + 0; 3285 } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection"); 3286 } 3287 } 3288 offsetC += cpdim; 3289 offsetF += fpdim; 3290 } 3291 for (f = 0; f < Nf; ++f) { 3292 PetscCall(PetscFEDestroy(&feRef[f])); 3293 PetscCall(PetscFVDestroy(&fvRef[f])); 3294 } 3295 PetscCall(PetscFree3(feRef, fvRef, needAvg)); 3296 3297 PetscCall(DMGetGlobalVector(dmf, &fv)); 3298 PetscCall(DMGetGlobalVector(dmc, &cv)); 3299 PetscCall(VecGetOwnershipRange(cv, &startC, &endC)); 3300 PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m)); 3301 PetscCall(PetscMalloc2(cTotDim, &cellCIndices, fTotDim, &cellFIndices)); 3302 PetscCall(PetscMalloc1(m, &cindices)); 3303 PetscCall(PetscMalloc1(m, &findices)); 3304 for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1; 3305 for (c = cStart; c < cEnd; ++c) { 3306 PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices)); 3307 for (d = 0; d < cTotDim; ++d) { 3308 if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue; 3309 PetscCheck(!(findices[cellCIndices[d] - startC] >= 0) || !(findices[cellCIndices[d] - startC] != cellFIndices[cmap[d]]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, cindices[cellCIndices[d] - startC], findices[cellCIndices[d] - startC], cellFIndices[cmap[d]]); 3310 cindices[cellCIndices[d] - startC] = cellCIndices[d]; 3311 findices[cellCIndices[d] - startC] = cellFIndices[cmap[d]]; 3312 } 3313 } 3314 PetscCall(PetscFree(cmap)); 3315 PetscCall(PetscFree2(cellCIndices, cellFIndices)); 3316 3317 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis)); 3318 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis)); 3319 PetscCall(VecScatterCreate(cv, cis, fv, fis, sc)); 3320 PetscCall(ISDestroy(&cis)); 3321 PetscCall(ISDestroy(&fis)); 3322 PetscCall(DMRestoreGlobalVector(dmf, &fv)); 3323 PetscCall(DMRestoreGlobalVector(dmc, &cv)); 3324 PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM, dmc, dmf, 0, 0)); 3325 PetscFunctionReturn(PETSC_SUCCESS); 3326 } 3327 3328 /*@C 3329 DMPlexGetCellFields - Retrieve the field values values for a chunk of cells 3330 3331 Input Parameters: 3332 + dm - The `DM` 3333 . cellIS - The cells to include 3334 . locX - A local vector with the solution fields 3335 . locX_t - A local vector with solution field time derivatives, or NULL 3336 - locA - A local vector with auxiliary fields, or NULL 3337 3338 Output Parameters: 3339 + u - The field coefficients 3340 . u_t - The fields derivative coefficients 3341 - a - The auxiliary field coefficients 3342 3343 Level: developer 3344 3345 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 3346 @*/ 3347 PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a) 3348 { 3349 DM plex, plexA = NULL; 3350 DMEnclosureType encAux; 3351 PetscSection section, sectionAux; 3352 PetscDS prob; 3353 const PetscInt *cells; 3354 PetscInt cStart, cEnd, numCells, totDim, totDimAux, c; 3355 3356 PetscFunctionBegin; 3357 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 3358 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 3359 if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); 3360 if (locA) PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); 3361 PetscValidPointer(u, 6); 3362 PetscValidPointer(u_t, 7); 3363 PetscValidPointer(a, 8); 3364 PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE)); 3365 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 3366 PetscCall(DMGetLocalSection(dm, §ion)); 3367 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL)); 3368 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 3369 if (locA) { 3370 DM dmAux; 3371 PetscDS probAux; 3372 3373 PetscCall(VecGetDM(locA, &dmAux)); 3374 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 3375 PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE)); 3376 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 3377 PetscCall(DMGetDS(dmAux, &probAux)); 3378 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 3379 } 3380 numCells = cEnd - cStart; 3381 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u)); 3382 if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t)); 3383 else *u_t = NULL; 3384 if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a)); 3385 else *a = NULL; 3386 for (c = cStart; c < cEnd; ++c) { 3387 const PetscInt cell = cells ? cells[c] : c; 3388 const PetscInt cind = c - cStart; 3389 PetscScalar *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a; 3390 PetscInt i; 3391 3392 PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x)); 3393 for (i = 0; i < totDim; ++i) ul[cind * totDim + i] = x[i]; 3394 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x)); 3395 if (locX_t) { 3396 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t)); 3397 for (i = 0; i < totDim; ++i) ul_t[cind * totDim + i] = x_t[i]; 3398 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t)); 3399 } 3400 if (locA) { 3401 PetscInt subcell; 3402 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell)); 3403 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x)); 3404 for (i = 0; i < totDimAux; ++i) al[cind * totDimAux + i] = x[i]; 3405 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x)); 3406 } 3407 } 3408 PetscCall(DMDestroy(&plex)); 3409 if (locA) PetscCall(DMDestroy(&plexA)); 3410 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 3411 PetscFunctionReturn(PETSC_SUCCESS); 3412 } 3413 3414 /*@C 3415 DMPlexRestoreCellFields - Restore the field values values for a chunk of cells 3416 3417 Input Parameters: 3418 + dm - The `DM` 3419 . cellIS - The cells to include 3420 . locX - A local vector with the solution fields 3421 . locX_t - A local vector with solution field time derivatives, or NULL 3422 - locA - A local vector with auxiliary fields, or NULL 3423 3424 Output Parameters: 3425 + u - The field coefficients 3426 . u_t - The fields derivative coefficients 3427 - a - The auxiliary field coefficients 3428 3429 Level: developer 3430 3431 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 3432 @*/ 3433 PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a) 3434 { 3435 PetscFunctionBegin; 3436 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u)); 3437 if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t)); 3438 if (locA) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a)); 3439 PetscFunctionReturn(PETSC_SUCCESS); 3440 } 3441 3442 PetscErrorCode DMPlexGetHybridCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a) 3443 { 3444 DM plex, plexA = NULL; 3445 DMEnclosureType encAux; 3446 PetscSection section, sectionAux; 3447 PetscDS ds, dsIn; 3448 const PetscInt *cells; 3449 PetscInt cStart, cEnd, numCells, c, totDim, totDimAux, Nf, f; 3450 3451 PetscFunctionBegin; 3452 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 3453 PetscValidHeaderSpecific(cellIS, IS_CLASSID, 2); 3454 PetscValidHeaderSpecific(locX, VEC_CLASSID, 3); 3455 if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); } 3456 if (locA) { PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); } 3457 PetscValidPointer(u, 6); 3458 PetscValidPointer(u_t, 7); 3459 PetscValidPointer(a, 8); 3460 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 3461 numCells = cEnd - cStart; 3462 PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE)); 3463 PetscCall(DMGetLocalSection(dm, §ion)); 3464 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn)); 3465 PetscCall(PetscDSGetNumFields(dsIn, &Nf)); 3466 PetscCall(PetscDSGetTotalDimension(dsIn, &totDim)); 3467 if (locA) { 3468 DM dmAux; 3469 PetscDS probAux; 3470 3471 PetscCall(VecGetDM(locA, &dmAux)); 3472 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 3473 PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE)); 3474 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 3475 PetscCall(DMGetDS(dmAux, &probAux)); 3476 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 3477 } 3478 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u)); 3479 if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t)); 3480 else { 3481 *u_t = NULL; 3482 } 3483 if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a)); 3484 else { 3485 *a = NULL; 3486 } 3487 // Loop over cohesive cells 3488 for (c = cStart; c < cEnd; ++c) { 3489 const PetscInt cell = cells ? cells[c] : c; 3490 const PetscInt cind = c - cStart; 3491 PetscScalar *xf = NULL, *xc = NULL, *x = NULL, *x_t = NULL, *ul = &(*u)[cind * totDim]; 3492 const PetscInt *cone, *ornt; 3493 PetscInt Nx = 0, Nxf, s; 3494 3495 PetscCall(DMPlexGetCone(dm, cell, &cone)); 3496 PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt)); 3497 // Put in cohesive unknowns 3498 PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, &Nxf, &xf)); 3499 for (f = 0; f < Nf; ++f) { 3500 PetscInt fdofIn, foff, foffIn; 3501 PetscBool cohesive; 3502 3503 PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive)); 3504 if (!cohesive) continue; 3505 PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn)); 3506 PetscCall(PetscDSGetFieldOffsetCohesive(ds, f, &foff)); 3507 PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn)); 3508 for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + i] = xf[foff + i]; 3509 Nx += fdofIn; 3510 } 3511 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, &Nxf, &xf)); 3512 // Loop over sides of surface 3513 for (s = 0; s < 2; ++s) { 3514 const PetscInt *support; 3515 const PetscInt face = cone[s]; 3516 PetscInt ssize, ncell, Nxc; 3517 3518 // I don't think I need the face to have 0 orientation in the hybrid cell 3519 //PetscCheck(!ornt[s], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", face, cell, ornt[s]); 3520 PetscCall(DMPlexGetSupport(dm, face, &support)); 3521 PetscCall(DMPlexGetSupportSize(dm, face, &ssize)); 3522 if (support[0] == cell) ncell = support[1]; 3523 else if (support[1] == cell) ncell = support[0]; 3524 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell); 3525 // Get closure of both face and cell, stick in cell for normal fields and face for cohesive fields 3526 PetscCall(DMPlexVecGetClosure(plex, section, locX, ncell, &Nxc, &xc)); 3527 for (f = 0; f < Nf; ++f) { 3528 PetscInt fdofIn, foffIn; 3529 PetscBool cohesive; 3530 3531 PetscCall(PetscDSGetCohesive(dsIn, f, &cohesive)); 3532 if (cohesive) continue; 3533 PetscCall(PetscDSGetFieldSize(dsIn, f, &fdofIn)); 3534 PetscCall(PetscDSGetFieldOffsetCohesive(dsIn, f, &foffIn)); 3535 for (PetscInt i = 0; i < fdofIn; ++i) ul[foffIn + s * fdofIn + i] = xc[foffIn + i]; 3536 Nx += fdofIn; 3537 } 3538 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, ncell, &Nxc, &xc)); 3539 } 3540 PetscCheck(Nx == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for cell %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, cell, totDim); 3541 3542 if (locX_t) { 3543 PetscScalar *ul_t = &(*u_t)[cind * totDim]; 3544 3545 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t)); 3546 for (PetscInt i = 0; i < totDim; ++i) ul_t[i] = x_t[i]; 3547 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t)); 3548 } 3549 if (locA) { 3550 PetscScalar *al = &(*a)[cind * totDimAux]; 3551 PetscInt subcell; 3552 3553 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell)); 3554 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, &Nx, &x)); 3555 PetscCheck(Nx == totDimAux, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subcell %" PetscInt_FMT "does not match DS size %" PetscInt_FMT, Nx, subcell, totDimAux); 3556 for (PetscInt i = 0; i < totDimAux; ++i) al[i] = x[i]; 3557 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, &Nx, &x)); 3558 } 3559 } 3560 PetscCall(DMDestroy(&plex)); 3561 PetscCall(DMDestroy(&plexA)); 3562 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 3563 PetscFunctionReturn(PETSC_SUCCESS); 3564 } 3565 3566 /* 3567 DMPlexGetHybridFields - Get the field values for the negative side (s = 0) and positive side (s = 1) of the interface 3568 3569 Input Parameters: 3570 + dm - The full domain DM 3571 . dmX - An array of DM for the field, say an auxiliary DM, indexed by s 3572 . dsX - An array of PetscDS for the field, indexed by s 3573 . cellIS - The interface cells for which we want values 3574 . locX - An array of local vectors with the field values, indexed by s 3575 - useCell - Flag to have values come from neighboring cell rather than endcap face 3576 3577 Output Parameter: 3578 . x - An array of field values, indexed by s 3579 3580 Note: 3581 The arrays in `x` will be allocated using `DMGetWorkArray()`, and must be returned using `DMPlexRestoreHybridFields()`. 3582 3583 Level: advanced 3584 3585 .seealso: `DMPlexRestoreHybridFields()`, `DMGetWorkArray()` 3586 */ 3587 static PetscErrorCode DMPlexGetHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[]) 3588 { 3589 DM plexX[2]; 3590 DMEnclosureType encX[2]; 3591 PetscSection sectionX[2]; 3592 const PetscInt *cells; 3593 PetscInt cStart, cEnd, numCells, c, s, totDimX[2]; 3594 3595 PetscFunctionBegin; 3596 PetscValidPointer(locX, 5); 3597 if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS); 3598 PetscValidPointer(dmX, 2); 3599 PetscValidPointer(dsX, 3); 3600 PetscValidHeaderSpecific(cellIS, IS_CLASSID, 4); 3601 PetscValidPointer(x, 7); 3602 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 3603 numCells = cEnd - cStart; 3604 for (s = 0; s < 2; ++s) { 3605 PetscValidHeaderSpecific(dmX[s], DM_CLASSID, 2); 3606 PetscValidHeaderSpecific(dsX[s], PETSCDS_CLASSID, 3); 3607 PetscValidHeaderSpecific(locX[s], VEC_CLASSID, 5); 3608 PetscCall(DMPlexConvertPlex(dmX[s], &plexX[s], PETSC_FALSE)); 3609 PetscCall(DMGetEnclosureRelation(dmX[s], dm, &encX[s])); 3610 PetscCall(DMGetLocalSection(dmX[s], §ionX[s])); 3611 PetscCall(PetscDSGetTotalDimension(dsX[s], &totDimX[s])); 3612 PetscCall(DMGetWorkArray(dmX[s], numCells * totDimX[s], MPIU_SCALAR, &x[s])); 3613 } 3614 for (c = cStart; c < cEnd; ++c) { 3615 const PetscInt cell = cells ? cells[c] : c; 3616 const PetscInt cind = c - cStart; 3617 const PetscInt *cone, *ornt; 3618 3619 PetscCall(DMPlexGetCone(dm, cell, &cone)); 3620 PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt)); 3621 //PetscCheck(!ornt[0], PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", cone[0], cell, ornt[0]); 3622 for (s = 0; s < 2; ++s) { 3623 const PetscInt tdX = totDimX[s]; 3624 PetscScalar *closure = NULL, *xl = &x[s][cind * tdX]; 3625 PetscInt face = cone[s], point = face, subpoint, Nx, i; 3626 3627 if (useCell) { 3628 const PetscInt *support; 3629 PetscInt ssize; 3630 3631 PetscCall(DMPlexGetSupport(dm, face, &support)); 3632 PetscCall(DMPlexGetSupportSize(dm, face, &ssize)); 3633 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", face, cell, ssize); 3634 if (support[0] == cell) point = support[1]; 3635 else if (support[1] == cell) point = support[0]; 3636 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", face, cell); 3637 } 3638 PetscCall(DMGetEnclosurePoint(plexX[s], dm, encX[s], point, &subpoint)); 3639 PetscCall(DMPlexVecGetOrientedClosure_Internal(plexX[s], sectionX[s], locX[s], subpoint, ornt[s], &Nx, &closure)); 3640 PetscCheck(Nx == tdX, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Closure size %" PetscInt_FMT " for subpoint %" PetscInt_FMT " does not match DS size %" PetscInt_FMT, Nx, subpoint, tdX); 3641 for (i = 0; i < Nx; ++i) xl[i] = closure[i]; 3642 PetscCall(DMPlexVecRestoreClosure(plexX[s], sectionX[s], locX[s], subpoint, &Nx, &closure)); 3643 } 3644 } 3645 for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexX[s])); 3646 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 3647 PetscFunctionReturn(PETSC_SUCCESS); 3648 } 3649 3650 static PetscErrorCode DMPlexRestoreHybridFields(DM dm, DM dmX[], PetscDS dsX[], IS cellIS, Vec locX[], PetscBool useCell, PetscScalar *x[]) 3651 { 3652 PetscFunctionBegin; 3653 if (!locX[0] || !locX[1]) PetscFunctionReturn(PETSC_SUCCESS); 3654 PetscCall(DMRestoreWorkArray(dmX[0], 0, MPIU_SCALAR, &x[0])); 3655 PetscCall(DMRestoreWorkArray(dmX[1], 0, MPIU_SCALAR, &x[1])); 3656 PetscFunctionReturn(PETSC_SUCCESS); 3657 } 3658 3659 /*@C 3660 DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces 3661 3662 Input Parameters: 3663 + dm - The `DM` 3664 . fStart - The first face to include 3665 . fEnd - The first face to exclude 3666 . locX - A local vector with the solution fields 3667 . locX_t - A local vector with solution field time derivatives, or NULL 3668 . faceGeometry - A local vector with face geometry 3669 . cellGeometry - A local vector with cell geometry 3670 - locaGrad - A local vector with field gradients, or NULL 3671 3672 Output Parameters: 3673 + Nface - The number of faces with field values 3674 . uL - The field values at the left side of the face 3675 - uR - The field values at the right side of the face 3676 3677 Level: developer 3678 3679 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()` 3680 @*/ 3681 PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR) 3682 { 3683 DM dmFace, dmCell, dmGrad = NULL; 3684 PetscSection section; 3685 PetscDS prob; 3686 DMLabel ghostLabel; 3687 const PetscScalar *facegeom, *cellgeom, *x, *lgrad; 3688 PetscBool *isFE; 3689 PetscInt dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face; 3690 3691 PetscFunctionBegin; 3692 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 3693 PetscValidHeaderSpecific(locX, VEC_CLASSID, 4); 3694 if (locX_t) PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5); 3695 PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6); 3696 PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7); 3697 if (locGrad) PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8); 3698 PetscValidPointer(uL, 10); 3699 PetscValidPointer(uR, 11); 3700 PetscCall(DMGetDimension(dm, &dim)); 3701 PetscCall(DMGetDS(dm, &prob)); 3702 PetscCall(DMGetLocalSection(dm, §ion)); 3703 PetscCall(PetscDSGetNumFields(prob, &Nf)); 3704 PetscCall(PetscDSGetTotalComponents(prob, &Nc)); 3705 PetscCall(PetscMalloc1(Nf, &isFE)); 3706 for (f = 0; f < Nf; ++f) { 3707 PetscObject obj; 3708 PetscClassId id; 3709 3710 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 3711 PetscCall(PetscObjectGetClassId(obj, &id)); 3712 if (id == PETSCFE_CLASSID) { 3713 isFE[f] = PETSC_TRUE; 3714 } else if (id == PETSCFV_CLASSID) { 3715 isFE[f] = PETSC_FALSE; 3716 } else { 3717 isFE[f] = PETSC_FALSE; 3718 } 3719 } 3720 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 3721 PetscCall(VecGetArrayRead(locX, &x)); 3722 PetscCall(VecGetDM(faceGeometry, &dmFace)); 3723 PetscCall(VecGetArrayRead(faceGeometry, &facegeom)); 3724 PetscCall(VecGetDM(cellGeometry, &dmCell)); 3725 PetscCall(VecGetArrayRead(cellGeometry, &cellgeom)); 3726 if (locGrad) { 3727 PetscCall(VecGetDM(locGrad, &dmGrad)); 3728 PetscCall(VecGetArrayRead(locGrad, &lgrad)); 3729 } 3730 PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uL)); 3731 PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uR)); 3732 /* Right now just eat the extra work for FE (could make a cell loop) */ 3733 for (face = fStart, iface = 0; face < fEnd; ++face) { 3734 const PetscInt *cells; 3735 PetscFVFaceGeom *fg; 3736 PetscFVCellGeom *cgL, *cgR; 3737 PetscScalar *xL, *xR, *gL, *gR; 3738 PetscScalar *uLl = *uL, *uRl = *uR; 3739 PetscInt ghost, nsupp, nchild; 3740 3741 PetscCall(DMLabelGetValue(ghostLabel, face, &ghost)); 3742 PetscCall(DMPlexGetSupportSize(dm, face, &nsupp)); 3743 PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL)); 3744 if (ghost >= 0 || nsupp > 2 || nchild > 0) continue; 3745 PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg)); 3746 PetscCall(DMPlexGetSupport(dm, face, &cells)); 3747 PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL)); 3748 PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR)); 3749 for (f = 0; f < Nf; ++f) { 3750 PetscInt off; 3751 3752 PetscCall(PetscDSGetComponentOffset(prob, f, &off)); 3753 if (isFE[f]) { 3754 const PetscInt *cone; 3755 PetscInt comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d; 3756 3757 xL = xR = NULL; 3758 PetscCall(PetscSectionGetFieldComponents(section, f, &comp)); 3759 PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **)&xL)); 3760 PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **)&xR)); 3761 PetscCall(DMPlexGetCone(dm, cells[0], &cone)); 3762 PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL)); 3763 for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL) 3764 if (cone[faceLocL] == face) break; 3765 PetscCall(DMPlexGetCone(dm, cells[1], &cone)); 3766 PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR)); 3767 for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR) 3768 if (cone[faceLocR] == face) break; 3769 PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]); 3770 /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */ 3771 /* TODO: this is a hack that might not be right for nonconforming */ 3772 if (faceLocL < coneSizeL) { 3773 PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface * Nc + off])); 3774 if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off])); 3775 else { 3776 for (d = 0; d < comp; ++d) uRl[iface * Nc + off + d] = uLl[iface * Nc + off + d]; 3777 } 3778 } else { 3779 PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off])); 3780 PetscCall(PetscSectionGetFieldComponents(section, f, &comp)); 3781 for (d = 0; d < comp; ++d) uLl[iface * Nc + off + d] = uRl[iface * Nc + off + d]; 3782 } 3783 PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **)&xL)); 3784 PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **)&xR)); 3785 } else { 3786 PetscFV fv; 3787 PetscInt numComp, c; 3788 3789 PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *)&fv)); 3790 PetscCall(PetscFVGetNumComponents(fv, &numComp)); 3791 PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL)); 3792 PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR)); 3793 if (dmGrad) { 3794 PetscReal dxL[3], dxR[3]; 3795 3796 PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL)); 3797 PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR)); 3798 DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL); 3799 DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR); 3800 for (c = 0; c < numComp; ++c) { 3801 uLl[iface * Nc + off + c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c * dim], dxL); 3802 uRl[iface * Nc + off + c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c * dim], dxR); 3803 } 3804 } else { 3805 for (c = 0; c < numComp; ++c) { 3806 uLl[iface * Nc + off + c] = xL[c]; 3807 uRl[iface * Nc + off + c] = xR[c]; 3808 } 3809 } 3810 } 3811 } 3812 ++iface; 3813 } 3814 *Nface = iface; 3815 PetscCall(VecRestoreArrayRead(locX, &x)); 3816 PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom)); 3817 PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom)); 3818 if (locGrad) PetscCall(VecRestoreArrayRead(locGrad, &lgrad)); 3819 PetscCall(PetscFree(isFE)); 3820 PetscFunctionReturn(PETSC_SUCCESS); 3821 } 3822 3823 /*@C 3824 DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces 3825 3826 Input Parameters: 3827 + dm - The `DM` 3828 . fStart - The first face to include 3829 . fEnd - The first face to exclude 3830 . locX - A local vector with the solution fields 3831 . locX_t - A local vector with solution field time derivatives, or NULL 3832 . faceGeometry - A local vector with face geometry 3833 . cellGeometry - A local vector with cell geometry 3834 - locaGrad - A local vector with field gradients, or NULL 3835 3836 Output Parameters: 3837 + Nface - The number of faces with field values 3838 . uL - The field values at the left side of the face 3839 - uR - The field values at the right side of the face 3840 3841 Level: developer 3842 3843 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 3844 @*/ 3845 PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR) 3846 { 3847 PetscFunctionBegin; 3848 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL)); 3849 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR)); 3850 PetscFunctionReturn(PETSC_SUCCESS); 3851 } 3852 3853 /*@C 3854 DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces 3855 3856 Input Parameters: 3857 + dm - The `DM` 3858 . fStart - The first face to include 3859 . fEnd - The first face to exclude 3860 . faceGeometry - A local vector with face geometry 3861 - cellGeometry - A local vector with cell geometry 3862 3863 Output Parameters: 3864 + Nface - The number of faces with field values 3865 . fgeom - The extract the face centroid and normal 3866 - vol - The cell volume 3867 3868 Level: developer 3869 3870 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellFields()` 3871 @*/ 3872 PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol) 3873 { 3874 DM dmFace, dmCell; 3875 DMLabel ghostLabel; 3876 const PetscScalar *facegeom, *cellgeom; 3877 PetscInt dim, numFaces = fEnd - fStart, iface, face; 3878 3879 PetscFunctionBegin; 3880 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 3881 PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4); 3882 PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5); 3883 PetscValidPointer(fgeom, 7); 3884 PetscValidPointer(vol, 8); 3885 PetscCall(DMGetDimension(dm, &dim)); 3886 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 3887 PetscCall(VecGetDM(faceGeometry, &dmFace)); 3888 PetscCall(VecGetArrayRead(faceGeometry, &facegeom)); 3889 PetscCall(VecGetDM(cellGeometry, &dmCell)); 3890 PetscCall(VecGetArrayRead(cellGeometry, &cellgeom)); 3891 PetscCall(PetscMalloc1(numFaces, fgeom)); 3892 PetscCall(DMGetWorkArray(dm, numFaces * 2, MPIU_SCALAR, vol)); 3893 for (face = fStart, iface = 0; face < fEnd; ++face) { 3894 const PetscInt *cells; 3895 PetscFVFaceGeom *fg; 3896 PetscFVCellGeom *cgL, *cgR; 3897 PetscFVFaceGeom *fgeoml = *fgeom; 3898 PetscReal *voll = *vol; 3899 PetscInt ghost, d, nchild, nsupp; 3900 3901 PetscCall(DMLabelGetValue(ghostLabel, face, &ghost)); 3902 PetscCall(DMPlexGetSupportSize(dm, face, &nsupp)); 3903 PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL)); 3904 if (ghost >= 0 || nsupp > 2 || nchild > 0) continue; 3905 PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg)); 3906 PetscCall(DMPlexGetSupport(dm, face, &cells)); 3907 PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL)); 3908 PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR)); 3909 for (d = 0; d < dim; ++d) { 3910 fgeoml[iface].centroid[d] = fg->centroid[d]; 3911 fgeoml[iface].normal[d] = fg->normal[d]; 3912 } 3913 voll[iface * 2 + 0] = cgL->volume; 3914 voll[iface * 2 + 1] = cgR->volume; 3915 ++iface; 3916 } 3917 *Nface = iface; 3918 PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom)); 3919 PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom)); 3920 PetscFunctionReturn(PETSC_SUCCESS); 3921 } 3922 3923 /*@C 3924 DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces 3925 3926 Input Parameters: 3927 + dm - The `DM` 3928 . fStart - The first face to include 3929 . fEnd - The first face to exclude 3930 . faceGeometry - A local vector with face geometry 3931 - cellGeometry - A local vector with cell geometry 3932 3933 Output Parameters: 3934 + Nface - The number of faces with field values 3935 . fgeom - The extract the face centroid and normal 3936 - vol - The cell volume 3937 3938 Level: developer 3939 3940 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetFaceFields()` 3941 @*/ 3942 PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol) 3943 { 3944 PetscFunctionBegin; 3945 PetscCall(PetscFree(*fgeom)); 3946 PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol)); 3947 PetscFunctionReturn(PETSC_SUCCESS); 3948 } 3949 3950 PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) 3951 { 3952 char composeStr[33] = {0}; 3953 PetscObjectId id; 3954 PetscContainer container; 3955 3956 PetscFunctionBegin; 3957 PetscCall(PetscObjectGetId((PetscObject)quad, &id)); 3958 PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id)); 3959 PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container)); 3960 if (container) { 3961 PetscCall(PetscContainerGetPointer(container, (void **)geom)); 3962 } else { 3963 PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom)); 3964 PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container)); 3965 PetscCall(PetscContainerSetPointer(container, (void *)*geom)); 3966 PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom)); 3967 PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container)); 3968 PetscCall(PetscContainerDestroy(&container)); 3969 } 3970 PetscFunctionReturn(PETSC_SUCCESS); 3971 } 3972 3973 PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) 3974 { 3975 PetscFunctionBegin; 3976 *geom = NULL; 3977 PetscFunctionReturn(PETSC_SUCCESS); 3978 } 3979 3980 PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user) 3981 { 3982 DM_Plex *mesh = (DM_Plex *)dm->data; 3983 const char *name = "Residual"; 3984 DM dmAux = NULL; 3985 DMLabel ghostLabel = NULL; 3986 PetscDS prob = NULL; 3987 PetscDS probAux = NULL; 3988 PetscBool useFEM = PETSC_FALSE; 3989 PetscBool isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE; 3990 DMField coordField = NULL; 3991 Vec locA; 3992 PetscScalar *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL; 3993 IS chunkIS; 3994 const PetscInt *cells; 3995 PetscInt cStart, cEnd, numCells; 3996 PetscInt Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd; 3997 PetscInt maxDegree = PETSC_MAX_INT; 3998 PetscFormKey key; 3999 PetscQuadrature affineQuad = NULL, *quads = NULL; 4000 PetscFEGeom *affineGeom = NULL, **geoms = NULL; 4001 4002 PetscFunctionBegin; 4003 PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 4004 /* FEM+FVM */ 4005 /* 1: Get sizes from dm and dmAux */ 4006 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 4007 PetscCall(DMGetDS(dm, &prob)); 4008 PetscCall(PetscDSGetNumFields(prob, &Nf)); 4009 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 4010 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA)); 4011 if (locA) { 4012 PetscCall(VecGetDM(locA, &dmAux)); 4013 PetscCall(DMGetDS(dmAux, &probAux)); 4014 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 4015 } 4016 /* 2: Get geometric data */ 4017 for (f = 0; f < Nf; ++f) { 4018 PetscObject obj; 4019 PetscClassId id; 4020 PetscBool fimp; 4021 4022 PetscCall(PetscDSGetImplicit(prob, f, &fimp)); 4023 if (isImplicit != fimp) continue; 4024 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4025 PetscCall(PetscObjectGetClassId(obj, &id)); 4026 if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE; 4027 PetscCheck(id != PETSCFV_CLASSID, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Use of FVM with PCPATCH not yet implemented"); 4028 } 4029 if (useFEM) { 4030 PetscCall(DMGetCoordinateField(dm, &coordField)); 4031 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 4032 if (maxDegree <= 1) { 4033 PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad)); 4034 if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 4035 } else { 4036 PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms)); 4037 for (f = 0; f < Nf; ++f) { 4038 PetscObject obj; 4039 PetscClassId id; 4040 PetscBool fimp; 4041 4042 PetscCall(PetscDSGetImplicit(prob, f, &fimp)); 4043 if (isImplicit != fimp) continue; 4044 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4045 PetscCall(PetscObjectGetClassId(obj, &id)); 4046 if (id == PETSCFE_CLASSID) { 4047 PetscFE fe = (PetscFE)obj; 4048 4049 PetscCall(PetscFEGetQuadrature(fe, &quads[f])); 4050 PetscCall(PetscObjectReference((PetscObject)quads[f])); 4051 PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 4052 } 4053 } 4054 } 4055 } 4056 /* Loop over chunks */ 4057 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 4058 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 4059 if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS)); 4060 numCells = cEnd - cStart; 4061 numChunks = 1; 4062 cellChunkSize = numCells / numChunks; 4063 numChunks = PetscMin(1, numCells); 4064 key.label = NULL; 4065 key.value = 0; 4066 key.part = 0; 4067 for (chunk = 0; chunk < numChunks; ++chunk) { 4068 PetscScalar *elemVec, *fluxL = NULL, *fluxR = NULL; 4069 PetscReal *vol = NULL; 4070 PetscFVFaceGeom *fgeom = NULL; 4071 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 4072 PetscInt numFaces = 0; 4073 4074 /* Extract field coefficients */ 4075 if (useFEM) { 4076 PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells)); 4077 PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 4078 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 4079 PetscCall(PetscArrayzero(elemVec, numCells * totDim)); 4080 } 4081 /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */ 4082 /* Loop over fields */ 4083 for (f = 0; f < Nf; ++f) { 4084 PetscObject obj; 4085 PetscClassId id; 4086 PetscBool fimp; 4087 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset; 4088 4089 key.field = f; 4090 PetscCall(PetscDSGetImplicit(prob, f, &fimp)); 4091 if (isImplicit != fimp) continue; 4092 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4093 PetscCall(PetscObjectGetClassId(obj, &id)); 4094 if (id == PETSCFE_CLASSID) { 4095 PetscFE fe = (PetscFE)obj; 4096 PetscFEGeom *geom = affineGeom ? affineGeom : geoms[f]; 4097 PetscFEGeom *chunkGeom = NULL; 4098 PetscQuadrature quad = affineQuad ? affineQuad : quads[f]; 4099 PetscInt Nq, Nb; 4100 4101 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4102 PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL)); 4103 PetscCall(PetscFEGetDimension(fe, &Nb)); 4104 blockSize = Nb; 4105 batchSize = numBlocks * blockSize; 4106 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4107 numChunks = numCells / (numBatches * batchSize); 4108 Ne = numChunks * numBatches * batchSize; 4109 Nr = numCells % (numBatches * batchSize); 4110 offset = numCells - Nr; 4111 /* Integrate FE residual to get elemVec (need fields at quadrature points) */ 4112 /* For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */ 4113 PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom)); 4114 PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec)); 4115 PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom)); 4116 PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, &elemVec[offset * totDim])); 4117 PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom)); 4118 } else if (id == PETSCFV_CLASSID) { 4119 PetscFV fv = (PetscFV)obj; 4120 4121 Ne = numFaces; 4122 /* Riemann solve over faces (need fields at face centroids) */ 4123 /* We need to evaluate FE fields at those coordinates */ 4124 PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR)); 4125 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 4126 } 4127 /* Loop over domain */ 4128 if (useFEM) { 4129 /* Add elemVec to locX */ 4130 for (c = cS; c < cE; ++c) { 4131 const PetscInt cell = cells ? cells[c] : c; 4132 const PetscInt cind = c - cStart; 4133 4134 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim])); 4135 if (ghostLabel) { 4136 PetscInt ghostVal; 4137 4138 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 4139 if (ghostVal > 0) continue; 4140 } 4141 PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES)); 4142 } 4143 } 4144 /* Handle time derivative */ 4145 if (locX_t) { 4146 PetscScalar *x_t, *fa; 4147 4148 PetscCall(VecGetArray(locF, &fa)); 4149 PetscCall(VecGetArray(locX_t, &x_t)); 4150 for (f = 0; f < Nf; ++f) { 4151 PetscFV fv; 4152 PetscObject obj; 4153 PetscClassId id; 4154 PetscInt pdim, d; 4155 4156 PetscCall(PetscDSGetDiscretization(prob, f, &obj)); 4157 PetscCall(PetscObjectGetClassId(obj, &id)); 4158 if (id != PETSCFV_CLASSID) continue; 4159 fv = (PetscFV)obj; 4160 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 4161 for (c = cS; c < cE; ++c) { 4162 const PetscInt cell = cells ? cells[c] : c; 4163 PetscScalar *u_t, *r; 4164 4165 if (ghostLabel) { 4166 PetscInt ghostVal; 4167 4168 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 4169 if (ghostVal > 0) continue; 4170 } 4171 PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t)); 4172 PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r)); 4173 for (d = 0; d < pdim; ++d) r[d] += u_t[d]; 4174 } 4175 } 4176 PetscCall(VecRestoreArray(locX_t, &x_t)); 4177 PetscCall(VecRestoreArray(locF, &fa)); 4178 } 4179 if (useFEM) { 4180 PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 4181 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 4182 } 4183 } 4184 if (useFEM) PetscCall(ISDestroy(&chunkIS)); 4185 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 4186 /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */ 4187 if (useFEM) { 4188 if (maxDegree <= 1) { 4189 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 4190 PetscCall(PetscQuadratureDestroy(&affineQuad)); 4191 } else { 4192 for (f = 0; f < Nf; ++f) { 4193 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 4194 PetscCall(PetscQuadratureDestroy(&quads[f])); 4195 } 4196 PetscCall(PetscFree2(quads, geoms)); 4197 } 4198 } 4199 PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 4200 PetscFunctionReturn(PETSC_SUCCESS); 4201 } 4202 4203 /* 4204 We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac 4205 4206 X - The local solution vector 4207 X_t - The local solution time derivative vector, or NULL 4208 */ 4209 PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx) 4210 { 4211 DM_Plex *mesh = (DM_Plex *)dm->data; 4212 const char *name = "Jacobian", *nameP = "JacobianPre"; 4213 DM dmAux = NULL; 4214 PetscDS prob, probAux = NULL; 4215 PetscSection sectionAux = NULL; 4216 Vec A; 4217 DMField coordField; 4218 PetscFEGeom *cgeomFEM; 4219 PetscQuadrature qGeom = NULL; 4220 Mat J = Jac, JP = JacP; 4221 PetscScalar *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL; 4222 PetscBool hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE; 4223 const PetscInt *cells; 4224 PetscFormKey key; 4225 PetscInt Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0; 4226 4227 PetscFunctionBegin; 4228 PetscCall(ISGetLocalSize(cellIS, &numCells)); 4229 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 4230 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 4231 PetscCall(DMGetDS(dm, &prob)); 4232 PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A)); 4233 if (A) { 4234 PetscCall(VecGetDM(A, &dmAux)); 4235 PetscCall(DMGetLocalSection(dmAux, §ionAux)); 4236 PetscCall(DMGetDS(dmAux, &probAux)); 4237 } 4238 /* Get flags */ 4239 PetscCall(PetscDSGetNumFields(prob, &Nf)); 4240 PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE)); 4241 for (fieldI = 0; fieldI < Nf; ++fieldI) { 4242 PetscObject disc; 4243 PetscClassId id; 4244 PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc)); 4245 PetscCall(PetscObjectGetClassId(disc, &id)); 4246 if (id == PETSCFE_CLASSID) { 4247 isFE[fieldI] = PETSC_TRUE; 4248 } else if (id == PETSCFV_CLASSID) { 4249 hasFV = PETSC_TRUE; 4250 isFE[fieldI] = PETSC_FALSE; 4251 } 4252 } 4253 PetscCall(PetscDSHasJacobian(prob, &hasJac)); 4254 PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec)); 4255 PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn)); 4256 assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE; 4257 hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE; 4258 if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */ 4259 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 4260 if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 4261 /* Compute batch sizes */ 4262 if (isFE[0]) { 4263 PetscFE fe; 4264 PetscQuadrature q; 4265 PetscInt numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb; 4266 4267 PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe)); 4268 PetscCall(PetscFEGetQuadrature(fe, &q)); 4269 PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL)); 4270 PetscCall(PetscFEGetDimension(fe, &Nb)); 4271 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4272 blockSize = Nb * numQuadPoints; 4273 batchSize = numBlocks * blockSize; 4274 chunkSize = numBatches * batchSize; 4275 numChunks = numCells / chunkSize + numCells % chunkSize; 4276 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4277 } else { 4278 chunkSize = numCells; 4279 numChunks = 1; 4280 } 4281 /* Get work space */ 4282 wsz = (((X ? 1 : 0) + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize; 4283 PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work)); 4284 PetscCall(PetscArrayzero(work, wsz)); 4285 off = 0; 4286 u = X ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL; 4287 u_t = X_t ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL; 4288 a = dmAux ? (sz = chunkSize * totDimAux, off += sz, work + off - sz) : NULL; 4289 elemMat = hasJac ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL; 4290 elemMatP = hasPrec ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL; 4291 elemMatD = hasDyn ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL; 4292 PetscCheck(off == wsz, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz); 4293 /* Setup geometry */ 4294 PetscCall(DMGetCoordinateField(dm, &coordField)); 4295 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 4296 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom)); 4297 if (!qGeom) { 4298 PetscFE fe; 4299 4300 PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe)); 4301 PetscCall(PetscFEGetQuadrature(fe, &qGeom)); 4302 PetscCall(PetscObjectReference((PetscObject)qGeom)); 4303 } 4304 PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 4305 /* Compute volume integrals */ 4306 if (assembleJac) PetscCall(MatZeroEntries(J)); 4307 PetscCall(MatZeroEntries(JP)); 4308 key.label = NULL; 4309 key.value = 0; 4310 key.part = 0; 4311 for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) { 4312 const PetscInt Ncell = PetscMin(chunkSize, numCells - offCell); 4313 PetscInt c; 4314 4315 /* Extract values */ 4316 for (c = 0; c < Ncell; ++c) { 4317 const PetscInt cell = cells ? cells[c + offCell] : c + offCell; 4318 PetscScalar *x = NULL, *x_t = NULL; 4319 PetscInt i; 4320 4321 if (X) { 4322 PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x)); 4323 for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i]; 4324 PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x)); 4325 } 4326 if (X_t) { 4327 PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t)); 4328 for (i = 0; i < totDim; ++i) u_t[c * totDim + i] = x_t[i]; 4329 PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t)); 4330 } 4331 if (dmAux) { 4332 PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x)); 4333 for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i]; 4334 PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x)); 4335 } 4336 } 4337 for (fieldI = 0; fieldI < Nf; ++fieldI) { 4338 PetscFE fe; 4339 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 4340 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 4341 key.field = fieldI * Nf + fieldJ; 4342 if (hasJac) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat)); 4343 if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP)); 4344 if (hasDyn) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD)); 4345 } 4346 /* For finite volume, add the identity */ 4347 if (!isFE[fieldI]) { 4348 PetscFV fv; 4349 PetscInt eOffset = 0, Nc, fc, foff; 4350 4351 PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff)); 4352 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv)); 4353 PetscCall(PetscFVGetNumComponents(fv, &Nc)); 4354 for (c = 0; c < chunkSize; ++c, eOffset += totDim * totDim) { 4355 for (fc = 0; fc < Nc; ++fc) { 4356 const PetscInt i = foff + fc; 4357 if (hasJac) elemMat[eOffset + i * totDim + i] = 1.0; 4358 if (hasPrec) elemMatP[eOffset + i * totDim + i] = 1.0; 4359 } 4360 } 4361 } 4362 } 4363 /* Add contribution from X_t */ 4364 if (hasDyn) { 4365 for (c = 0; c < chunkSize * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c]; 4366 } 4367 /* Insert values into matrix */ 4368 for (c = 0; c < Ncell; ++c) { 4369 const PetscInt cell = cells ? cells[c + offCell] : c + offCell; 4370 if (mesh->printFEM > 1) { 4371 if (hasJac) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[(c - cStart) * totDim * totDim])); 4372 if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c - cStart) * totDim * totDim])); 4373 } 4374 if (assembleJac) PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES)); 4375 PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JP, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES)); 4376 } 4377 } 4378 /* Cleanup */ 4379 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 4380 PetscCall(PetscQuadratureDestroy(&qGeom)); 4381 if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE)); 4382 PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE)); 4383 PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize, MPIU_SCALAR, &work)); 4384 /* Compute boundary integrals */ 4385 /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */ 4386 /* Assemble matrix */ 4387 if (assembleJac) { 4388 PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY)); 4389 PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY)); 4390 } 4391 PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY)); 4392 PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY)); 4393 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 4394 PetscFunctionReturn(PETSC_SUCCESS); 4395 } 4396 4397 /******** FEM Assembly Function ********/ 4398 4399 static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy) 4400 { 4401 PetscBool isPlex; 4402 4403 PetscFunctionBegin; 4404 PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex)); 4405 if (isPlex) { 4406 *plex = dm; 4407 PetscCall(PetscObjectReference((PetscObject)dm)); 4408 } else { 4409 PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex)); 4410 if (!*plex) { 4411 PetscCall(DMConvert(dm, DMPLEX, plex)); 4412 PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex)); 4413 if (copy) PetscCall(DMCopyAuxiliaryVec(dm, *plex)); 4414 } else { 4415 PetscCall(PetscObjectReference((PetscObject)*plex)); 4416 } 4417 } 4418 PetscFunctionReturn(PETSC_SUCCESS); 4419 } 4420 4421 /*@ 4422 DMPlexGetGeometryFVM - Return precomputed geometric data 4423 4424 Collective 4425 4426 Input Parameter: 4427 . dm - The `DM` 4428 4429 Output Parameters: 4430 + facegeom - The values precomputed from face geometry 4431 . cellgeom - The values precomputed from cell geometry 4432 - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell 4433 4434 Level: developer 4435 4436 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMTSSetRHSFunctionLocal()` 4437 @*/ 4438 PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius) 4439 { 4440 DM plex; 4441 4442 PetscFunctionBegin; 4443 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4444 PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE)); 4445 PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL)); 4446 if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius)); 4447 PetscCall(DMDestroy(&plex)); 4448 PetscFunctionReturn(PETSC_SUCCESS); 4449 } 4450 4451 /*@ 4452 DMPlexGetGradientDM - Return gradient data layout 4453 4454 Collective 4455 4456 Input Parameters: 4457 + dm - The `DM` 4458 - fv - The `PetscFV` 4459 4460 Output Parameter: 4461 . dmGrad - The layout for gradient values 4462 4463 Level: developer 4464 4465 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetGeometryFVM()` 4466 @*/ 4467 PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad) 4468 { 4469 DM plex; 4470 PetscBool computeGradients; 4471 4472 PetscFunctionBegin; 4473 PetscValidHeaderSpecific(dm, DM_CLASSID, 1); 4474 PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2); 4475 PetscValidPointer(dmGrad, 3); 4476 PetscCall(PetscFVGetComputeGradients(fv, &computeGradients)); 4477 if (!computeGradients) { 4478 *dmGrad = NULL; 4479 PetscFunctionReturn(PETSC_SUCCESS); 4480 } 4481 PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE)); 4482 PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad)); 4483 PetscCall(DMDestroy(&plex)); 4484 PetscFunctionReturn(PETSC_SUCCESS); 4485 } 4486 4487 static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS) 4488 { 4489 DM_Plex *mesh = (DM_Plex *)dm->data; 4490 DM plex = NULL, plexA = NULL; 4491 DMEnclosureType encAux; 4492 PetscDS prob, probAux = NULL; 4493 PetscSection section, sectionAux = NULL; 4494 Vec locA = NULL; 4495 PetscScalar *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL; 4496 PetscInt totDim, totDimAux = 0; 4497 4498 PetscFunctionBegin; 4499 PetscCall(DMConvert(dm, DMPLEX, &plex)); 4500 PetscCall(DMGetLocalSection(dm, §ion)); 4501 PetscCall(DMGetDS(dm, &prob)); 4502 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 4503 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA)); 4504 if (locA) { 4505 DM dmAux; 4506 4507 PetscCall(VecGetDM(locA, &dmAux)); 4508 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 4509 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 4510 PetscCall(DMGetDS(plexA, &probAux)); 4511 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 4512 PetscCall(DMGetLocalSection(plexA, §ionAux)); 4513 } 4514 { 4515 PetscFEGeom *fgeom; 4516 PetscInt maxDegree; 4517 PetscQuadrature qGeom = NULL; 4518 IS pointIS; 4519 const PetscInt *points; 4520 PetscInt numFaces, face, Nq; 4521 4522 PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS)); 4523 if (!pointIS) goto end; /* No points with that id on this process */ 4524 { 4525 IS isectIS; 4526 4527 /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */ 4528 PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS)); 4529 PetscCall(ISDestroy(&pointIS)); 4530 pointIS = isectIS; 4531 } 4532 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 4533 PetscCall(ISGetIndices(pointIS, &points)); 4534 PetscCall(PetscMalloc4(numFaces * totDim, &u, locX_t ? numFaces * totDim : 0, &u_t, numFaces * totDim, &elemVec, locA ? numFaces * totDimAux : 0, &a)); 4535 PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree)); 4536 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom)); 4537 if (!qGeom) { 4538 PetscFE fe; 4539 4540 PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe)); 4541 PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom)); 4542 PetscCall(PetscObjectReference((PetscObject)qGeom)); 4543 } 4544 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 4545 PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 4546 for (face = 0; face < numFaces; ++face) { 4547 const PetscInt point = points[face], *support; 4548 PetscScalar *x = NULL; 4549 PetscInt i; 4550 4551 PetscCall(DMPlexGetSupport(dm, point, &support)); 4552 PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x)); 4553 for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i]; 4554 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x)); 4555 if (locX_t) { 4556 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x)); 4557 for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i]; 4558 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x)); 4559 } 4560 if (locA) { 4561 PetscInt subp; 4562 4563 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp)); 4564 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x)); 4565 for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i]; 4566 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x)); 4567 } 4568 } 4569 PetscCall(PetscArrayzero(elemVec, numFaces * totDim)); 4570 { 4571 PetscFE fe; 4572 PetscInt Nb; 4573 PetscFEGeom *chunkGeom = NULL; 4574 /* Conforming batches */ 4575 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 4576 /* Remainder */ 4577 PetscInt Nr, offset; 4578 4579 PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe)); 4580 PetscCall(PetscFEGetDimension(fe, &Nb)); 4581 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4582 /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */ 4583 blockSize = Nb; 4584 batchSize = numBlocks * blockSize; 4585 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4586 numChunks = numFaces / (numBatches * batchSize); 4587 Ne = numChunks * numBatches * batchSize; 4588 Nr = numFaces % (numBatches * batchSize); 4589 offset = numFaces - Nr; 4590 PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom)); 4591 PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec)); 4592 PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom)); 4593 PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom)); 4594 PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, a ? &a[offset * totDimAux] : NULL, t, &elemVec[offset * totDim])); 4595 PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom)); 4596 } 4597 for (face = 0; face < numFaces; ++face) { 4598 const PetscInt point = points[face], *support; 4599 4600 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, "BdResidual", totDim, &elemVec[face * totDim])); 4601 PetscCall(DMPlexGetSupport(plex, point, &support)); 4602 PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face * totDim], ADD_ALL_VALUES)); 4603 } 4604 PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 4605 PetscCall(PetscQuadratureDestroy(&qGeom)); 4606 PetscCall(ISRestoreIndices(pointIS, &points)); 4607 PetscCall(ISDestroy(&pointIS)); 4608 PetscCall(PetscFree4(u, u_t, elemVec, a)); 4609 } 4610 end: 4611 PetscCall(DMDestroy(&plex)); 4612 PetscCall(DMDestroy(&plexA)); 4613 PetscFunctionReturn(PETSC_SUCCESS); 4614 } 4615 4616 PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF) 4617 { 4618 DMField coordField; 4619 DMLabel depthLabel; 4620 IS facetIS; 4621 PetscInt dim; 4622 4623 PetscFunctionBegin; 4624 PetscCall(DMGetDimension(dm, &dim)); 4625 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 4626 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 4627 PetscCall(DMGetCoordinateField(dm, &coordField)); 4628 PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS)); 4629 PetscCall(ISDestroy(&facetIS)); 4630 PetscFunctionReturn(PETSC_SUCCESS); 4631 } 4632 4633 PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) 4634 { 4635 PetscDS prob; 4636 PetscInt numBd, bd; 4637 DMField coordField = NULL; 4638 IS facetIS = NULL; 4639 DMLabel depthLabel; 4640 PetscInt dim; 4641 4642 PetscFunctionBegin; 4643 PetscCall(DMGetDS(dm, &prob)); 4644 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 4645 PetscCall(DMGetDimension(dm, &dim)); 4646 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 4647 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 4648 for (bd = 0; bd < numBd; ++bd) { 4649 PetscWeakForm wf; 4650 DMBoundaryConditionType type; 4651 DMLabel label; 4652 const PetscInt *values; 4653 PetscInt field, numValues, v; 4654 PetscObject obj; 4655 PetscClassId id; 4656 PetscFormKey key; 4657 4658 PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL)); 4659 if (type & DM_BC_ESSENTIAL) continue; 4660 PetscCall(PetscDSGetDiscretization(prob, field, &obj)); 4661 PetscCall(PetscObjectGetClassId(obj, &id)); 4662 if (id != PETSCFE_CLASSID) continue; 4663 if (!facetIS) { 4664 DMLabel depthLabel; 4665 PetscInt dim; 4666 4667 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 4668 PetscCall(DMGetDimension(dm, &dim)); 4669 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 4670 } 4671 PetscCall(DMGetCoordinateField(dm, &coordField)); 4672 for (v = 0; v < numValues; ++v) { 4673 key.label = label; 4674 key.value = values[v]; 4675 key.field = field; 4676 key.part = 0; 4677 PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS)); 4678 } 4679 } 4680 PetscCall(ISDestroy(&facetIS)); 4681 PetscFunctionReturn(PETSC_SUCCESS); 4682 } 4683 4684 PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) 4685 { 4686 DM_Plex *mesh = (DM_Plex *)dm->data; 4687 const char *name = "Residual"; 4688 DM dmAux = NULL; 4689 DM dmGrad = NULL; 4690 DMLabel ghostLabel = NULL; 4691 PetscDS ds = NULL; 4692 PetscDS dsAux = NULL; 4693 PetscSection section = NULL; 4694 PetscBool useFEM = PETSC_FALSE; 4695 PetscBool useFVM = PETSC_FALSE; 4696 PetscBool isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE; 4697 PetscFV fvm = NULL; 4698 PetscFVCellGeom *cgeomFVM = NULL; 4699 PetscFVFaceGeom *fgeomFVM = NULL; 4700 DMField coordField = NULL; 4701 Vec locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, grad, locGrad = NULL; 4702 PetscScalar *u = NULL, *u_t, *a, *uL, *uR; 4703 IS chunkIS; 4704 const PetscInt *cells; 4705 PetscInt cStart, cEnd, numCells; 4706 PetscInt Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd; 4707 PetscInt maxDegree = PETSC_MAX_INT; 4708 PetscQuadrature affineQuad = NULL, *quads = NULL; 4709 PetscFEGeom *affineGeom = NULL, **geoms = NULL; 4710 4711 PetscFunctionBegin; 4712 if (!cellIS) PetscFunctionReturn(PETSC_SUCCESS); 4713 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 4714 if (cStart >= cEnd) PetscFunctionReturn(PETSC_SUCCESS); 4715 PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 4716 /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */ 4717 /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */ 4718 /* FEM+FVM */ 4719 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 4720 /* 1: Get sizes from dm and dmAux */ 4721 PetscCall(DMGetLocalSection(dm, §ion)); 4722 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 4723 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, NULL)); 4724 PetscCall(PetscDSGetNumFields(ds, &Nf)); 4725 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 4726 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA)); 4727 if (locA) { 4728 PetscInt subcell; 4729 PetscCall(VecGetDM(locA, &dmAux)); 4730 PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cells ? cells[cStart] : cStart, &subcell)); 4731 PetscCall(DMGetCellDS(dmAux, subcell, &dsAux, NULL)); 4732 PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux)); 4733 } 4734 /* 2: Get geometric data */ 4735 for (f = 0; f < Nf; ++f) { 4736 PetscObject obj; 4737 PetscClassId id; 4738 PetscBool fimp; 4739 4740 PetscCall(PetscDSGetImplicit(ds, f, &fimp)); 4741 if (isImplicit != fimp) continue; 4742 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 4743 PetscCall(PetscObjectGetClassId(obj, &id)); 4744 if (id == PETSCFE_CLASSID) useFEM = PETSC_TRUE; 4745 if (id == PETSCFV_CLASSID) { 4746 useFVM = PETSC_TRUE; 4747 fvm = (PetscFV)obj; 4748 } 4749 } 4750 if (useFEM) { 4751 PetscCall(DMGetCoordinateField(dm, &coordField)); 4752 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 4753 if (maxDegree <= 1) { 4754 PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad)); 4755 if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 4756 } else { 4757 PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms)); 4758 for (f = 0; f < Nf; ++f) { 4759 PetscObject obj; 4760 PetscClassId id; 4761 PetscBool fimp; 4762 4763 PetscCall(PetscDSGetImplicit(ds, f, &fimp)); 4764 if (isImplicit != fimp) continue; 4765 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 4766 PetscCall(PetscObjectGetClassId(obj, &id)); 4767 if (id == PETSCFE_CLASSID) { 4768 PetscFE fe = (PetscFE)obj; 4769 4770 PetscCall(PetscFEGetQuadrature(fe, &quads[f])); 4771 PetscCall(PetscObjectReference((PetscObject)quads[f])); 4772 PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 4773 } 4774 } 4775 } 4776 } 4777 if (useFVM) { 4778 PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL)); 4779 PetscCall(VecGetArrayRead(faceGeometryFVM, (const PetscScalar **)&fgeomFVM)); 4780 PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM)); 4781 /* Reconstruct and limit cell gradients */ 4782 PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad)); 4783 if (dmGrad) { 4784 PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd)); 4785 PetscCall(DMGetGlobalVector(dmGrad, &grad)); 4786 PetscCall(DMPlexReconstructGradients_Internal(dm, fvm, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad)); 4787 /* Communicate gradient values */ 4788 PetscCall(DMGetLocalVector(dmGrad, &locGrad)); 4789 PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad)); 4790 PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad)); 4791 PetscCall(DMRestoreGlobalVector(dmGrad, &grad)); 4792 } 4793 /* Handle non-essential (e.g. outflow) boundary values */ 4794 PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad)); 4795 } 4796 /* Loop over chunks */ 4797 if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS)); 4798 numCells = cEnd - cStart; 4799 numChunks = 1; 4800 cellChunkSize = numCells / numChunks; 4801 faceChunkSize = (fEnd - fStart) / numChunks; 4802 numChunks = PetscMin(1, numCells); 4803 for (chunk = 0; chunk < numChunks; ++chunk) { 4804 PetscScalar *elemVec, *fluxL, *fluxR; 4805 PetscReal *vol; 4806 PetscFVFaceGeom *fgeom; 4807 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 4808 PetscInt fS = fStart + chunk * faceChunkSize, fE = PetscMin(fS + faceChunkSize, fEnd), numFaces = 0, face; 4809 4810 /* Extract field coefficients */ 4811 if (useFEM) { 4812 PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells)); 4813 PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 4814 PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 4815 PetscCall(PetscArrayzero(elemVec, numCells * totDim)); 4816 } 4817 if (useFVM) { 4818 PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR)); 4819 PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol)); 4820 PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL)); 4821 PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR)); 4822 PetscCall(PetscArrayzero(fluxL, numFaces * totDim)); 4823 PetscCall(PetscArrayzero(fluxR, numFaces * totDim)); 4824 } 4825 /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */ 4826 /* Loop over fields */ 4827 for (f = 0; f < Nf; ++f) { 4828 PetscObject obj; 4829 PetscClassId id; 4830 PetscBool fimp; 4831 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset; 4832 4833 key.field = f; 4834 PetscCall(PetscDSGetImplicit(ds, f, &fimp)); 4835 if (isImplicit != fimp) continue; 4836 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 4837 PetscCall(PetscObjectGetClassId(obj, &id)); 4838 if (id == PETSCFE_CLASSID) { 4839 PetscFE fe = (PetscFE)obj; 4840 PetscFEGeom *geom = affineGeom ? affineGeom : geoms[f]; 4841 PetscFEGeom *chunkGeom = NULL; 4842 PetscQuadrature quad = affineQuad ? affineQuad : quads[f]; 4843 PetscInt Nq, Nb; 4844 4845 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 4846 PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL)); 4847 PetscCall(PetscFEGetDimension(fe, &Nb)); 4848 blockSize = Nb; 4849 batchSize = numBlocks * blockSize; 4850 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 4851 numChunks = numCells / (numBatches * batchSize); 4852 Ne = numChunks * numBatches * batchSize; 4853 Nr = numCells % (numBatches * batchSize); 4854 offset = numCells - Nr; 4855 /* Integrate FE residual to get elemVec (need fields at quadrature points) */ 4856 /* For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */ 4857 PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom)); 4858 PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec)); 4859 PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom)); 4860 PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux, &a[offset * totDimAux], t, &elemVec[offset * totDim])); 4861 PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom)); 4862 } else if (id == PETSCFV_CLASSID) { 4863 PetscFV fv = (PetscFV)obj; 4864 4865 Ne = numFaces; 4866 /* Riemann solve over faces (need fields at face centroids) */ 4867 /* We need to evaluate FE fields at those coordinates */ 4868 PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR)); 4869 } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f); 4870 } 4871 /* Loop over domain */ 4872 if (useFEM) { 4873 /* Add elemVec to locX */ 4874 for (c = cS; c < cE; ++c) { 4875 const PetscInt cell = cells ? cells[c] : c; 4876 const PetscInt cind = c - cStart; 4877 4878 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim])); 4879 if (ghostLabel) { 4880 PetscInt ghostVal; 4881 4882 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 4883 if (ghostVal > 0) continue; 4884 } 4885 PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES)); 4886 } 4887 } 4888 if (useFVM) { 4889 PetscScalar *fa; 4890 PetscInt iface; 4891 4892 PetscCall(VecGetArray(locF, &fa)); 4893 for (f = 0; f < Nf; ++f) { 4894 PetscFV fv; 4895 PetscObject obj; 4896 PetscClassId id; 4897 PetscInt foff, pdim; 4898 4899 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 4900 PetscCall(PetscDSGetFieldOffset(ds, f, &foff)); 4901 PetscCall(PetscObjectGetClassId(obj, &id)); 4902 if (id != PETSCFV_CLASSID) continue; 4903 fv = (PetscFV)obj; 4904 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 4905 /* Accumulate fluxes to cells */ 4906 for (face = fS, iface = 0; face < fE; ++face) { 4907 const PetscInt *scells; 4908 PetscScalar *fL = NULL, *fR = NULL; 4909 PetscInt ghost, d, nsupp, nchild; 4910 4911 PetscCall(DMLabelGetValue(ghostLabel, face, &ghost)); 4912 PetscCall(DMPlexGetSupportSize(dm, face, &nsupp)); 4913 PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL)); 4914 if (ghost >= 0 || nsupp > 2 || nchild > 0) continue; 4915 PetscCall(DMPlexGetSupport(dm, face, &scells)); 4916 PetscCall(DMLabelGetValue(ghostLabel, scells[0], &ghost)); 4917 if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL)); 4918 PetscCall(DMLabelGetValue(ghostLabel, scells[1], &ghost)); 4919 if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR)); 4920 for (d = 0; d < pdim; ++d) { 4921 if (fL) fL[d] -= fluxL[iface * totDim + foff + d]; 4922 if (fR) fR[d] += fluxR[iface * totDim + foff + d]; 4923 } 4924 ++iface; 4925 } 4926 } 4927 PetscCall(VecRestoreArray(locF, &fa)); 4928 } 4929 /* Handle time derivative */ 4930 if (locX_t) { 4931 PetscScalar *x_t, *fa; 4932 4933 PetscCall(VecGetArray(locF, &fa)); 4934 PetscCall(VecGetArray(locX_t, &x_t)); 4935 for (f = 0; f < Nf; ++f) { 4936 PetscFV fv; 4937 PetscObject obj; 4938 PetscClassId id; 4939 PetscInt pdim, d; 4940 4941 PetscCall(PetscDSGetDiscretization(ds, f, &obj)); 4942 PetscCall(PetscObjectGetClassId(obj, &id)); 4943 if (id != PETSCFV_CLASSID) continue; 4944 fv = (PetscFV)obj; 4945 PetscCall(PetscFVGetNumComponents(fv, &pdim)); 4946 for (c = cS; c < cE; ++c) { 4947 const PetscInt cell = cells ? cells[c] : c; 4948 PetscScalar *u_t, *r; 4949 4950 if (ghostLabel) { 4951 PetscInt ghostVal; 4952 4953 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 4954 if (ghostVal > 0) continue; 4955 } 4956 PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t)); 4957 PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r)); 4958 for (d = 0; d < pdim; ++d) r[d] += u_t[d]; 4959 } 4960 } 4961 PetscCall(VecRestoreArray(locX_t, &x_t)); 4962 PetscCall(VecRestoreArray(locF, &fa)); 4963 } 4964 if (useFEM) { 4965 PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a)); 4966 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec)); 4967 } 4968 if (useFVM) { 4969 PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR)); 4970 PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol)); 4971 PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL)); 4972 PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR)); 4973 if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad)); 4974 } 4975 } 4976 if (useFEM) PetscCall(ISDestroy(&chunkIS)); 4977 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 4978 4979 if (useFEM) { 4980 PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user)); 4981 4982 if (maxDegree <= 1) { 4983 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 4984 PetscCall(PetscQuadratureDestroy(&affineQuad)); 4985 } else { 4986 for (f = 0; f < Nf; ++f) { 4987 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 4988 PetscCall(PetscQuadratureDestroy(&quads[f])); 4989 } 4990 PetscCall(PetscFree2(quads, geoms)); 4991 } 4992 } 4993 4994 /* FEM */ 4995 /* 1: Get sizes from dm and dmAux */ 4996 /* 2: Get geometric data */ 4997 /* 3: Handle boundary values */ 4998 /* 4: Loop over domain */ 4999 /* Extract coefficients */ 5000 /* Loop over fields */ 5001 /* Set tiling for FE*/ 5002 /* Integrate FE residual to get elemVec */ 5003 /* Loop over subdomain */ 5004 /* Loop over quad points */ 5005 /* Transform coords to real space */ 5006 /* Evaluate field and aux fields at point */ 5007 /* Evaluate residual at point */ 5008 /* Transform residual to real space */ 5009 /* Add residual to elemVec */ 5010 /* Loop over domain */ 5011 /* Add elemVec to locX */ 5012 5013 /* FVM */ 5014 /* Get geometric data */ 5015 /* If using gradients */ 5016 /* Compute gradient data */ 5017 /* Loop over domain faces */ 5018 /* Count computational faces */ 5019 /* Reconstruct cell gradient */ 5020 /* Loop over domain cells */ 5021 /* Limit cell gradients */ 5022 /* Handle boundary values */ 5023 /* Loop over domain faces */ 5024 /* Read out field, centroid, normal, volume for each side of face */ 5025 /* Riemann solve over faces */ 5026 /* Loop over domain faces */ 5027 /* Accumulate fluxes to cells */ 5028 /* TODO Change printFEM to printDisc here */ 5029 if (mesh->printFEM) { 5030 Vec locFbc; 5031 PetscInt pStart, pEnd, p, maxDof; 5032 PetscScalar *zeroes; 5033 5034 PetscCall(VecDuplicate(locF, &locFbc)); 5035 PetscCall(VecCopy(locF, locFbc)); 5036 PetscCall(PetscSectionGetChart(section, &pStart, &pEnd)); 5037 PetscCall(PetscSectionGetMaxDof(section, &maxDof)); 5038 PetscCall(PetscCalloc1(maxDof, &zeroes)); 5039 for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES)); 5040 PetscCall(PetscFree(zeroes)); 5041 PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc)); 5042 PetscCall(VecDestroy(&locFbc)); 5043 } 5044 PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5045 PetscFunctionReturn(PETSC_SUCCESS); 5046 } 5047 5048 /* 5049 1) Allow multiple kernels for BdResidual for hybrid DS 5050 5051 DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux 5052 5053 DONE 3) Change DMGetCellFields() to get different aux data a[] for each side 5054 - I think I just need to replace a[] with the closure from each face 5055 5056 4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before 5057 */ 5058 PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) 5059 { 5060 DM_Plex *mesh = (DM_Plex *)dm->data; 5061 const char *name = "Hybrid Residual"; 5062 DM dmAux[3] = {NULL, NULL, NULL}; 5063 DMLabel ghostLabel = NULL; 5064 PetscDS ds = NULL; 5065 PetscDS dsIn = NULL; 5066 PetscDS dsAux[3] = {NULL, NULL, NULL}; 5067 Vec locA[3] = {NULL, NULL, NULL}; 5068 DM dmScale[3] = {NULL, NULL, NULL}; 5069 PetscDS dsScale[3] = {NULL, NULL, NULL}; 5070 Vec locS[3] = {NULL, NULL, NULL}; 5071 PetscSection section = NULL; 5072 DMField coordField = NULL; 5073 PetscScalar *a[3] = {NULL, NULL, NULL}; 5074 PetscScalar *s[3] = {NULL, NULL, NULL}; 5075 PetscScalar *u = NULL, *u_t; 5076 PetscScalar *elemVecNeg, *elemVecPos, *elemVecCoh; 5077 IS chunkIS; 5078 const PetscInt *cells; 5079 PetscInt *faces; 5080 PetscInt cStart, cEnd, numCells; 5081 PetscInt Nf, f, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk; 5082 PetscInt maxDegree = PETSC_MAX_INT; 5083 PetscQuadrature affineQuad = NULL, *quads = NULL; 5084 PetscFEGeom *affineGeom = NULL, **geoms = NULL; 5085 5086 PetscFunctionBegin; 5087 if (!cellIS) PetscFunctionReturn(PETSC_SUCCESS); 5088 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 5089 PetscCall(ISGetLocalSize(cellIS, &numCells)); 5090 if (cStart >= cEnd) PetscFunctionReturn(PETSC_SUCCESS); 5091 if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) { 5092 const char *name; 5093 PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name)); 5094 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part); 5095 } 5096 PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5097 /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */ 5098 /* FEM */ 5099 /* 1: Get sizes from dm and dmAux */ 5100 PetscCall(DMGetSection(dm, §ion)); 5101 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 5102 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn)); 5103 PetscCall(PetscDSGetNumFields(ds, &Nf)); 5104 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 5105 PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn)); 5106 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2])); 5107 if (locA[2]) { 5108 const PetscInt cellStart = cells ? cells[cStart] : cStart; 5109 5110 PetscCall(VecGetDM(locA[2], &dmAux[2])); 5111 PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL)); 5112 PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2])); 5113 { 5114 const PetscInt *cone; 5115 PetscInt c; 5116 5117 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 5118 for (c = 0; c < 2; ++c) { 5119 const PetscInt *support; 5120 PetscInt ssize, s; 5121 5122 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 5123 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 5124 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 5125 if (support[0] == cellStart) s = 1; 5126 else if (support[1] == cellStart) s = 0; 5127 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 5128 PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c])); 5129 PetscCheck(locA[c], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Must have auxiliary vector for (%p, %" PetscInt_FMT ", %" PetscInt_FMT ")", (void *)key[c].label, key[c].value, key[c].part); 5130 if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c])); 5131 else dmAux[c] = dmAux[2]; 5132 PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL)); 5133 PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c])); 5134 } 5135 } 5136 } 5137 /* Handle mass matrix scaling 5138 The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */ 5139 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2])); 5140 if (locS[2]) { 5141 const PetscInt cellStart = cells ? cells[cStart] : cStart; 5142 PetscInt Nb, Nbs; 5143 5144 PetscCall(VecGetDM(locS[2], &dmScale[2])); 5145 PetscCall(DMGetCellDS(dmScale[2], cellStart, &dsScale[2], NULL)); 5146 PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2])); 5147 // BRAD: This is not set correctly 5148 key[2].field = 2; 5149 PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb)); 5150 PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs)); 5151 PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs); 5152 { 5153 const PetscInt *cone; 5154 PetscInt c; 5155 5156 locS[1] = locS[0] = locS[2]; 5157 dmScale[1] = dmScale[0] = dmScale[2]; 5158 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 5159 for (c = 0; c < 2; ++c) { 5160 const PetscInt *support; 5161 PetscInt ssize, s; 5162 5163 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 5164 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 5165 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 5166 if (support[0] == cellStart) s = 1; 5167 else if (support[1] == cellStart) s = 0; 5168 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 5169 PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL)); 5170 PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c])); 5171 } 5172 } 5173 } 5174 /* 2: Setup geometric data */ 5175 PetscCall(DMGetCoordinateField(dm, &coordField)); 5176 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 5177 if (maxDegree > 1) { 5178 PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms)); 5179 for (f = 0; f < Nf; ++f) { 5180 PetscFE fe; 5181 5182 PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe)); 5183 if (fe) { 5184 PetscCall(PetscFEGetQuadrature(fe, &quads[f])); 5185 PetscCall(PetscObjectReference((PetscObject)quads[f])); 5186 } 5187 } 5188 } 5189 /* Loop over chunks */ 5190 cellChunkSize = numCells; 5191 numChunks = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize); 5192 PetscCall(PetscCalloc1(2 * cellChunkSize, &faces)); 5193 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS)); 5194 /* Extract field coefficients */ 5195 /* NOTE This needs the end cap faces to have identical orientations */ 5196 PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 5197 PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 5198 PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s)); 5199 PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecNeg)); 5200 PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecPos)); 5201 PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVecCoh)); 5202 for (chunk = 0; chunk < numChunks; ++chunk) { 5203 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 5204 5205 PetscCall(PetscArrayzero(elemVecNeg, cellChunkSize * totDim)); 5206 PetscCall(PetscArrayzero(elemVecPos, cellChunkSize * totDim)); 5207 PetscCall(PetscArrayzero(elemVecCoh, cellChunkSize * totDim)); 5208 /* Get faces */ 5209 for (c = cS; c < cE; ++c) { 5210 const PetscInt cell = cells ? cells[c] : c; 5211 const PetscInt *cone; 5212 PetscCall(DMPlexGetCone(dm, cell, &cone)); 5213 faces[(c - cS) * 2 + 0] = cone[0]; 5214 faces[(c - cS) * 2 + 1] = cone[1]; 5215 } 5216 PetscCall(ISGeneralSetIndices(chunkIS, 2 * cellChunkSize, faces, PETSC_USE_POINTER)); 5217 /* Get geometric data */ 5218 if (maxDegree <= 1) { 5219 if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad)); 5220 if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom)); 5221 } else { 5222 for (f = 0; f < Nf; ++f) { 5223 if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f])); 5224 } 5225 } 5226 /* Loop over fields */ 5227 for (f = 0; f < Nf; ++f) { 5228 PetscFE fe; 5229 PetscFEGeom *geom = affineGeom ? affineGeom : geoms[f]; 5230 PetscFEGeom *chunkGeom = NULL, *remGeom = NULL; 5231 PetscQuadrature quad = affineQuad ? affineQuad : quads[f]; 5232 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb; 5233 PetscBool isCohesiveField; 5234 5235 PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe)); 5236 if (!fe) continue; 5237 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 5238 PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL)); 5239 PetscCall(PetscFEGetDimension(fe, &Nb)); 5240 blockSize = Nb; 5241 batchSize = numBlocks * blockSize; 5242 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 5243 numChunks = numCells / (numBatches * batchSize); 5244 Ne = numChunks * numBatches * batchSize; 5245 Nr = numCells % (numBatches * batchSize); 5246 offset = numCells - Nr; 5247 PetscCall(PetscFEGeomGetChunk(geom, 0, offset * 2, &chunkGeom)); 5248 PetscCall(PetscFEGeomGetChunk(geom, offset * 2, numCells * 2, &remGeom)); 5249 PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField)); 5250 chunkGeom->isCohesive = remGeom->isCohesive = PETSC_TRUE; 5251 key[0].field = f; 5252 key[1].field = f; 5253 key[2].field = f; 5254 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, elemVecNeg)); 5255 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[0], 0, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[0], &a[0][offset * totDimAux[0]], t, &elemVecNeg[offset * totDim])); 5256 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, elemVecPos)); 5257 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[1], 1, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[1], &a[1][offset * totDimAux[1]], t, &elemVecPos[offset * totDim])); 5258 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, elemVecCoh)); 5259 PetscCall(PetscFEIntegrateHybridResidual(ds, dsIn, key[2], 2, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[2], &a[2][offset * totDimAux[2]], t, &elemVecCoh[offset * totDim])); 5260 PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &remGeom)); 5261 PetscCall(PetscFEGeomRestoreChunk(geom, 0, offset, &chunkGeom)); 5262 } 5263 /* Add elemVec to locX */ 5264 for (c = cS; c < cE; ++c) { 5265 const PetscInt cell = cells ? cells[c] : c; 5266 const PetscInt cind = c - cStart; 5267 PetscInt i; 5268 5269 /* Scale element values */ 5270 if (locS[0]) { 5271 PetscInt Nb, off = cind * totDim, soff = cind * totDimScale[0]; 5272 PetscBool cohesive; 5273 5274 for (f = 0; f < Nf; ++f) { 5275 PetscCall(PetscDSGetFieldSize(ds, f, &Nb)); 5276 PetscCall(PetscDSGetCohesive(ds, f, &cohesive)); 5277 if (f == key[2].field) { 5278 PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields"); 5279 // No cohesive scaling field is currently input 5280 for (i = 0; i < Nb; ++i) elemVecCoh[off + i] += s[0][soff + i] * elemVecNeg[off + i] + s[1][soff + i] * elemVecPos[off + i]; 5281 off += Nb; 5282 } else { 5283 const PetscInt N = cohesive ? Nb : Nb * 2; 5284 5285 for (i = 0; i < N; ++i) elemVecCoh[off + i] += elemVecNeg[off + i] + elemVecPos[off + i]; 5286 off += N; 5287 } 5288 } 5289 } else { 5290 for (i = cind * totDim; i < (cind + 1) * totDim; ++i) elemVecCoh[i] += elemVecNeg[i] + elemVecPos[i]; 5291 } 5292 if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVecCoh[cind * totDim])); 5293 if (ghostLabel) { 5294 PetscInt ghostVal; 5295 5296 PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal)); 5297 if (ghostVal > 0) continue; 5298 } 5299 PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVecCoh[cind * totDim], ADD_ALL_VALUES)); 5300 } 5301 } 5302 PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 5303 PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 5304 PetscCall(DMPlexRestoreHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s)); 5305 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecNeg)); 5306 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecPos)); 5307 PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVecCoh)); 5308 PetscCall(PetscFree(faces)); 5309 PetscCall(ISDestroy(&chunkIS)); 5310 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 5311 if (maxDegree <= 1) { 5312 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 5313 PetscCall(PetscQuadratureDestroy(&affineQuad)); 5314 } else { 5315 for (f = 0; f < Nf; ++f) { 5316 if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 5317 if (quads) PetscCall(PetscQuadratureDestroy(&quads[f])); 5318 } 5319 PetscCall(PetscFree2(quads, geoms)); 5320 } 5321 if (mesh->printFEM) { 5322 Vec locFbc; 5323 PetscInt pStart, pEnd, p, maxDof; 5324 PetscScalar *zeroes; 5325 5326 PetscCall(VecDuplicate(locF, &locFbc)); 5327 PetscCall(VecCopy(locF, locFbc)); 5328 PetscCall(PetscSectionGetChart(section, &pStart, &pEnd)); 5329 PetscCall(PetscSectionGetMaxDof(section, &maxDof)); 5330 PetscCall(PetscCalloc1(maxDof, &zeroes)); 5331 for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES)); 5332 PetscCall(PetscFree(zeroes)); 5333 PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc)); 5334 PetscCall(VecDestroy(&locFbc)); 5335 } 5336 PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0)); 5337 PetscFunctionReturn(PETSC_SUCCESS); 5338 } 5339 5340 PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS) 5341 { 5342 DM_Plex *mesh = (DM_Plex *)dm->data; 5343 DM plex = NULL, plexA = NULL, tdm; 5344 DMEnclosureType encAux; 5345 PetscDS prob, probAux = NULL; 5346 PetscSection section, sectionAux = NULL; 5347 PetscSection globalSection; 5348 Vec locA = NULL, tv; 5349 PetscScalar *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL; 5350 PetscInt v; 5351 PetscInt Nf, totDim, totDimAux = 0; 5352 PetscBool transform; 5353 5354 PetscFunctionBegin; 5355 PetscCall(DMConvert(dm, DMPLEX, &plex)); 5356 PetscCall(DMHasBasisTransform(dm, &transform)); 5357 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 5358 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 5359 PetscCall(DMGetLocalSection(dm, §ion)); 5360 PetscCall(DMGetDS(dm, &prob)); 5361 PetscCall(PetscDSGetNumFields(prob, &Nf)); 5362 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 5363 PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA)); 5364 if (locA) { 5365 DM dmAux; 5366 5367 PetscCall(VecGetDM(locA, &dmAux)); 5368 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 5369 PetscCall(DMConvert(dmAux, DMPLEX, &plexA)); 5370 PetscCall(DMGetDS(plexA, &probAux)); 5371 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 5372 PetscCall(DMGetLocalSection(plexA, §ionAux)); 5373 } 5374 5375 PetscCall(DMGetGlobalSection(dm, &globalSection)); 5376 for (v = 0; v < numValues; ++v) { 5377 PetscFEGeom *fgeom; 5378 PetscInt maxDegree; 5379 PetscQuadrature qGeom = NULL; 5380 IS pointIS; 5381 const PetscInt *points; 5382 PetscFormKey key; 5383 PetscInt numFaces, face, Nq; 5384 5385 key.label = label; 5386 key.value = values[v]; 5387 key.part = 0; 5388 PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS)); 5389 if (!pointIS) continue; /* No points with that id on this process */ 5390 { 5391 IS isectIS; 5392 5393 /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */ 5394 PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS)); 5395 PetscCall(ISDestroy(&pointIS)); 5396 pointIS = isectIS; 5397 } 5398 PetscCall(ISGetLocalSize(pointIS, &numFaces)); 5399 PetscCall(ISGetIndices(pointIS, &points)); 5400 PetscCall(PetscMalloc4(numFaces * totDim, &u, locX_t ? numFaces * totDim : 0, &u_t, numFaces * totDim * totDim, &elemMat, locA ? numFaces * totDimAux : 0, &a)); 5401 PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree)); 5402 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom)); 5403 if (!qGeom) { 5404 PetscFE fe; 5405 5406 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 5407 PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom)); 5408 PetscCall(PetscObjectReference((PetscObject)qGeom)); 5409 } 5410 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 5411 PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 5412 for (face = 0; face < numFaces; ++face) { 5413 const PetscInt point = points[face], *support; 5414 PetscScalar *x = NULL; 5415 PetscInt i; 5416 5417 PetscCall(DMPlexGetSupport(dm, point, &support)); 5418 PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x)); 5419 for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i]; 5420 PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x)); 5421 if (locX_t) { 5422 PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x)); 5423 for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i]; 5424 PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x)); 5425 } 5426 if (locA) { 5427 PetscInt subp; 5428 PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp)); 5429 PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x)); 5430 for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i]; 5431 PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x)); 5432 } 5433 } 5434 PetscCall(PetscArrayzero(elemMat, numFaces * totDim * totDim)); 5435 { 5436 PetscFE fe; 5437 PetscInt Nb; 5438 /* Conforming batches */ 5439 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 5440 /* Remainder */ 5441 PetscFEGeom *chunkGeom = NULL; 5442 PetscInt fieldJ, Nr, offset; 5443 5444 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 5445 PetscCall(PetscFEGetDimension(fe, &Nb)); 5446 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 5447 blockSize = Nb; 5448 batchSize = numBlocks * blockSize; 5449 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 5450 numChunks = numFaces / (numBatches * batchSize); 5451 Ne = numChunks * numBatches * batchSize; 5452 Nr = numFaces % (numBatches * batchSize); 5453 offset = numFaces - Nr; 5454 PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom)); 5455 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 5456 key.field = fieldI * Nf + fieldJ; 5457 PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat)); 5458 } 5459 PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom)); 5460 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 5461 key.field = fieldI * Nf + fieldJ; 5462 PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, a ? &a[offset * totDimAux] : NULL, t, X_tShift, &elemMat[offset * totDim * totDim])); 5463 } 5464 PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom)); 5465 } 5466 for (face = 0; face < numFaces; ++face) { 5467 const PetscInt point = points[face], *support; 5468 5469 /* Transform to global basis before insertion in Jacobian */ 5470 PetscCall(DMPlexGetSupport(plex, point, &support)); 5471 if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face * totDim * totDim])); 5472 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim])); 5473 PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, support[0], &elemMat[face * totDim * totDim], ADD_VALUES)); 5474 } 5475 PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom)); 5476 PetscCall(PetscQuadratureDestroy(&qGeom)); 5477 PetscCall(ISRestoreIndices(pointIS, &points)); 5478 PetscCall(ISDestroy(&pointIS)); 5479 PetscCall(PetscFree4(u, u_t, elemMat, a)); 5480 } 5481 if (plex) PetscCall(DMDestroy(&plex)); 5482 if (plexA) PetscCall(DMDestroy(&plexA)); 5483 PetscFunctionReturn(PETSC_SUCCESS); 5484 } 5485 5486 PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP) 5487 { 5488 DMField coordField; 5489 DMLabel depthLabel; 5490 IS facetIS; 5491 PetscInt dim; 5492 5493 PetscFunctionBegin; 5494 PetscCall(DMGetDimension(dm, &dim)); 5495 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 5496 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 5497 PetscCall(DMGetCoordinateField(dm, &coordField)); 5498 PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS)); 5499 PetscCall(ISDestroy(&facetIS)); 5500 PetscFunctionReturn(PETSC_SUCCESS); 5501 } 5502 5503 PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user) 5504 { 5505 PetscDS prob; 5506 PetscInt dim, numBd, bd; 5507 DMLabel depthLabel; 5508 DMField coordField = NULL; 5509 IS facetIS; 5510 5511 PetscFunctionBegin; 5512 PetscCall(DMGetDS(dm, &prob)); 5513 PetscCall(DMPlexGetDepthLabel(dm, &depthLabel)); 5514 PetscCall(DMGetDimension(dm, &dim)); 5515 PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS)); 5516 PetscCall(PetscDSGetNumBoundary(prob, &numBd)); 5517 PetscCall(DMGetCoordinateField(dm, &coordField)); 5518 for (bd = 0; bd < numBd; ++bd) { 5519 PetscWeakForm wf; 5520 DMBoundaryConditionType type; 5521 DMLabel label; 5522 const PetscInt *values; 5523 PetscInt fieldI, numValues; 5524 PetscObject obj; 5525 PetscClassId id; 5526 5527 PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL)); 5528 if (type & DM_BC_ESSENTIAL) continue; 5529 PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj)); 5530 PetscCall(PetscObjectGetClassId(obj, &id)); 5531 if (id != PETSCFE_CLASSID) continue; 5532 PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS)); 5533 } 5534 PetscCall(ISDestroy(&facetIS)); 5535 PetscFunctionReturn(PETSC_SUCCESS); 5536 } 5537 5538 PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *user) 5539 { 5540 DM_Plex *mesh = (DM_Plex *)dm->data; 5541 const char *name = "Jacobian"; 5542 DM dmAux = NULL, plex, tdm; 5543 DMEnclosureType encAux; 5544 Vec A, tv; 5545 DMField coordField; 5546 PetscDS prob, probAux = NULL; 5547 PetscSection section, globalSection, sectionAux; 5548 PetscScalar *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL; 5549 const PetscInt *cells; 5550 PetscInt Nf, fieldI, fieldJ; 5551 PetscInt totDim, totDimAux = 0, cStart, cEnd, numCells, c; 5552 PetscBool hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, hasDyn, hasFV = PETSC_FALSE, transform; 5553 5554 PetscFunctionBegin; 5555 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 5556 if (!cellIS) goto end; 5557 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 5558 PetscCall(ISGetLocalSize(cellIS, &numCells)); 5559 if (cStart >= cEnd) goto end; 5560 PetscCall(DMHasBasisTransform(dm, &transform)); 5561 PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm)); 5562 PetscCall(DMGetBasisTransformVec_Internal(dm, &tv)); 5563 PetscCall(DMGetLocalSection(dm, §ion)); 5564 PetscCall(DMGetGlobalSection(dm, &globalSection)); 5565 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL)); 5566 PetscCall(PetscDSGetNumFields(prob, &Nf)); 5567 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 5568 PetscCall(PetscDSHasJacobian(prob, &hasJac)); 5569 PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec)); 5570 /* user passed in the same matrix, avoid double contributions and 5571 only assemble the Jacobian */ 5572 if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE; 5573 PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn)); 5574 hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE; 5575 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A)); 5576 if (A) { 5577 PetscCall(VecGetDM(A, &dmAux)); 5578 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 5579 PetscCall(DMConvert(dmAux, DMPLEX, &plex)); 5580 PetscCall(DMGetLocalSection(plex, §ionAux)); 5581 PetscCall(DMGetDS(dmAux, &probAux)); 5582 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 5583 } 5584 PetscCall(PetscMalloc5(numCells * totDim, &u, X_t ? numCells * totDim : 0, &u_t, hasJac ? numCells * totDim * totDim : 0, &elemMat, hasPrec ? numCells * totDim * totDim : 0, &elemMatP, hasDyn ? numCells * totDim * totDim : 0, &elemMatD)); 5585 if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a)); 5586 PetscCall(DMGetCoordinateField(dm, &coordField)); 5587 for (c = cStart; c < cEnd; ++c) { 5588 const PetscInt cell = cells ? cells[c] : c; 5589 const PetscInt cind = c - cStart; 5590 PetscScalar *x = NULL, *x_t = NULL; 5591 PetscInt i; 5592 5593 PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x)); 5594 for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i]; 5595 PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x)); 5596 if (X_t) { 5597 PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t)); 5598 for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i]; 5599 PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t)); 5600 } 5601 if (dmAux) { 5602 PetscInt subcell; 5603 PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell)); 5604 PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x)); 5605 for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i]; 5606 PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x)); 5607 } 5608 } 5609 if (hasJac) PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim)); 5610 if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells * totDim * totDim)); 5611 if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim)); 5612 for (fieldI = 0; fieldI < Nf; ++fieldI) { 5613 PetscClassId id; 5614 PetscFE fe; 5615 PetscQuadrature qGeom = NULL; 5616 PetscInt Nb; 5617 /* Conforming batches */ 5618 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 5619 /* Remainder */ 5620 PetscInt Nr, offset, Nq; 5621 PetscInt maxDegree; 5622 PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL; 5623 5624 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 5625 PetscCall(PetscObjectGetClassId((PetscObject)fe, &id)); 5626 if (id == PETSCFV_CLASSID) { 5627 hasFV = PETSC_TRUE; 5628 continue; 5629 } 5630 PetscCall(PetscFEGetDimension(fe, &Nb)); 5631 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 5632 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 5633 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom)); 5634 if (!qGeom) { 5635 PetscCall(PetscFEGetQuadrature(fe, &qGeom)); 5636 PetscCall(PetscObjectReference((PetscObject)qGeom)); 5637 } 5638 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 5639 PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 5640 blockSize = Nb; 5641 batchSize = numBlocks * blockSize; 5642 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 5643 numChunks = numCells / (numBatches * batchSize); 5644 Ne = numChunks * numBatches * batchSize; 5645 Nr = numCells % (numBatches * batchSize); 5646 offset = numCells - Nr; 5647 PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom)); 5648 PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom)); 5649 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 5650 key.field = fieldI * Nf + fieldJ; 5651 if (hasJac) { 5652 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat)); 5653 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMat[offset * totDim * totDim])); 5654 } 5655 if (hasPrec) { 5656 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP)); 5657 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMatP[offset * totDim * totDim])); 5658 } 5659 if (hasDyn) { 5660 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD)); 5661 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim])); 5662 } 5663 } 5664 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom)); 5665 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom)); 5666 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 5667 PetscCall(PetscQuadratureDestroy(&qGeom)); 5668 } 5669 /* Add contribution from X_t */ 5670 if (hasDyn) { 5671 for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c]; 5672 } 5673 if (hasFV) { 5674 PetscClassId id; 5675 PetscFV fv; 5676 PetscInt offsetI, NcI, NbI = 1, fc, f; 5677 5678 for (fieldI = 0; fieldI < Nf; ++fieldI) { 5679 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv)); 5680 PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI)); 5681 PetscCall(PetscObjectGetClassId((PetscObject)fv, &id)); 5682 if (id != PETSCFV_CLASSID) continue; 5683 /* Put in the identity */ 5684 PetscCall(PetscFVGetNumComponents(fv, &NcI)); 5685 for (c = cStart; c < cEnd; ++c) { 5686 const PetscInt cind = c - cStart; 5687 const PetscInt eOffset = cind * totDim * totDim; 5688 for (fc = 0; fc < NcI; ++fc) { 5689 for (f = 0; f < NbI; ++f) { 5690 const PetscInt i = offsetI + f * NcI + fc; 5691 if (hasPrec) { 5692 if (hasJac) elemMat[eOffset + i * totDim + i] = 1.0; 5693 elemMatP[eOffset + i * totDim + i] = 1.0; 5694 } else { 5695 elemMat[eOffset + i * totDim + i] = 1.0; 5696 } 5697 } 5698 } 5699 } 5700 } 5701 /* No allocated space for FV stuff, so ignore the zero entries */ 5702 PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); 5703 } 5704 /* Insert values into matrix */ 5705 for (c = cStart; c < cEnd; ++c) { 5706 const PetscInt cell = cells ? cells[c] : c; 5707 const PetscInt cind = c - cStart; 5708 5709 /* Transform to global basis before insertion in Jacobian */ 5710 if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind * totDim * totDim])); 5711 if (hasPrec) { 5712 if (hasJac) { 5713 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim])); 5714 PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[cind * totDim * totDim], ADD_VALUES)); 5715 } 5716 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind * totDim * totDim])); 5717 PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMatP[cind * totDim * totDim], ADD_VALUES)); 5718 } else { 5719 if (hasJac) { 5720 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim])); 5721 PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMat[cind * totDim * totDim], ADD_VALUES)); 5722 } 5723 } 5724 } 5725 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 5726 if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE)); 5727 PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, elemMatD)); 5728 if (dmAux) { 5729 PetscCall(PetscFree(a)); 5730 PetscCall(DMDestroy(&plex)); 5731 } 5732 /* Compute boundary integrals */ 5733 PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user)); 5734 /* Assemble matrix */ 5735 end : { 5736 PetscBool assOp = hasJac && hasPrec ? PETSC_TRUE : PETSC_FALSE, gassOp; 5737 5738 PetscCall(MPIU_Allreduce(&assOp, &gassOp, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm))); 5739 if (hasJac && hasPrec) { 5740 PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY)); 5741 PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY)); 5742 } 5743 } 5744 PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY)); 5745 PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY)); 5746 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 5747 PetscFunctionReturn(PETSC_SUCCESS); 5748 } 5749 5750 PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user) 5751 { 5752 DM_Plex *mesh = (DM_Plex *)dm->data; 5753 const char *name = "Hybrid Jacobian"; 5754 DM dmAux[3] = {NULL, NULL, NULL}; 5755 DMLabel ghostLabel = NULL; 5756 DM plex = NULL; 5757 DM plexA = NULL; 5758 PetscDS ds = NULL; 5759 PetscDS dsIn = NULL; 5760 PetscDS dsAux[3] = {NULL, NULL, NULL}; 5761 Vec locA[3] = {NULL, NULL, NULL}; 5762 DM dmScale[3] = {NULL, NULL, NULL}; 5763 PetscDS dsScale[3] = {NULL, NULL, NULL}; 5764 Vec locS[3] = {NULL, NULL, NULL}; 5765 PetscSection section = NULL; 5766 PetscSection sectionAux[3] = {NULL, NULL, NULL}; 5767 DMField coordField = NULL; 5768 PetscScalar *a[3] = {NULL, NULL, NULL}; 5769 PetscScalar *s[3] = {NULL, NULL, NULL}; 5770 PetscScalar *u = NULL, *u_t; 5771 PetscScalar *elemMatNeg, *elemMatPos, *elemMatCoh; 5772 PetscScalar *elemMatNegP, *elemMatPosP, *elemMatCohP; 5773 PetscSection globalSection; 5774 IS chunkIS; 5775 const PetscInt *cells; 5776 PetscInt *faces; 5777 PetscInt cStart, cEnd, numCells; 5778 PetscInt Nf, fieldI, fieldJ, totDim, totDimIn, totDimAux[3], totDimScale[3], numChunks, cellChunkSize, chunk; 5779 PetscInt maxDegree = PETSC_MAX_INT; 5780 PetscQuadrature affineQuad = NULL, *quads = NULL; 5781 PetscFEGeom *affineGeom = NULL, **geoms = NULL; 5782 PetscBool hasBdJac, hasBdPrec; 5783 5784 PetscFunctionBegin; 5785 if (!cellIS) PetscFunctionReturn(PETSC_SUCCESS); 5786 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 5787 PetscCall(ISGetLocalSize(cellIS, &numCells)); 5788 if (cStart >= cEnd) PetscFunctionReturn(PETSC_SUCCESS); 5789 if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) { 5790 const char *name; 5791 PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name)); 5792 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part); 5793 } 5794 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 5795 PetscCall(DMConvert(dm, DMPLEX, &plex)); 5796 PetscCall(DMGetSection(dm, §ion)); 5797 PetscCall(DMGetGlobalSection(dm, &globalSection)); 5798 PetscCall(DMGetLabel(dm, "ghost", &ghostLabel)); 5799 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds, &dsIn)); 5800 PetscCall(PetscDSGetNumFields(ds, &Nf)); 5801 PetscCall(PetscDSGetTotalDimension(ds, &totDim)); 5802 PetscCall(PetscDSGetTotalDimension(dsIn, &totDimIn)); 5803 PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac)); 5804 PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec)); 5805 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2])); 5806 if (locA[2]) { 5807 const PetscInt cellStart = cells ? cells[cStart] : cStart; 5808 5809 PetscCall(VecGetDM(locA[2], &dmAux[2])); 5810 PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA)); 5811 PetscCall(DMGetSection(dmAux[2], §ionAux[2])); 5812 PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2], NULL)); 5813 PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2])); 5814 { 5815 const PetscInt *cone; 5816 PetscInt c; 5817 5818 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 5819 for (c = 0; c < 2; ++c) { 5820 const PetscInt *support; 5821 PetscInt ssize, s; 5822 5823 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 5824 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 5825 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 5826 if (support[0] == cellStart) s = 1; 5827 else if (support[1] == cellStart) s = 0; 5828 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 5829 PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c])); 5830 if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c])); 5831 else dmAux[c] = dmAux[2]; 5832 PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c], NULL)); 5833 PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c])); 5834 } 5835 } 5836 } 5837 /* Handle mass matrix scaling 5838 The field in key[2] is the field to be scaled, and the scaling field is the first in the dsScale */ 5839 PetscCall(DMGetAuxiliaryVec(dm, key[2].label, -key[2].value, key[2].part, &locS[2])); 5840 if (locS[2]) { 5841 const PetscInt cellStart = cells ? cells[cStart] : cStart; 5842 PetscInt Nb, Nbs; 5843 5844 PetscCall(VecGetDM(locS[2], &dmScale[2])); 5845 PetscCall(DMGetCellDS(dmScale[2], cells ? cells[cStart] : cStart, &dsScale[2], NULL)); 5846 PetscCall(PetscDSGetTotalDimension(dsScale[2], &totDimScale[2])); 5847 // BRAD: This is not set correctly 5848 key[2].field = 2; 5849 PetscCall(PetscDSGetFieldSize(ds, key[2].field, &Nb)); 5850 PetscCall(PetscDSGetFieldSize(dsScale[2], 0, &Nbs)); 5851 PetscCheck(Nb == Nbs, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Field %" PetscInt_FMT " of size %" PetscInt_FMT " cannot be scaled by field of size %" PetscInt_FMT, key[2].field, Nb, Nbs); 5852 { 5853 const PetscInt *cone; 5854 PetscInt c; 5855 5856 locS[1] = locS[0] = locS[2]; 5857 dmScale[1] = dmScale[0] = dmScale[2]; 5858 PetscCall(DMPlexGetCone(dm, cellStart, &cone)); 5859 for (c = 0; c < 2; ++c) { 5860 const PetscInt *support; 5861 PetscInt ssize, s; 5862 5863 PetscCall(DMPlexGetSupport(dm, cone[c], &support)); 5864 PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize)); 5865 PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize); 5866 if (support[0] == cellStart) s = 1; 5867 else if (support[1] == cellStart) s = 0; 5868 else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart); 5869 PetscCall(DMGetCellDS(dmScale[c], support[s], &dsScale[c], NULL)); 5870 PetscCall(PetscDSGetTotalDimension(dsScale[c], &totDimScale[c])); 5871 } 5872 } 5873 } 5874 /* 2: Setup geometric data */ 5875 PetscCall(DMGetCoordinateField(dm, &coordField)); 5876 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 5877 if (maxDegree > 1) { 5878 PetscInt f; 5879 PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms)); 5880 for (f = 0; f < Nf; ++f) { 5881 PetscFE fe; 5882 5883 PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe)); 5884 if (fe) { 5885 PetscCall(PetscFEGetQuadrature(fe, &quads[f])); 5886 PetscCall(PetscObjectReference((PetscObject)quads[f])); 5887 } 5888 } 5889 } 5890 /* Loop over chunks */ 5891 cellChunkSize = numCells; 5892 numChunks = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize); 5893 PetscCall(PetscCalloc1(2 * cellChunkSize, &faces)); 5894 PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 1 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS)); 5895 /* Extract field coefficients */ 5896 /* NOTE This needs the end cap faces to have identical orientations */ 5897 PetscCall(DMPlexGetHybridCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 5898 PetscCall(DMPlexGetHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 5899 PetscCall(DMPlexGetHybridFields(dm, dmScale, dsScale, cellIS, locS, PETSC_TRUE, s)); 5900 PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg)); 5901 PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos)); 5902 PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh)); 5903 PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP)); 5904 PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP)); 5905 PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP)); 5906 for (chunk = 0; chunk < numChunks; ++chunk) { 5907 PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c; 5908 5909 if (hasBdJac) { 5910 PetscCall(PetscArrayzero(elemMatNeg, cellChunkSize * totDim * totDim)); 5911 PetscCall(PetscArrayzero(elemMatPos, cellChunkSize * totDim * totDim)); 5912 PetscCall(PetscArrayzero(elemMatCoh, cellChunkSize * totDim * totDim)); 5913 } 5914 if (hasBdPrec) { 5915 PetscCall(PetscArrayzero(elemMatNegP, cellChunkSize * totDim * totDim)); 5916 PetscCall(PetscArrayzero(elemMatPosP, cellChunkSize * totDim * totDim)); 5917 PetscCall(PetscArrayzero(elemMatCohP, cellChunkSize * totDim * totDim)); 5918 } 5919 /* Get faces */ 5920 for (c = cS; c < cE; ++c) { 5921 const PetscInt cell = cells ? cells[c] : c; 5922 const PetscInt *cone; 5923 PetscCall(DMPlexGetCone(plex, cell, &cone)); 5924 faces[(c - cS) * 2 + 0] = cone[0]; 5925 faces[(c - cS) * 2 + 1] = cone[1]; 5926 } 5927 PetscCall(ISGeneralSetIndices(chunkIS, 2 * cellChunkSize, faces, PETSC_USE_POINTER)); 5928 if (maxDegree <= 1) { 5929 if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad)); 5930 if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom)); 5931 } else { 5932 PetscInt f; 5933 for (f = 0; f < Nf; ++f) { 5934 if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f])); 5935 } 5936 } 5937 5938 for (fieldI = 0; fieldI < Nf; ++fieldI) { 5939 PetscFE feI; 5940 PetscFEGeom *geom = affineGeom ? affineGeom : geoms[fieldI]; 5941 PetscFEGeom *chunkGeom = NULL, *remGeom = NULL; 5942 PetscQuadrature quad = affineQuad ? affineQuad : quads[fieldI]; 5943 PetscInt numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb; 5944 PetscBool isCohesiveField; 5945 5946 PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&feI)); 5947 if (!feI) continue; 5948 PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches)); 5949 PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL)); 5950 PetscCall(PetscFEGetDimension(feI, &Nb)); 5951 blockSize = Nb; 5952 batchSize = numBlocks * blockSize; 5953 PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches)); 5954 numChunks = numCells / (numBatches * batchSize); 5955 Ne = numChunks * numBatches * batchSize; 5956 Nr = numCells % (numBatches * batchSize); 5957 offset = numCells - Nr; 5958 PetscCall(PetscFEGeomGetChunk(geom, 0, offset * 2, &chunkGeom)); 5959 PetscCall(PetscFEGeomGetChunk(geom, offset * 2, numCells * 2, &remGeom)); 5960 PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField)); 5961 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 5962 PetscFE feJ; 5963 5964 PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *)&feJ)); 5965 if (!feJ) continue; 5966 key[0].field = fieldI * Nf + fieldJ; 5967 key[1].field = fieldI * Nf + fieldJ; 5968 key[2].field = fieldI * Nf + fieldJ; 5969 if (hasBdJac) { 5970 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNeg)); 5971 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMatNeg[offset * totDim * totDim])); 5972 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPos)); 5973 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMatPos[offset * totDim * totDim])); 5974 } 5975 if (hasBdPrec) { 5976 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatNegP)); 5977 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMatNegP[offset * totDim * totDim])); 5978 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatPosP)); 5979 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMatPosP[offset * totDim * totDim])); 5980 } 5981 if (hasBdJac) { 5982 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCoh)); 5983 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMatCoh[offset * totDim * totDim])); 5984 } 5985 if (hasBdPrec) { 5986 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatCohP)); 5987 PetscCall(PetscFEIntegrateHybridJacobian(ds, dsIn, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeom, &u[offset * totDimIn], u_t ? &u_t[offset * totDimIn] : NULL, dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMatCohP[offset * totDim * totDim])); 5988 } 5989 } 5990 PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &remGeom)); 5991 PetscCall(PetscFEGeomRestoreChunk(geom, 0, offset, &chunkGeom)); 5992 } 5993 /* Insert values into matrix */ 5994 for (c = cS; c < cE; ++c) { 5995 const PetscInt cell = cells ? cells[c] : c; 5996 const PetscInt cind = c - cS, coff = cind * totDim * totDim; 5997 PetscInt i, j; 5998 5999 /* Scale element values */ 6000 if (locS[0]) { 6001 PetscInt Nb, soff = cind * totDimScale[0], off = 0; 6002 PetscBool cohesive; 6003 6004 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6005 PetscCall(PetscDSGetFieldSize(ds, fieldI, &Nb)); 6006 PetscCall(PetscDSGetCohesive(ds, fieldI, &cohesive)); 6007 6008 if (fieldI == key[2].field) { 6009 PetscCheck(cohesive, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Scaling should not happen for face fields"); 6010 for (i = 0; i < Nb; ++i) { 6011 for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNeg[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPos[coff + (off + i) * totDim + j]; 6012 if (hasBdPrec) 6013 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += s[0][soff + i] * elemMatNegP[coff + (off + i) * totDim + j] + s[1][soff + i] * elemMatPosP[coff + (off + i) * totDim + j]; 6014 } 6015 off += Nb; 6016 } else { 6017 const PetscInt N = cohesive ? Nb : Nb * 2; 6018 6019 for (i = 0; i < N; ++i) { 6020 for (j = 0; j < totDim; ++j) elemMatCoh[coff + (off + i) * totDim + j] += elemMatNeg[coff + (off + i) * totDim + j] + elemMatPos[coff + (off + i) * totDim + j]; 6021 if (hasBdPrec) 6022 for (j = 0; j < totDim; ++j) elemMatCohP[coff + (off + i) * totDim + j] += elemMatNegP[coff + (off + i) * totDim + j] + elemMatPosP[coff + (off + i) * totDim + j]; 6023 } 6024 off += N; 6025 } 6026 } 6027 } else { 6028 for (i = 0; i < totDim * totDim; ++i) elemMatCoh[coff + i] += elemMatNeg[coff + i] + elemMatPos[coff + i]; 6029 if (hasBdPrec) 6030 for (i = 0; i < totDim * totDim; ++i) elemMatCohP[coff + i] += elemMatNegP[coff + i] + elemMatPosP[coff + i]; 6031 } 6032 if (hasBdPrec) { 6033 if (hasBdJac) { 6034 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim])); 6035 PetscCall(DMPlexMatSetClosure(plex, section, globalSection, Jac, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES)); 6036 } 6037 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCohP[cind * totDim * totDim])); 6038 PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatCohP[cind * totDim * totDim], ADD_VALUES)); 6039 } else if (hasBdJac) { 6040 if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatCoh[cind * totDim * totDim])); 6041 PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatCoh[cind * totDim * totDim], ADD_VALUES)); 6042 } 6043 } 6044 } 6045 PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2])); 6046 PetscCall(DMPlexRestoreHybridFields(dm, dmAux, dsAux, cellIS, locA, PETSC_TRUE, a)); 6047 PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNeg)); 6048 PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPos)); 6049 PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCoh)); 6050 PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatNegP)); 6051 PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatPosP)); 6052 PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatCohP)); 6053 PetscCall(PetscFree(faces)); 6054 PetscCall(ISDestroy(&chunkIS)); 6055 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 6056 if (maxDegree <= 1) { 6057 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom)); 6058 PetscCall(PetscQuadratureDestroy(&affineQuad)); 6059 } else { 6060 PetscInt f; 6061 for (f = 0; f < Nf; ++f) { 6062 if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f])); 6063 if (quads) PetscCall(PetscQuadratureDestroy(&quads[f])); 6064 } 6065 PetscCall(PetscFree2(quads, geoms)); 6066 } 6067 if (dmAux[2]) PetscCall(DMDestroy(&plexA)); 6068 PetscCall(DMDestroy(&plex)); 6069 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6070 PetscFunctionReturn(PETSC_SUCCESS); 6071 } 6072 6073 /* 6074 DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user. 6075 6076 Input Parameters: 6077 + dm - The mesh 6078 . key - The PetscWeakFormKey indcating where integration should happen 6079 . cellIS - The cells to integrate over 6080 . t - The time 6081 . X_tShift - The multiplier for the Jacobian with repsect to X_t 6082 . X - Local solution vector 6083 . X_t - Time-derivative of the local solution vector 6084 . Y - Local input vector 6085 - user - the user context 6086 6087 Output Parameter: 6088 . Z - Local output vector 6089 6090 Note: 6091 We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator, 6092 like a GPU, or vectorize on a multicore machine. 6093 */ 6094 PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user) 6095 { 6096 DM_Plex *mesh = (DM_Plex *)dm->data; 6097 const char *name = "Jacobian"; 6098 DM dmAux = NULL, plex, plexAux = NULL; 6099 DMEnclosureType encAux; 6100 Vec A; 6101 DMField coordField; 6102 PetscDS prob, probAux = NULL; 6103 PetscQuadrature quad; 6104 PetscSection section, globalSection, sectionAux; 6105 PetscScalar *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z; 6106 const PetscInt *cells; 6107 PetscInt Nf, fieldI, fieldJ; 6108 PetscInt totDim, totDimAux = 0, cStart, cEnd, numCells, c; 6109 PetscBool hasDyn; 6110 6111 PetscFunctionBegin; 6112 PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6113 PetscCall(DMConvert(dm, DMPLEX, &plex)); 6114 if (!cellIS) { 6115 PetscInt depth; 6116 6117 PetscCall(DMPlexGetDepth(plex, &depth)); 6118 PetscCall(DMGetStratumIS(plex, "dim", depth, &cellIS)); 6119 if (!cellIS) PetscCall(DMGetStratumIS(plex, "depth", depth, &cellIS)); 6120 } else { 6121 PetscCall(PetscObjectReference((PetscObject)cellIS)); 6122 } 6123 PetscCall(ISGetLocalSize(cellIS, &numCells)); 6124 PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells)); 6125 PetscCall(DMGetLocalSection(dm, §ion)); 6126 PetscCall(DMGetGlobalSection(dm, &globalSection)); 6127 PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob, NULL)); 6128 PetscCall(PetscDSGetNumFields(prob, &Nf)); 6129 PetscCall(PetscDSGetTotalDimension(prob, &totDim)); 6130 PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn)); 6131 hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE; 6132 PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A)); 6133 if (A) { 6134 PetscCall(VecGetDM(A, &dmAux)); 6135 PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux)); 6136 PetscCall(DMConvert(dmAux, DMPLEX, &plexAux)); 6137 PetscCall(DMGetLocalSection(plexAux, §ionAux)); 6138 PetscCall(DMGetDS(dmAux, &probAux)); 6139 PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux)); 6140 } 6141 PetscCall(VecSet(Z, 0.0)); 6142 PetscCall(PetscMalloc6(numCells * totDim, &u, X_t ? numCells * totDim : 0, &u_t, numCells * totDim * totDim, &elemMat, hasDyn ? numCells * totDim * totDim : 0, &elemMatD, numCells * totDim, &y, totDim, &z)); 6143 if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a)); 6144 PetscCall(DMGetCoordinateField(dm, &coordField)); 6145 for (c = cStart; c < cEnd; ++c) { 6146 const PetscInt cell = cells ? cells[c] : c; 6147 const PetscInt cind = c - cStart; 6148 PetscScalar *x = NULL, *x_t = NULL; 6149 PetscInt i; 6150 6151 PetscCall(DMPlexVecGetClosure(plex, section, X, cell, NULL, &x)); 6152 for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i]; 6153 PetscCall(DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x)); 6154 if (X_t) { 6155 PetscCall(DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t)); 6156 for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i]; 6157 PetscCall(DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t)); 6158 } 6159 if (dmAux) { 6160 PetscInt subcell; 6161 PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell)); 6162 PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x)); 6163 for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i]; 6164 PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x)); 6165 } 6166 PetscCall(DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x)); 6167 for (i = 0; i < totDim; ++i) y[cind * totDim + i] = x[i]; 6168 PetscCall(DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x)); 6169 } 6170 PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim)); 6171 if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim)); 6172 for (fieldI = 0; fieldI < Nf; ++fieldI) { 6173 PetscFE fe; 6174 PetscInt Nb; 6175 /* Conforming batches */ 6176 PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize; 6177 /* Remainder */ 6178 PetscInt Nr, offset, Nq; 6179 PetscQuadrature qGeom = NULL; 6180 PetscInt maxDegree; 6181 PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL; 6182 6183 PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe)); 6184 PetscCall(PetscFEGetQuadrature(fe, &quad)); 6185 PetscCall(PetscFEGetDimension(fe, &Nb)); 6186 PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches)); 6187 PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree)); 6188 if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom)); 6189 if (!qGeom) { 6190 PetscCall(PetscFEGetQuadrature(fe, &qGeom)); 6191 PetscCall(PetscObjectReference((PetscObject)qGeom)); 6192 } 6193 PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL)); 6194 PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 6195 blockSize = Nb; 6196 batchSize = numBlocks * blockSize; 6197 PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches)); 6198 numChunks = numCells / (numBatches * batchSize); 6199 Ne = numChunks * numBatches * batchSize; 6200 Nr = numCells % (numBatches * batchSize); 6201 offset = numCells - Nr; 6202 PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom)); 6203 PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom)); 6204 for (fieldJ = 0; fieldJ < Nf; ++fieldJ) { 6205 key.field = fieldI * Nf + fieldJ; 6206 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat)); 6207 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMat[offset * totDim * totDim])); 6208 if (hasDyn) { 6209 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD)); 6210 PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim])); 6211 } 6212 } 6213 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom)); 6214 PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom)); 6215 PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM)); 6216 PetscCall(PetscQuadratureDestroy(&qGeom)); 6217 } 6218 if (hasDyn) { 6219 for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c]; 6220 } 6221 for (c = cStart; c < cEnd; ++c) { 6222 const PetscInt cell = cells ? cells[c] : c; 6223 const PetscInt cind = c - cStart; 6224 const PetscBLASInt M = totDim, one = 1; 6225 const PetscScalar a = 1.0, b = 0.0; 6226 6227 PetscCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind * totDim * totDim], &M, &y[cind * totDim], &one, &b, z, &one)); 6228 if (mesh->printFEM > 1) { 6229 PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind * totDim * totDim])); 6230 PetscCall(DMPrintCellVector(c, "Y", totDim, &y[cind * totDim])); 6231 PetscCall(DMPrintCellVector(c, "Z", totDim, z)); 6232 } 6233 PetscCall(DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES)); 6234 } 6235 PetscCall(PetscFree6(u, u_t, elemMat, elemMatD, y, z)); 6236 if (mesh->printFEM) { 6237 PetscCall(PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n")); 6238 PetscCall(VecView(Z, NULL)); 6239 } 6240 PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells)); 6241 PetscCall(PetscFree(a)); 6242 PetscCall(ISDestroy(&cellIS)); 6243 PetscCall(DMDestroy(&plexAux)); 6244 PetscCall(DMDestroy(&plex)); 6245 PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0)); 6246 PetscFunctionReturn(PETSC_SUCCESS); 6247 } 6248