xref: /petsc/src/dm/impls/plex/plexfem.c (revision 362febeeeb69b91ebadcb4b2dc0a22cb6dfc4097)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2da97024aSMatthew G. Knepley #include <petscsf.h>
3cb1e1211SMatthew G Knepley 
48e3a2eefSMatthew G. Knepley #include <petscblaslapack.h>
5e8f14785SLisandro Dalcin #include <petsc/private/hashsetij.h>
6af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
7af0996ceSBarry Smith #include <petsc/private/petscfvimpl.h>
8a0845e3aSMatthew G. Knepley 
92f856554SMatthew G. Knepley static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy)
102f856554SMatthew G. Knepley {
112f856554SMatthew G. Knepley   PetscBool      isPlex;
122f856554SMatthew G. Knepley   PetscErrorCode ierr;
132f856554SMatthew G. Knepley 
142f856554SMatthew G. Knepley   PetscFunctionBegin;
152f856554SMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex);CHKERRQ(ierr);
162f856554SMatthew G. Knepley   if (isPlex) {
172f856554SMatthew G. Knepley     *plex = dm;
182f856554SMatthew G. Knepley     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr);
192f856554SMatthew G. Knepley   } else {
202f856554SMatthew G. Knepley     ierr = PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex);CHKERRQ(ierr);
212f856554SMatthew G. Knepley     if (!*plex) {
222f856554SMatthew G. Knepley       ierr = DMConvert(dm, DMPLEX, plex);CHKERRQ(ierr);
232f856554SMatthew G. Knepley       ierr = PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex);CHKERRQ(ierr);
242f856554SMatthew G. Knepley       if (copy) {
252f856554SMatthew G. Knepley         DMSubDomainHookLink link;
269a2a23afSMatthew G. Knepley 
279a2a23afSMatthew G. Knepley         ierr = DMCopyAuxiliaryVec(dm, *plex);CHKERRQ(ierr);
289a2a23afSMatthew G. Knepley         /* Run the subdomain hook (this will copy the DMSNES/DMTS) */
292f856554SMatthew G. Knepley         for (link = dm->subdomainhook; link; link = link->next) {
302f856554SMatthew G. Knepley           if (link->ddhook) {ierr = (*link->ddhook)(dm, *plex, link->ctx);CHKERRQ(ierr);}
312f856554SMatthew G. Knepley         }
322f856554SMatthew G. Knepley       }
332f856554SMatthew G. Knepley     } else {
342f856554SMatthew G. Knepley       ierr = PetscObjectReference((PetscObject) *plex);CHKERRQ(ierr);
352f856554SMatthew G. Knepley     }
362f856554SMatthew G. Knepley   }
372f856554SMatthew G. Knepley   PetscFunctionReturn(0);
382f856554SMatthew G. Knepley }
392f856554SMatthew G. Knepley 
409b6f715bSMatthew G. Knepley static PetscErrorCode PetscContainerUserDestroy_PetscFEGeom (void *ctx)
419b6f715bSMatthew G. Knepley {
429b6f715bSMatthew G. Knepley   PetscFEGeom *geom = (PetscFEGeom *) ctx;
439b6f715bSMatthew G. Knepley   PetscErrorCode ierr;
449b6f715bSMatthew G. Knepley 
459b6f715bSMatthew G. Knepley   PetscFunctionBegin;
469b6f715bSMatthew G. Knepley   ierr = PetscFEGeomDestroy(&geom);CHKERRQ(ierr);
479b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
489b6f715bSMatthew G. Knepley }
499b6f715bSMatthew G. Knepley 
509b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
519b6f715bSMatthew G. Knepley {
529b6f715bSMatthew G. Knepley   char            composeStr[33] = {0};
539b6f715bSMatthew G. Knepley   PetscObjectId   id;
549b6f715bSMatthew G. Knepley   PetscContainer  container;
559b6f715bSMatthew G. Knepley   PetscErrorCode  ierr;
569b6f715bSMatthew G. Knepley 
579b6f715bSMatthew G. Knepley   PetscFunctionBegin;
589b6f715bSMatthew G. Knepley   ierr = PetscObjectGetId((PetscObject)quad,&id);CHKERRQ(ierr);
599b6f715bSMatthew G. Knepley   ierr = PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%x\n", id);CHKERRQ(ierr);
609b6f715bSMatthew G. Knepley   ierr = PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container);CHKERRQ(ierr);
619b6f715bSMatthew G. Knepley   if (container) {
629b6f715bSMatthew G. Knepley     ierr = PetscContainerGetPointer(container, (void **) geom);CHKERRQ(ierr);
639b6f715bSMatthew G. Knepley   } else {
649b6f715bSMatthew G. Knepley     ierr = DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom);CHKERRQ(ierr);
659b6f715bSMatthew G. Knepley     ierr = PetscContainerCreate(PETSC_COMM_SELF,&container);CHKERRQ(ierr);
669b6f715bSMatthew G. Knepley     ierr = PetscContainerSetPointer(container, (void *) *geom);CHKERRQ(ierr);
679b6f715bSMatthew G. Knepley     ierr = PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom);CHKERRQ(ierr);
689b6f715bSMatthew G. Knepley     ierr = PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container);CHKERRQ(ierr);
699b6f715bSMatthew G. Knepley     ierr = PetscContainerDestroy(&container);CHKERRQ(ierr);
709b6f715bSMatthew G. Knepley   }
719b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
729b6f715bSMatthew G. Knepley }
739b6f715bSMatthew G. Knepley 
749b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
759b6f715bSMatthew G. Knepley {
769b6f715bSMatthew G. Knepley   PetscFunctionBegin;
779b6f715bSMatthew G. Knepley   *geom = NULL;
789b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
799b6f715bSMatthew G. Knepley }
809b6f715bSMatthew G. Knepley 
8146fa42a0SMatthew G. Knepley /*@
8246fa42a0SMatthew G. Knepley   DMPlexGetScale - Get the scale for the specified fundamental unit
8346fa42a0SMatthew G. Knepley 
8446fa42a0SMatthew G. Knepley   Not collective
8546fa42a0SMatthew G. Knepley 
8646fa42a0SMatthew G. Knepley   Input Arguments:
8746fa42a0SMatthew G. Knepley + dm   - the DM
8846fa42a0SMatthew G. Knepley - unit - The SI unit
8946fa42a0SMatthew G. Knepley 
9046fa42a0SMatthew G. Knepley   Output Argument:
9146fa42a0SMatthew G. Knepley . scale - The value used to scale all quantities with this unit
9246fa42a0SMatthew G. Knepley 
9346fa42a0SMatthew G. Knepley   Level: advanced
9446fa42a0SMatthew G. Knepley 
9546fa42a0SMatthew G. Knepley .seealso: DMPlexSetScale(), PetscUnit
9646fa42a0SMatthew G. Knepley @*/
97cb1e1211SMatthew G Knepley PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale)
98cb1e1211SMatthew G Knepley {
99cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex*) dm->data;
100cb1e1211SMatthew G Knepley 
101cb1e1211SMatthew G Knepley   PetscFunctionBegin;
102cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
103cb1e1211SMatthew G Knepley   PetscValidPointer(scale, 3);
104cb1e1211SMatthew G Knepley   *scale = mesh->scale[unit];
105cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
106cb1e1211SMatthew G Knepley }
107cb1e1211SMatthew G Knepley 
10846fa42a0SMatthew G. Knepley /*@
10946fa42a0SMatthew G. Knepley   DMPlexSetScale - Set the scale for the specified fundamental unit
11046fa42a0SMatthew G. Knepley 
11146fa42a0SMatthew G. Knepley   Not collective
11246fa42a0SMatthew G. Knepley 
11346fa42a0SMatthew G. Knepley   Input Arguments:
11446fa42a0SMatthew G. Knepley + dm   - the DM
11546fa42a0SMatthew G. Knepley . unit - The SI unit
11646fa42a0SMatthew G. Knepley - scale - The value used to scale all quantities with this unit
11746fa42a0SMatthew G. Knepley 
11846fa42a0SMatthew G. Knepley   Level: advanced
11946fa42a0SMatthew G. Knepley 
12046fa42a0SMatthew G. Knepley .seealso: DMPlexGetScale(), PetscUnit
12146fa42a0SMatthew G. Knepley @*/
122cb1e1211SMatthew G Knepley PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale)
123cb1e1211SMatthew G Knepley {
124cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex*) dm->data;
125cb1e1211SMatthew G Knepley 
126cb1e1211SMatthew G Knepley   PetscFunctionBegin;
127cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
128cb1e1211SMatthew G Knepley   mesh->scale[unit] = scale;
129cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
130cb1e1211SMatthew G Knepley }
131cb1e1211SMatthew G Knepley 
132327779f2SMatthew Knepley static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx)
133026175e5SToby Isaac {
13412adca46SMatthew G. Knepley   const PetscInt eps[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
135026175e5SToby Isaac   PetscInt *ctxInt  = (PetscInt *) ctx;
136ad917190SMatthew G. Knepley   PetscInt  dim2    = ctxInt[0];
137026175e5SToby Isaac   PetscInt  d       = ctxInt[1];
138026175e5SToby Isaac   PetscInt  i, j, k = dim > 2 ? d - dim : d;
139026175e5SToby Isaac 
140ad917190SMatthew G. Knepley   PetscFunctionBegin;
141ff1e0c32SBarry Smith   if (dim != dim2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %D does not match context dimension %D", dim, dim2);
142026175e5SToby Isaac   for (i = 0; i < dim; i++) mode[i] = 0.;
143026175e5SToby Isaac   if (d < dim) {
14412adca46SMatthew G. Knepley     mode[d] = 1.; /* Translation along axis d */
145ad917190SMatthew G. Knepley   } else {
146026175e5SToby Isaac     for (i = 0; i < dim; i++) {
147026175e5SToby Isaac       for (j = 0; j < dim; j++) {
14812adca46SMatthew G. Knepley         mode[j] += eps[i][j][k]*X[i]; /* Rotation about axis d */
149026175e5SToby Isaac       }
150026175e5SToby Isaac     }
151026175e5SToby Isaac   }
152ad917190SMatthew G. Knepley   PetscFunctionReturn(0);
153026175e5SToby Isaac }
154026175e5SToby Isaac 
155cc4e42d9SMartin Diehl /*@
15612adca46SMatthew G. Knepley   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation
157cb1e1211SMatthew G Knepley 
158d083f849SBarry Smith   Collective on dm
159cb1e1211SMatthew G Knepley 
160cb1e1211SMatthew G Knepley   Input Arguments:
16156cf3b9cSMatthew G. Knepley + dm - the DM
16256cf3b9cSMatthew G. Knepley - field - The field number for the rigid body space, or 0 for the default
163cb1e1211SMatthew G Knepley 
164cb1e1211SMatthew G Knepley   Output Argument:
165cb1e1211SMatthew G Knepley . sp - the null space
166cb1e1211SMatthew G Knepley 
16712adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
168cb1e1211SMatthew G Knepley 
169cb1e1211SMatthew G Knepley   Level: advanced
170cb1e1211SMatthew G Knepley 
17112adca46SMatthew G. Knepley .seealso: MatNullSpaceCreate(), PCGAMG
172cb1e1211SMatthew G Knepley @*/
17356cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp)
174cb1e1211SMatthew G Knepley {
17556cf3b9cSMatthew G. Knepley   PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
176cb1e1211SMatthew G Knepley   MPI_Comm          comm;
177026175e5SToby Isaac   Vec               mode[6];
178026175e5SToby Isaac   PetscSection      section, globalSection;
17956cf3b9cSMatthew G. Knepley   PetscInt          dim, dimEmbed, Nf, n, m, mmin, d, i, j;
180cb1e1211SMatthew G Knepley   PetscErrorCode    ierr;
181cb1e1211SMatthew G Knepley 
182cb1e1211SMatthew G Knepley   PetscFunctionBegin;
183cb1e1211SMatthew G Knepley   ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
184c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1859d8fbdeaSMatthew G. Knepley   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
18656cf3b9cSMatthew G. Knepley   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
18756cf3b9cSMatthew G. Knepley   if (Nf && (field < 0 || field >= Nf)) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %D is not in [0, Nf)", field, Nf);
18856cf3b9cSMatthew G. Knepley   if (dim == 1 && Nf < 2) {
189cb1e1211SMatthew G Knepley     ierr = MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp);CHKERRQ(ierr);
190cb1e1211SMatthew G Knepley     PetscFunctionReturn(0);
191cb1e1211SMatthew G Knepley   }
19292fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
193e87a4003SBarry Smith   ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);
194cb1e1211SMatthew G Knepley   ierr = PetscSectionGetConstrainedStorageSize(globalSection, &n);CHKERRQ(ierr);
19556cf3b9cSMatthew G. Knepley   ierr = PetscCalloc1(Nf, &func);CHKERRQ(ierr);
196b247467aSMatthew G. Knepley   m    = (dim*(dim+1))/2;
197cb1e1211SMatthew G Knepley   ierr = VecCreate(comm, &mode[0]);CHKERRQ(ierr);
198cb1e1211SMatthew G Knepley   ierr = VecSetSizes(mode[0], n, PETSC_DETERMINE);CHKERRQ(ierr);
199cb1e1211SMatthew G Knepley   ierr = VecSetUp(mode[0]);CHKERRQ(ierr);
200b247467aSMatthew G. Knepley   ierr = VecGetSize(mode[0], &n);CHKERRQ(ierr);
201b247467aSMatthew G. Knepley   mmin = PetscMin(m, n);
20256cf3b9cSMatthew G. Knepley   func[field] = DMPlexProjectRigidBody_Private;
203cb1e1211SMatthew G Knepley   for (i = 1; i < m; ++i) {ierr = VecDuplicate(mode[0], &mode[i]);CHKERRQ(ierr);}
204026175e5SToby Isaac   for (d = 0; d < m; d++) {
205330485fdSToby Isaac     PetscInt ctx[2];
206026175e5SToby Isaac     void    *voidctx = (void *) (&ctx[0]);
207cb1e1211SMatthew G Knepley 
2089d8fbdeaSMatthew G. Knepley     ctx[0] = dimEmbed;
209330485fdSToby Isaac     ctx[1] = d;
21056cf3b9cSMatthew G. Knepley     ierr = DMProjectFunction(dm, 0.0, func, &voidctx, INSERT_VALUES, mode[d]);CHKERRQ(ierr);
211cb1e1211SMatthew G Knepley   }
2123b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
213b50a2c0aSJacob Faibussowitsch   for (i = 0; i < mmin; ++i) {
214b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
215b50a2c0aSJacob Faibussowitsch 
2169019a6d7SJacob Faibussowitsch     ierr = VecNormalize(mode[i], NULL);CHKERRQ(ierr);
217b77f2eeeSJacob Faibussowitsch     ierr = VecMDot(mode[i], mmin-i-1, mode+i+1, dots+i+1);CHKERRQ(ierr);
218b50a2c0aSJacob Faibussowitsch     for (j = i+1; j < mmin; ++j) {
219b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
220b77f2eeeSJacob Faibussowitsch       ierr = VecAXPY(mode[j], dots[j], mode[i]);CHKERRQ(ierr);
221b50a2c0aSJacob Faibussowitsch     }
222cb1e1211SMatthew G Knepley   }
223b247467aSMatthew G. Knepley   ierr = MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp);CHKERRQ(ierr);
224b247467aSMatthew G. Knepley   for (i = 0; i < m; ++i) {ierr = VecDestroy(&mode[i]);CHKERRQ(ierr);}
22556cf3b9cSMatthew G. Knepley   ierr = PetscFree(func);CHKERRQ(ierr);
226cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
227cb1e1211SMatthew G Knepley }
228cb1e1211SMatthew G Knepley 
229cc4e42d9SMartin Diehl /*@
23012adca46SMatthew G. Knepley   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation
23112adca46SMatthew G. Knepley 
232d083f849SBarry Smith   Collective on dm
23312adca46SMatthew G. Knepley 
23412adca46SMatthew G. Knepley   Input Arguments:
23512adca46SMatthew G. Knepley + dm    - the DM
23612adca46SMatthew G. Knepley . nb    - The number of bodies
23712adca46SMatthew G. Knepley . label - The DMLabel marking each domain
23812adca46SMatthew G. Knepley . nids  - The number of ids per body
23912adca46SMatthew G. Knepley - ids   - An array of the label ids in sequence for each domain
24012adca46SMatthew G. Knepley 
24112adca46SMatthew G. Knepley   Output Argument:
24212adca46SMatthew G. Knepley . sp - the null space
24312adca46SMatthew G. Knepley 
24412adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
24512adca46SMatthew G. Knepley 
24612adca46SMatthew G. Knepley   Level: advanced
24712adca46SMatthew G. Knepley 
24812adca46SMatthew G. Knepley .seealso: MatNullSpaceCreate()
24912adca46SMatthew G. Knepley @*/
25012adca46SMatthew G. Knepley PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp)
25112adca46SMatthew G. Knepley {
25212adca46SMatthew G. Knepley   MPI_Comm       comm;
25312adca46SMatthew G. Knepley   PetscSection   section, globalSection;
25412adca46SMatthew G. Knepley   Vec           *mode;
25512adca46SMatthew G. Knepley   PetscScalar   *dots;
25612adca46SMatthew G. Knepley   PetscInt       dim, dimEmbed, n, m, b, d, i, j, off;
25712adca46SMatthew G. Knepley   PetscErrorCode ierr;
25812adca46SMatthew G. Knepley 
25912adca46SMatthew G. Knepley   PetscFunctionBegin;
26012adca46SMatthew G. Knepley   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
26112adca46SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
26212adca46SMatthew G. Knepley   ierr = DMGetCoordinateDim(dm, &dimEmbed);CHKERRQ(ierr);
26392fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
264e87a4003SBarry Smith   ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);
26512adca46SMatthew G. Knepley   ierr = PetscSectionGetConstrainedStorageSize(globalSection, &n);CHKERRQ(ierr);
26612adca46SMatthew G. Knepley   m    = nb * (dim*(dim+1))/2;
26712adca46SMatthew G. Knepley   ierr = PetscMalloc2(m, &mode, m, &dots);CHKERRQ(ierr);
26812adca46SMatthew G. Knepley   ierr = VecCreate(comm, &mode[0]);CHKERRQ(ierr);
26912adca46SMatthew G. Knepley   ierr = VecSetSizes(mode[0], n, PETSC_DETERMINE);CHKERRQ(ierr);
27012adca46SMatthew G. Knepley   ierr = VecSetUp(mode[0]);CHKERRQ(ierr);
27112adca46SMatthew G. Knepley   for (i = 1; i < m; ++i) {ierr = VecDuplicate(mode[0], &mode[i]);CHKERRQ(ierr);}
27212adca46SMatthew G. Knepley   for (b = 0, off = 0; b < nb; ++b) {
27312adca46SMatthew G. Knepley     for (d = 0; d < m/nb; ++d) {
27412adca46SMatthew G. Knepley       PetscInt         ctx[2];
27512adca46SMatthew G. Knepley       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
27612adca46SMatthew G. Knepley       void            *voidctx = (void *) (&ctx[0]);
27712adca46SMatthew G. Knepley 
27812adca46SMatthew G. Knepley       ctx[0] = dimEmbed;
27912adca46SMatthew G. Knepley       ctx[1] = d;
28012adca46SMatthew G. Knepley       ierr = DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]);CHKERRQ(ierr);
28112adca46SMatthew G. Knepley       off   += nids[b];
28212adca46SMatthew G. Knepley     }
28312adca46SMatthew G. Knepley   }
2843b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
285606c1a1cSJacob Faibussowitsch   for (i = 0; i < m; ++i) {
286b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
2875a0e29b9SJacob Faibussowitsch 
2889019a6d7SJacob Faibussowitsch     ierr = VecNormalize(mode[i], NULL);CHKERRQ(ierr);
289b77f2eeeSJacob Faibussowitsch     ierr = VecMDot(mode[i], m-i-1, mode+i+1, dots+i+1);CHKERRQ(ierr);
2905a0e29b9SJacob Faibussowitsch     for (j = i+1; j < m; ++j) {
291b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
292b77f2eeeSJacob Faibussowitsch       ierr = VecAXPY(mode[j], dots[j], mode[i]);CHKERRQ(ierr);
2935a0e29b9SJacob Faibussowitsch     }
29412adca46SMatthew G. Knepley   }
29512adca46SMatthew G. Knepley   ierr = MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp);CHKERRQ(ierr);
29612adca46SMatthew G. Knepley   for (i = 0; i< m; ++i) {ierr = VecDestroy(&mode[i]);CHKERRQ(ierr);}
29712adca46SMatthew G. Knepley   ierr = PetscFree2(mode, dots);CHKERRQ(ierr);
29812adca46SMatthew G. Knepley   PetscFunctionReturn(0);
29912adca46SMatthew G. Knepley }
30012adca46SMatthew G. Knepley 
301b29cfa1cSToby Isaac /*@
302b29cfa1cSToby Isaac   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
303b29cfa1cSToby Isaac   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
304b29cfa1cSToby Isaac   evaluating the dual space basis of that point.  A basis function is associated with the point in its
305b29cfa1cSToby Isaac   transitively-closed support whose mesh height is highest (w.r.t. DAG height), but not greater than the maximum
306b29cfa1cSToby Isaac   projection height, which is set with this function.  By default, the maximum projection height is zero, which means
307b29cfa1cSToby Isaac   that only mesh cells are used to project basis functions.  A height of one, for example, evaluates a cell-interior
308b29cfa1cSToby Isaac   basis functions using its cells dual space basis, but all other basis functions with the dual space basis of a face.
309b29cfa1cSToby Isaac 
310b29cfa1cSToby Isaac   Input Parameters:
311b29cfa1cSToby Isaac + dm - the DMPlex object
312b29cfa1cSToby Isaac - height - the maximum projection height >= 0
313b29cfa1cSToby Isaac 
314b29cfa1cSToby Isaac   Level: advanced
315b29cfa1cSToby Isaac 
3164d6f44ffSToby Isaac .seealso: DMPlexGetMaxProjectionHeight(), DMProjectFunctionLocal(), DMProjectFunctionLabelLocal()
317b29cfa1cSToby Isaac @*/
318b29cfa1cSToby Isaac PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height)
319b29cfa1cSToby Isaac {
320b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *) dm->data;
321b29cfa1cSToby Isaac 
322b29cfa1cSToby Isaac   PetscFunctionBegin;
323b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
324b29cfa1cSToby Isaac   plex->maxProjectionHeight = height;
325b29cfa1cSToby Isaac   PetscFunctionReturn(0);
326b29cfa1cSToby Isaac }
327b29cfa1cSToby Isaac 
328b29cfa1cSToby Isaac /*@
329b29cfa1cSToby Isaac   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
330b29cfa1cSToby Isaac   DMPlexProjectXXXLocal() functions.
331b29cfa1cSToby Isaac 
332b29cfa1cSToby Isaac   Input Parameters:
333b29cfa1cSToby Isaac . dm - the DMPlex object
334b29cfa1cSToby Isaac 
335b29cfa1cSToby Isaac   Output Parameters:
336b29cfa1cSToby Isaac . height - the maximum projection height
337b29cfa1cSToby Isaac 
338b29cfa1cSToby Isaac   Level: intermediate
339b29cfa1cSToby Isaac 
3404d6f44ffSToby Isaac .seealso: DMPlexSetMaxProjectionHeight(), DMProjectFunctionLocal(), DMProjectFunctionLabelLocal()
341b29cfa1cSToby Isaac @*/
342b29cfa1cSToby Isaac PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height)
343b29cfa1cSToby Isaac {
344b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *) dm->data;
345b29cfa1cSToby Isaac 
346b29cfa1cSToby Isaac   PetscFunctionBegin;
347b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
348b29cfa1cSToby Isaac   *height = plex->maxProjectionHeight;
349b29cfa1cSToby Isaac   PetscFunctionReturn(0);
350b29cfa1cSToby Isaac }
351b29cfa1cSToby Isaac 
352ca3d3a14SMatthew G. Knepley typedef struct {
353ca3d3a14SMatthew G. Knepley   PetscReal    alpha; /* The first Euler angle, and in 2D the only one */
354ca3d3a14SMatthew G. Knepley   PetscReal    beta;  /* The second Euler angle */
355ca3d3a14SMatthew G. Knepley   PetscReal    gamma; /* The third Euler angle */
356ca3d3a14SMatthew G. Knepley   PetscInt     dim;   /* The dimension of R */
357ca3d3a14SMatthew G. Knepley   PetscScalar *R;     /* The rotation matrix, transforming a vector in the local basis to the global basis */
358ca3d3a14SMatthew G. Knepley   PetscScalar *RT;    /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */
359ca3d3a14SMatthew G. Knepley } RotCtx;
360ca3d3a14SMatthew G. Knepley 
361ca3d3a14SMatthew G. Knepley /*
362ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
363ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
364ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
365ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
366ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
367ca3d3a14SMatthew G. Knepley */
368ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx)
369ca3d3a14SMatthew G. Knepley {
370ca3d3a14SMatthew G. Knepley   RotCtx        *rc  = (RotCtx *) ctx;
371ca3d3a14SMatthew G. Knepley   PetscInt       dim = rc->dim;
372ca3d3a14SMatthew G. Knepley   PetscReal      c1, s1, c2, s2, c3, s3;
373ca3d3a14SMatthew G. Knepley   PetscErrorCode ierr;
374ca3d3a14SMatthew G. Knepley 
375ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
376ca3d3a14SMatthew G. Knepley   ierr = PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT);CHKERRQ(ierr);
377ca3d3a14SMatthew G. Knepley   switch (dim) {
378ca3d3a14SMatthew G. Knepley   case 2:
379ca3d3a14SMatthew G. Knepley     c1 = PetscCosReal(rc->alpha);s1 = PetscSinReal(rc->alpha);
380ca3d3a14SMatthew G. Knepley     rc->R[0] =  c1;rc->R[1] = s1;
381ca3d3a14SMatthew G. Knepley     rc->R[2] = -s1;rc->R[3] = c1;
382580bdb30SBarry Smith     ierr = PetscArraycpy(rc->RT, rc->R, PetscSqr(dim));CHKERRQ(ierr);
383b458e8f1SJose E. Roman     DMPlex_Transpose2D_Internal(rc->RT);
384ca3d3a14SMatthew G. Knepley     break;
385ca3d3a14SMatthew G. Knepley   case 3:
386ca3d3a14SMatthew G. Knepley     c1 = PetscCosReal(rc->alpha);s1 = PetscSinReal(rc->alpha);
387ca3d3a14SMatthew G. Knepley     c2 = PetscCosReal(rc->beta); s2 = PetscSinReal(rc->beta);
388ca3d3a14SMatthew G. Knepley     c3 = PetscCosReal(rc->gamma);s3 = PetscSinReal(rc->gamma);
389ca3d3a14SMatthew G. Knepley     rc->R[0] =  c1*c3 - c2*s1*s3;rc->R[1] =  c3*s1    + c1*c2*s3;rc->R[2] = s2*s3;
390ca3d3a14SMatthew G. Knepley     rc->R[3] = -c1*s3 - c2*c3*s1;rc->R[4] =  c1*c2*c3 - s1*s3;   rc->R[5] = c3*s2;
391ca3d3a14SMatthew G. Knepley     rc->R[6] =  s1*s2;           rc->R[7] = -c1*s2;              rc->R[8] = c2;
392580bdb30SBarry Smith     ierr = PetscArraycpy(rc->RT, rc->R, PetscSqr(dim));CHKERRQ(ierr);
393b458e8f1SJose E. Roman     DMPlex_Transpose3D_Internal(rc->RT);
394ca3d3a14SMatthew G. Knepley     break;
395ca3d3a14SMatthew G. Knepley   default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %D not supported", dim);
396ca3d3a14SMatthew G. Knepley   }
397ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
398ca3d3a14SMatthew G. Knepley }
399ca3d3a14SMatthew G. Knepley 
400ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx)
401ca3d3a14SMatthew G. Knepley {
402ca3d3a14SMatthew G. Knepley   RotCtx        *rc = (RotCtx *) ctx;
403ca3d3a14SMatthew G. Knepley   PetscErrorCode ierr;
404ca3d3a14SMatthew G. Knepley 
405ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
406ca3d3a14SMatthew G. Knepley   ierr = PetscFree2(rc->R, rc->RT);CHKERRQ(ierr);
407ca3d3a14SMatthew G. Knepley   ierr = PetscFree(rc);CHKERRQ(ierr);
408ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
409ca3d3a14SMatthew G. Knepley }
410ca3d3a14SMatthew G. Knepley 
411ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx)
412ca3d3a14SMatthew G. Knepley {
413ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *) ctx;
414ca3d3a14SMatthew G. Knepley 
415ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
416ca3d3a14SMatthew G. Knepley   PetscValidPointer(ctx, 5);
417ca3d3a14SMatthew G. Knepley   if (l2g) {*A = rc->R;}
418ca3d3a14SMatthew G. Knepley   else     {*A = rc->RT;}
419ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
420ca3d3a14SMatthew G. Knepley }
421ca3d3a14SMatthew G. Knepley 
422ab6a9622SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx)
423ab6a9622SMatthew G. Knepley {
424ec277c0fSMatthew G. Knepley   PetscErrorCode ierr;
425ec277c0fSMatthew G. Knepley 
426ec277c0fSMatthew G. Knepley   PetscFunctionBegin;
427ab6a9622SMatthew G. Knepley   #if defined(PETSC_USE_COMPLEX)
428ab6a9622SMatthew G. Knepley   switch (dim) {
429ab6a9622SMatthew G. Knepley     case 2:
430ab6a9622SMatthew G. Knepley     {
43142e9364cSSatish Balay       PetscScalar yt[2], zt[2] = {0.0,0.0};
432ab6a9622SMatthew G. Knepley 
433ab6a9622SMatthew G. Knepley       yt[0] = y[0]; yt[1] = y[1];
434ec277c0fSMatthew G. Knepley       ierr = DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx);CHKERRQ(ierr);
435ab6a9622SMatthew G. Knepley       z[0] = PetscRealPart(zt[0]); z[1] = PetscRealPart(zt[1]);
436ab6a9622SMatthew G. Knepley     }
4379b4815dcSMatthew G. Knepley     break;
438ab6a9622SMatthew G. Knepley     case 3:
439ab6a9622SMatthew G. Knepley     {
44042e9364cSSatish Balay       PetscScalar yt[3], zt[3] = {0.0,0.0,0.0};
441ab6a9622SMatthew G. Knepley 
442ab6a9622SMatthew G. Knepley       yt[0] = y[0]; yt[1] = y[1]; yt[2] = y[2];
443ec277c0fSMatthew G. Knepley       ierr = DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx);CHKERRQ(ierr);
444ab6a9622SMatthew G. Knepley       z[0] = PetscRealPart(zt[0]); z[1] = PetscRealPart(zt[1]); z[2] = PetscRealPart(zt[2]);
445ab6a9622SMatthew G. Knepley     }
4469b4815dcSMatthew G. Knepley     break;
447ab6a9622SMatthew G. Knepley   }
448ab6a9622SMatthew G. Knepley   #else
449ab6a9622SMatthew G. Knepley   ierr = DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx);CHKERRQ(ierr);
450ab6a9622SMatthew G. Knepley   #endif
451ec277c0fSMatthew G. Knepley   PetscFunctionReturn(0);
452ab6a9622SMatthew G. Knepley }
453ab6a9622SMatthew G. Knepley 
454ec277c0fSMatthew G. Knepley PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx)
455ca3d3a14SMatthew G. Knepley {
456ca3d3a14SMatthew G. Knepley   const PetscScalar *A;
457ca3d3a14SMatthew G. Knepley   PetscErrorCode     ierr;
458ca3d3a14SMatthew G. Knepley 
459ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
460ca3d3a14SMatthew G. Knepley   ierr = (*dm->transformGetMatrix)(dm, x, l2g, &A, ctx);CHKERRQ(ierr);
461ca3d3a14SMatthew G. Knepley   switch (dim) {
4621fba962aSMatthew G. Knepley   case 2: DMPlex_Mult2D_Internal(A, 1, y, z);break;
4631fba962aSMatthew G. Knepley   case 3: DMPlex_Mult3D_Internal(A, 1, y, z);break;
464ca3d3a14SMatthew G. Knepley   }
465ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
466ca3d3a14SMatthew G. Knepley }
467ca3d3a14SMatthew G. Knepley 
468ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a)
469ca3d3a14SMatthew G. Knepley {
470ca3d3a14SMatthew G. Knepley   PetscSection       ts;
471ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tva;
472ca3d3a14SMatthew G. Knepley   PetscInt           dof;
473ca3d3a14SMatthew G. Knepley   PetscErrorCode     ierr;
474ca3d3a14SMatthew G. Knepley 
475ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
47692fd8e1eSJed Brown   ierr = DMGetLocalSection(tdm, &ts);CHKERRQ(ierr);
477ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetFieldDof(ts, p, f, &dof);CHKERRQ(ierr);
478ca3d3a14SMatthew G. Knepley   ierr = VecGetArrayRead(tv, &ta);CHKERRQ(ierr);
479ca3d3a14SMatthew G. Knepley   ierr = DMPlexPointLocalFieldRead(tdm, p, f, ta, (void *) &tva);CHKERRQ(ierr);
480ca3d3a14SMatthew G. Knepley   if (l2g) {
481ca3d3a14SMatthew G. Knepley     switch (dof) {
4821fba962aSMatthew G. Knepley     case 4: DMPlex_Mult2D_Internal(tva, 1, a, a);break;
4831fba962aSMatthew G. Knepley     case 9: DMPlex_Mult3D_Internal(tva, 1, a, a);break;
484ca3d3a14SMatthew G. Knepley     }
485ca3d3a14SMatthew G. Knepley   } else {
486ca3d3a14SMatthew G. Knepley     switch (dof) {
4871fba962aSMatthew G. Knepley     case 4: DMPlex_MultTranspose2D_Internal(tva, 1, a, a);break;
4881fba962aSMatthew G. Knepley     case 9: DMPlex_MultTranspose3D_Internal(tva, 1, a, a);break;
489ca3d3a14SMatthew G. Knepley     }
490ca3d3a14SMatthew G. Knepley   }
491ca3d3a14SMatthew G. Knepley   ierr = VecRestoreArrayRead(tv, &ta);CHKERRQ(ierr);
492ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
493ca3d3a14SMatthew G. Knepley }
494ca3d3a14SMatthew G. Knepley 
495ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a)
496ca3d3a14SMatthew G. Knepley {
497ca3d3a14SMatthew G. Knepley   PetscSection       s, ts;
498ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tvaf, *tvag;
499ca3d3a14SMatthew G. Knepley   PetscInt           fdof, gdof, fpdof, gpdof;
500ca3d3a14SMatthew G. Knepley   PetscErrorCode     ierr;
501ca3d3a14SMatthew G. Knepley 
502ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
50392fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
50492fd8e1eSJed Brown   ierr = DMGetLocalSection(tdm, &ts);CHKERRQ(ierr);
505ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetFieldDof(s, pf, f, &fpdof);CHKERRQ(ierr);
506ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetFieldDof(s, pg, g, &gpdof);CHKERRQ(ierr);
507ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetFieldDof(ts, pf, f, &fdof);CHKERRQ(ierr);
508ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetFieldDof(ts, pg, g, &gdof);CHKERRQ(ierr);
509ca3d3a14SMatthew G. Knepley   ierr = VecGetArrayRead(tv, &ta);CHKERRQ(ierr);
510ca3d3a14SMatthew G. Knepley   ierr = DMPlexPointLocalFieldRead(tdm, pf, f, ta, (void *) &tvaf);CHKERRQ(ierr);
511ca3d3a14SMatthew G. Knepley   ierr = DMPlexPointLocalFieldRead(tdm, pg, g, ta, (void *) &tvag);CHKERRQ(ierr);
512ca3d3a14SMatthew G. Knepley   if (l2g) {
513ca3d3a14SMatthew G. Knepley     switch (fdof) {
514ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a);break;
515ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a);break;
516ca3d3a14SMatthew G. Knepley     }
517ca3d3a14SMatthew G. Knepley     switch (gdof) {
518ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a);break;
519ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a);break;
520ca3d3a14SMatthew G. Knepley     }
521ca3d3a14SMatthew G. Knepley   } else {
522ca3d3a14SMatthew G. Knepley     switch (fdof) {
523ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a);break;
524ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a);break;
525ca3d3a14SMatthew G. Knepley     }
526ca3d3a14SMatthew G. Knepley     switch (gdof) {
527ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a);break;
528ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a);break;
529ca3d3a14SMatthew G. Knepley     }
530ca3d3a14SMatthew G. Knepley   }
531ca3d3a14SMatthew G. Knepley   ierr = VecRestoreArrayRead(tv, &ta);CHKERRQ(ierr);
532ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
533ca3d3a14SMatthew G. Knepley }
534ca3d3a14SMatthew G. Knepley 
535ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a)
536ca3d3a14SMatthew G. Knepley {
537ca3d3a14SMatthew G. Knepley   PetscSection    s;
538ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
539ca3d3a14SMatthew G. Knepley   IS              clPoints;
540ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
541ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
542ca3d3a14SMatthew G. Knepley   PetscInt        Nf, f, Np, cp, dof, d = 0;
543ca3d3a14SMatthew G. Knepley   PetscErrorCode  ierr;
544ca3d3a14SMatthew G. Knepley 
545ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
54692fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
547ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
548ca3d3a14SMatthew G. Knepley   ierr = DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
549ca3d3a14SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
550ca3d3a14SMatthew G. Knepley     for (cp = 0; cp < Np*2; cp += 2) {
551ca3d3a14SMatthew G. Knepley       ierr = PetscSectionGetFieldDof(s, points[cp], f, &dof);CHKERRQ(ierr);
552ca3d3a14SMatthew G. Knepley       if (!dof) continue;
553ca3d3a14SMatthew G. Knepley       if (fieldActive[f]) {ierr = DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d]);CHKERRQ(ierr);}
554ca3d3a14SMatthew G. Knepley       d += dof;
555ca3d3a14SMatthew G. Knepley     }
556ca3d3a14SMatthew G. Knepley   }
557ca3d3a14SMatthew G. Knepley   ierr = DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
558ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
559ca3d3a14SMatthew G. Knepley }
560ca3d3a14SMatthew G. Knepley 
561ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a)
562ca3d3a14SMatthew G. Knepley {
563ca3d3a14SMatthew G. Knepley   PetscSection    s;
564ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
565ca3d3a14SMatthew G. Knepley   IS              clPoints;
566ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
567ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
5688bdb3c71SMatthew G. Knepley   PetscInt        Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0;
569ca3d3a14SMatthew G. Knepley   PetscErrorCode  ierr;
570ca3d3a14SMatthew G. Knepley 
571ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
57292fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
573ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
574ca3d3a14SMatthew G. Knepley   ierr = DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
575ca3d3a14SMatthew G. Knepley   for (f = 0, r = 0; f < Nf; ++f) {
576ca3d3a14SMatthew G. Knepley     for (cpf = 0; cpf < Np*2; cpf += 2) {
577ca3d3a14SMatthew G. Knepley       ierr = PetscSectionGetFieldDof(s, points[cpf], f, &fdof);CHKERRQ(ierr);
578ca3d3a14SMatthew G. Knepley       for (g = 0, c = 0; g < Nf; ++g) {
579ca3d3a14SMatthew G. Knepley         for (cpg = 0; cpg < Np*2; cpg += 2) {
580ca3d3a14SMatthew G. Knepley           ierr = PetscSectionGetFieldDof(s, points[cpg], g, &gdof);CHKERRQ(ierr);
581ca3d3a14SMatthew G. Knepley           ierr = DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r*lda+c]);CHKERRQ(ierr);
582ca3d3a14SMatthew G. Knepley           c += gdof;
583ca3d3a14SMatthew G. Knepley         }
584ca3d3a14SMatthew G. Knepley       }
585ca3d3a14SMatthew G. Knepley       if (c != lda) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %D should be %D", c, lda);
586ca3d3a14SMatthew G. Knepley       r += fdof;
587ca3d3a14SMatthew G. Knepley     }
588ca3d3a14SMatthew G. Knepley   }
589ca3d3a14SMatthew G. Knepley   if (r != lda) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %D should be %D", c, lda);
590ca3d3a14SMatthew G. Knepley   ierr = DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp);CHKERRQ(ierr);
591ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
592ca3d3a14SMatthew G. Knepley }
593ca3d3a14SMatthew G. Knepley 
594ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g)
595ca3d3a14SMatthew G. Knepley {
596ca3d3a14SMatthew G. Knepley   DM                 tdm;
597ca3d3a14SMatthew G. Knepley   Vec                tv;
598ca3d3a14SMatthew G. Knepley   PetscSection       ts, s;
599ca3d3a14SMatthew G. Knepley   const PetscScalar *ta;
600ca3d3a14SMatthew G. Knepley   PetscScalar       *a, *va;
601ca3d3a14SMatthew G. Knepley   PetscInt           pStart, pEnd, p, Nf, f;
602ca3d3a14SMatthew G. Knepley   PetscErrorCode     ierr;
603ca3d3a14SMatthew G. Knepley 
604ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
605ca3d3a14SMatthew G. Knepley   ierr = DMGetBasisTransformDM_Internal(dm, &tdm);CHKERRQ(ierr);
606ca3d3a14SMatthew G. Knepley   ierr = DMGetBasisTransformVec_Internal(dm, &tv);CHKERRQ(ierr);
60792fd8e1eSJed Brown   ierr = DMGetLocalSection(tdm, &ts);CHKERRQ(ierr);
60892fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
609ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
610ca3d3a14SMatthew G. Knepley   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
611ca3d3a14SMatthew G. Knepley   ierr = VecGetArray(lv, &a);CHKERRQ(ierr);
612ca3d3a14SMatthew G. Knepley   ierr = VecGetArrayRead(tv, &ta);CHKERRQ(ierr);
613ca3d3a14SMatthew G. Knepley   for (p = pStart; p < pEnd; ++p) {
614ca3d3a14SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
615ca3d3a14SMatthew G. Knepley       ierr = DMPlexPointLocalFieldRef(dm, p, f, a, (void *) &va);CHKERRQ(ierr);
616ca3d3a14SMatthew G. Knepley       ierr = DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va);CHKERRQ(ierr);
617ca3d3a14SMatthew G. Knepley     }
618ca3d3a14SMatthew G. Knepley   }
619ca3d3a14SMatthew G. Knepley   ierr = VecRestoreArray(lv, &a);CHKERRQ(ierr);
620ca3d3a14SMatthew G. Knepley   ierr = VecRestoreArrayRead(tv, &ta);CHKERRQ(ierr);
621ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
622ca3d3a14SMatthew G. Knepley }
623ca3d3a14SMatthew G. Knepley 
624ca3d3a14SMatthew G. Knepley /*@
625ca3d3a14SMatthew G. Knepley   DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis
626ca3d3a14SMatthew G. Knepley 
627ca3d3a14SMatthew G. Knepley   Input Parameters:
628ca3d3a14SMatthew G. Knepley + dm - The DM
629ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the global basis
630ca3d3a14SMatthew G. Knepley 
631ca3d3a14SMatthew G. Knepley   Output Parameters:
632ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the local basis
633ca3d3a14SMatthew G. Knepley 
634c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
635c0f8e1fdSMatthew G. Knepley 
636ca3d3a14SMatthew G. Knepley   Level: developer
637ca3d3a14SMatthew G. Knepley 
638436bc73aSJed Brown .seealso: DMPlexLocalToGlobalBasis(), DMGetLocalSection(), DMPlexCreateBasisRotation()
639ca3d3a14SMatthew G. Knepley @*/
640ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv)
641ca3d3a14SMatthew G. Knepley {
642ca3d3a14SMatthew G. Knepley   PetscErrorCode ierr;
643ca3d3a14SMatthew G. Knepley 
644ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
645ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
646ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
647ca3d3a14SMatthew G. Knepley   ierr = DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE);CHKERRQ(ierr);
648ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
649ca3d3a14SMatthew G. Knepley }
650ca3d3a14SMatthew G. Knepley 
651ca3d3a14SMatthew G. Knepley /*@
652ca3d3a14SMatthew G. Knepley   DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis
653ca3d3a14SMatthew G. Knepley 
654ca3d3a14SMatthew G. Knepley   Input Parameters:
655ca3d3a14SMatthew G. Knepley + dm - The DM
656ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the local basis
657ca3d3a14SMatthew G. Knepley 
658ca3d3a14SMatthew G. Knepley   Output Parameters:
659ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the global basis
660ca3d3a14SMatthew G. Knepley 
661c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
662c0f8e1fdSMatthew G. Knepley 
663ca3d3a14SMatthew G. Knepley   Level: developer
664ca3d3a14SMatthew G. Knepley 
665436bc73aSJed Brown .seealso: DMPlexGlobalToLocalBasis(), DMGetLocalSection(), DMPlexCreateBasisRotation()
666ca3d3a14SMatthew G. Knepley @*/
667ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv)
668ca3d3a14SMatthew G. Knepley {
669ca3d3a14SMatthew G. Knepley   PetscErrorCode ierr;
670ca3d3a14SMatthew G. Knepley 
671ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
672ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
673ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
674ca3d3a14SMatthew G. Knepley   ierr = DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE);CHKERRQ(ierr);
675ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
676ca3d3a14SMatthew G. Knepley }
677ca3d3a14SMatthew G. Knepley 
678ca3d3a14SMatthew G. Knepley /*@
679ca3d3a14SMatthew G. Knepley   DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions
680ca3d3a14SMatthew G. Knepley     and global solutions, to a local basis, appropriate for discretization integrals and assembly.
681ca3d3a14SMatthew G. Knepley 
682ca3d3a14SMatthew G. Knepley   Input Parameters:
683ca3d3a14SMatthew G. Knepley + dm    - The DM
684ca3d3a14SMatthew G. Knepley . alpha - The first Euler angle, and in 2D the only one
685ca3d3a14SMatthew G. Knepley . beta  - The second Euler angle
686f0fc11ceSJed Brown - gamma - The third Euler angle
687ca3d3a14SMatthew G. Knepley 
688ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
689ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
690ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
691ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
692ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
693ca3d3a14SMatthew G. Knepley 
694ca3d3a14SMatthew G. Knepley   Level: developer
695ca3d3a14SMatthew G. Knepley 
696ca3d3a14SMatthew G. Knepley .seealso: DMPlexGlobalToLocalBasis(), DMPlexLocalToGlobalBasis()
697ca3d3a14SMatthew G. Knepley @*/
698ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma)
699ca3d3a14SMatthew G. Knepley {
700ca3d3a14SMatthew G. Knepley   RotCtx        *rc;
701ca3d3a14SMatthew G. Knepley   PetscInt       cdim;
702ca3d3a14SMatthew G. Knepley   PetscErrorCode ierr;
703ca3d3a14SMatthew G. Knepley 
704*362febeeSStefano Zampini   PetscFunctionBegin;
705ca3d3a14SMatthew G. Knepley   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
706ca3d3a14SMatthew G. Knepley   ierr = PetscMalloc1(1, &rc);CHKERRQ(ierr);
707ca3d3a14SMatthew G. Knepley   dm->transformCtx       = rc;
708ca3d3a14SMatthew G. Knepley   dm->transformSetUp     = DMPlexBasisTransformSetUp_Rotation_Internal;
709ca3d3a14SMatthew G. Knepley   dm->transformDestroy   = DMPlexBasisTransformDestroy_Rotation_Internal;
710ca3d3a14SMatthew G. Knepley   dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal;
711ca3d3a14SMatthew G. Knepley   rc->dim   = cdim;
712ca3d3a14SMatthew G. Knepley   rc->alpha = alpha;
713ca3d3a14SMatthew G. Knepley   rc->beta  = beta;
714ca3d3a14SMatthew G. Knepley   rc->gamma = gamma;
715ca3d3a14SMatthew G. Knepley   ierr = (*dm->transformSetUp)(dm, dm->transformCtx);CHKERRQ(ierr);
716ca3d3a14SMatthew G. Knepley   ierr = DMConstructBasisTransform_Internal(dm);CHKERRQ(ierr);
717ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
718ca3d3a14SMatthew G. Knepley }
719ca3d3a14SMatthew G. Knepley 
720b278463cSMatthew G. Knepley /*@C
721ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates
722b278463cSMatthew G. Knepley 
723b278463cSMatthew G. Knepley   Input Parameters:
724b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
725b278463cSMatthew G. Knepley . time   - The time
726b278463cSMatthew G. Knepley . field  - The field to constrain
7271c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7281c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
729b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
730b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
731b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
732b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
733b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
734b278463cSMatthew G. Knepley 
735b278463cSMatthew G. Knepley   Output Parameter:
736b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
737b278463cSMatthew G. Knepley 
738b278463cSMatthew G. Knepley   Level: developer
739b278463cSMatthew G. Knepley 
740ece3a9fcSMatthew G. Knepley .seealso: DMPlexInsertBoundaryValuesEssentialField(), DMPlexInsertBoundaryValuesEssentialBdField(), DMAddBoundary()
741b278463cSMatthew G. Knepley @*/
7421c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX)
74355f2e967SMatthew G. Knepley {
7440163fd50SMatthew G. Knepley   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
74555f2e967SMatthew G. Knepley   void            **ctxs;
746d7ddef95SMatthew G. Knepley   PetscInt          numFields;
747d7ddef95SMatthew G. Knepley   PetscErrorCode    ierr;
748d7ddef95SMatthew G. Knepley 
749d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
750d7ddef95SMatthew G. Knepley   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
751d7ddef95SMatthew G. Knepley   ierr = PetscCalloc2(numFields,&funcs,numFields,&ctxs);CHKERRQ(ierr);
752d7ddef95SMatthew G. Knepley   funcs[field] = func;
753d7ddef95SMatthew G. Knepley   ctxs[field]  = ctx;
7541c531cf8SMatthew G. Knepley   ierr = DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX);CHKERRQ(ierr);
755d7ddef95SMatthew G. Knepley   ierr = PetscFree2(funcs,ctxs);CHKERRQ(ierr);
756d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
757d7ddef95SMatthew G. Knepley }
758d7ddef95SMatthew G. Knepley 
759b278463cSMatthew G. Knepley /*@C
760ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data
761b278463cSMatthew G. Knepley 
762b278463cSMatthew G. Knepley   Input Parameters:
763b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
764b278463cSMatthew G. Knepley . time   - The time
765b278463cSMatthew G. Knepley . locU   - A local vector with the input solution values
766b278463cSMatthew G. Knepley . field  - The field to constrain
7671c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7681c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
769b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
770b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
771b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
772b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
773b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
774b278463cSMatthew G. Knepley 
775b278463cSMatthew G. Knepley   Output Parameter:
776b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
777b278463cSMatthew G. Knepley 
778b278463cSMatthew G. Knepley   Level: developer
779b278463cSMatthew G. Knepley 
780ece3a9fcSMatthew G. Knepley .seealso: DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialBdField(), DMAddBoundary()
781b278463cSMatthew G. Knepley @*/
7821c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
783c60e475cSMatthew G. Knepley                                                         void (*func)(PetscInt, PetscInt, PetscInt,
784c60e475cSMatthew G. Knepley                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
785c60e475cSMatthew G. Knepley                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
78697b6e6e8SMatthew G. Knepley                                                                      PetscReal, const PetscReal[], PetscInt, const PetscScalar[],
787b278463cSMatthew G. Knepley                                                                      PetscScalar[]),
788b278463cSMatthew G. Knepley                                                         void *ctx, Vec locX)
789c60e475cSMatthew G. Knepley {
790c60e475cSMatthew G. Knepley   void (**funcs)(PetscInt, PetscInt, PetscInt,
791c60e475cSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
792c60e475cSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
79397b6e6e8SMatthew G. Knepley                  PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
794c60e475cSMatthew G. Knepley   void            **ctxs;
795c60e475cSMatthew G. Knepley   PetscInt          numFields;
796c60e475cSMatthew G. Knepley   PetscErrorCode    ierr;
797c60e475cSMatthew G. Knepley 
798c60e475cSMatthew G. Knepley   PetscFunctionBegin;
799c60e475cSMatthew G. Knepley   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
800c60e475cSMatthew G. Knepley   ierr = PetscCalloc2(numFields,&funcs,numFields,&ctxs);CHKERRQ(ierr);
801c60e475cSMatthew G. Knepley   funcs[field] = func;
802c60e475cSMatthew G. Knepley   ctxs[field]  = ctx;
8031c531cf8SMatthew G. Knepley   ierr = DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX);CHKERRQ(ierr);
804c60e475cSMatthew G. Knepley   ierr = PetscFree2(funcs,ctxs);CHKERRQ(ierr);
805c60e475cSMatthew G. Knepley   PetscFunctionReturn(0);
806c60e475cSMatthew G. Knepley }
807c60e475cSMatthew G. Knepley 
808b278463cSMatthew G. Knepley /*@C
809ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coodinates and boundary field data
810ece3a9fcSMatthew G. Knepley 
811ece3a9fcSMatthew G. Knepley   Collective on dm
812ece3a9fcSMatthew G. Knepley 
813ece3a9fcSMatthew G. Knepley   Input Parameters:
814ece3a9fcSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
815ece3a9fcSMatthew G. Knepley . time   - The time
816ece3a9fcSMatthew G. Knepley . locU   - A local vector with the input solution values
817ece3a9fcSMatthew G. Knepley . field  - The field to constrain
818ece3a9fcSMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
819ece3a9fcSMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
820ece3a9fcSMatthew G. Knepley . label  - The DMLabel defining constrained points
821ece3a9fcSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
822ece3a9fcSMatthew G. Knepley . ids    - An array of ids for constrained points
823ece3a9fcSMatthew G. Knepley . func   - A pointwise function giving boundary values, the calling sequence is given in DMProjectBdFieldLabelLocal()
824ece3a9fcSMatthew G. Knepley - ctx    - An optional user context for bcFunc
825ece3a9fcSMatthew G. Knepley 
826ece3a9fcSMatthew G. Knepley   Output Parameter:
827ece3a9fcSMatthew G. Knepley . locX   - A local vector to receive the boundary values
828ece3a9fcSMatthew G. Knepley 
829ece3a9fcSMatthew G. Knepley   Level: developer
830ece3a9fcSMatthew G. Knepley 
831ece3a9fcSMatthew G. Knepley .seealso: DMProjectBdFieldLabelLocal(), DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialField(), DMAddBoundary()
832ece3a9fcSMatthew G. Knepley @*/
833ece3a9fcSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
834ece3a9fcSMatthew G. Knepley                                                           void (*func)(PetscInt, PetscInt, PetscInt,
835ece3a9fcSMatthew G. Knepley                                                                        const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
836ece3a9fcSMatthew G. Knepley                                                                        const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
837ece3a9fcSMatthew G. Knepley                                                                        PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[],
838ece3a9fcSMatthew G. Knepley                                                                        PetscScalar[]),
839ece3a9fcSMatthew G. Knepley                                                           void *ctx, Vec locX)
840ece3a9fcSMatthew G. Knepley {
841ece3a9fcSMatthew G. Knepley   void (**funcs)(PetscInt, PetscInt, PetscInt,
842ece3a9fcSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
843ece3a9fcSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
844ece3a9fcSMatthew G. Knepley                  PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
845ece3a9fcSMatthew G. Knepley   void            **ctxs;
846ece3a9fcSMatthew G. Knepley   PetscInt          numFields;
847ece3a9fcSMatthew G. Knepley   PetscErrorCode    ierr;
848ece3a9fcSMatthew G. Knepley 
849ece3a9fcSMatthew G. Knepley   PetscFunctionBegin;
850ece3a9fcSMatthew G. Knepley   ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
851ece3a9fcSMatthew G. Knepley   ierr = PetscCalloc2(numFields,&funcs,numFields,&ctxs);CHKERRQ(ierr);
852ece3a9fcSMatthew G. Knepley   funcs[field] = func;
853ece3a9fcSMatthew G. Knepley   ctxs[field]  = ctx;
854ece3a9fcSMatthew G. Knepley   ierr = DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX);CHKERRQ(ierr);
855ece3a9fcSMatthew G. Knepley   ierr = PetscFree2(funcs,ctxs);CHKERRQ(ierr);
856ece3a9fcSMatthew G. Knepley   PetscFunctionReturn(0);
857ece3a9fcSMatthew G. Knepley }
858ece3a9fcSMatthew G. Knepley 
859ece3a9fcSMatthew G. Knepley /*@C
860b278463cSMatthew G. Knepley   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector
861b278463cSMatthew G. Knepley 
862b278463cSMatthew G. Knepley   Input Parameters:
863b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
864b278463cSMatthew G. Knepley . time   - The time
865b278463cSMatthew G. Knepley . faceGeometry - A vector with the FVM face geometry information
866b278463cSMatthew G. Knepley . cellGeometry - A vector with the FVM cell geometry information
867b278463cSMatthew G. Knepley . Grad         - A vector with the FVM cell gradient information
868b278463cSMatthew G. Knepley . field  - The field to constrain
8691c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
8701c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
871b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
872b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
873b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
874b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
875b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
876b278463cSMatthew G. Knepley 
877b278463cSMatthew G. Knepley   Output Parameter:
878b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
879b278463cSMatthew G. Knepley 
880b278463cSMatthew G. Knepley   Note: This implementation currently ignores the numcomps/comps argument from DMAddBoundary()
881b278463cSMatthew G. Knepley 
882b278463cSMatthew G. Knepley   Level: developer
883b278463cSMatthew G. Knepley 
884b278463cSMatthew G. Knepley .seealso: DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialField(), DMAddBoundary()
885b278463cSMatthew G. Knepley @*/
8861c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
887b278463cSMatthew G. Knepley                                                  PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*), void *ctx, Vec locX)
888d7ddef95SMatthew G. Knepley {
88961f58d28SMatthew G. Knepley   PetscDS            prob;
890da97024aSMatthew G. Knepley   PetscSF            sf;
891d7ddef95SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad;
89220369375SToby Isaac   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
893da97024aSMatthew G. Knepley   const PetscInt    *leaves;
894d7ddef95SMatthew G. Knepley   PetscScalar       *x, *fx;
895520b3818SMatthew G. Knepley   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
896e735a8a9SMatthew G. Knepley   PetscErrorCode     ierr, ierru = 0;
897d7ddef95SMatthew G. Knepley 
898d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
899da97024aSMatthew G. Knepley   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
900da97024aSMatthew G. Knepley   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
901da97024aSMatthew G. Knepley   nleaves = PetscMax(0, nleaves);
902d7ddef95SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
903d7ddef95SMatthew G. Knepley   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
90461f58d28SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
905d7ddef95SMatthew G. Knepley   ierr = VecGetDM(faceGeometry, &dmFace);CHKERRQ(ierr);
906d7ddef95SMatthew G. Knepley   ierr = VecGetArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
90720369375SToby Isaac   if (cellGeometry) {
908d7ddef95SMatthew G. Knepley     ierr = VecGetDM(cellGeometry, &dmCell);CHKERRQ(ierr);
909d7ddef95SMatthew G. Knepley     ierr = VecGetArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
91020369375SToby Isaac   }
911d7ddef95SMatthew G. Knepley   if (Grad) {
912c0a6632aSMatthew G. Knepley     PetscFV fv;
913c0a6632aSMatthew G. Knepley 
914c0a6632aSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, field, (PetscObject *) &fv);CHKERRQ(ierr);
915d7ddef95SMatthew G. Knepley     ierr = VecGetDM(Grad, &dmGrad);CHKERRQ(ierr);
916d7ddef95SMatthew G. Knepley     ierr = VecGetArrayRead(Grad, &grad);CHKERRQ(ierr);
917d7ddef95SMatthew G. Knepley     ierr = PetscFVGetNumComponents(fv, &pdim);CHKERRQ(ierr);
91869291d52SBarry Smith     ierr = DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx);CHKERRQ(ierr);
919d7ddef95SMatthew G. Knepley   }
920d7ddef95SMatthew G. Knepley   ierr = VecGetArray(locX, &x);CHKERRQ(ierr);
921d7ddef95SMatthew G. Knepley   for (i = 0; i < numids; ++i) {
922d7ddef95SMatthew G. Knepley     IS              faceIS;
923d7ddef95SMatthew G. Knepley     const PetscInt *faces;
924d7ddef95SMatthew G. Knepley     PetscInt        numFaces, f;
925d7ddef95SMatthew G. Knepley 
926d7ddef95SMatthew G. Knepley     ierr = DMLabelGetStratumIS(label, ids[i], &faceIS);CHKERRQ(ierr);
927d7ddef95SMatthew G. Knepley     if (!faceIS) continue; /* No points with that id on this process */
928d7ddef95SMatthew G. Knepley     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
929d7ddef95SMatthew G. Knepley     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
930d7ddef95SMatthew G. Knepley     for (f = 0; f < numFaces; ++f) {
931d7ddef95SMatthew G. Knepley       const PetscInt         face = faces[f], *cells;
932640bce14SSatish Balay       PetscFVFaceGeom        *fg;
933d7ddef95SMatthew G. Knepley 
934d7ddef95SMatthew G. Knepley       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
935da97024aSMatthew G. Knepley       ierr = PetscFindInt(face, nleaves, (PetscInt *) leaves, &loc);CHKERRQ(ierr);
936da97024aSMatthew G. Knepley       if (loc >= 0) continue;
937d7ddef95SMatthew G. Knepley       ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
938d7ddef95SMatthew G. Knepley       ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
939d7ddef95SMatthew G. Knepley       if (Grad) {
940640bce14SSatish Balay         PetscFVCellGeom       *cg;
941640bce14SSatish Balay         PetscScalar           *cx, *cgrad;
942d7ddef95SMatthew G. Knepley         PetscScalar           *xG;
943d7ddef95SMatthew G. Knepley         PetscReal              dx[3];
944d7ddef95SMatthew G. Knepley         PetscInt               d;
945d7ddef95SMatthew G. Knepley 
946d7ddef95SMatthew G. Knepley         ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg);CHKERRQ(ierr);
947d7ddef95SMatthew G. Knepley         ierr = DMPlexPointLocalRead(dm, cells[0], x, &cx);CHKERRQ(ierr);
948d7ddef95SMatthew G. Knepley         ierr = DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad);CHKERRQ(ierr);
949520b3818SMatthew G. Knepley         ierr = DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG);CHKERRQ(ierr);
950d7ddef95SMatthew G. Knepley         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
951d7ddef95SMatthew G. Knepley         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d*dim], dx);
952e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, fx, xG, ctx);
953e735a8a9SMatthew G. Knepley         if (ierru) {
954e735a8a9SMatthew G. Knepley           ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
955e735a8a9SMatthew G. Knepley           ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
956e735a8a9SMatthew G. Knepley           goto cleanup;
957e735a8a9SMatthew G. Knepley         }
958d7ddef95SMatthew G. Knepley       } else {
959640bce14SSatish Balay         PetscScalar       *xI;
960d7ddef95SMatthew G. Knepley         PetscScalar       *xG;
961d7ddef95SMatthew G. Knepley 
962d7ddef95SMatthew G. Knepley         ierr = DMPlexPointLocalRead(dm, cells[0], x, &xI);CHKERRQ(ierr);
963520b3818SMatthew G. Knepley         ierr = DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG);CHKERRQ(ierr);
964e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
965e735a8a9SMatthew G. Knepley         if (ierru) {
966e735a8a9SMatthew G. Knepley           ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
967e735a8a9SMatthew G. Knepley           ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
968e735a8a9SMatthew G. Knepley           goto cleanup;
969e735a8a9SMatthew G. Knepley         }
970d7ddef95SMatthew G. Knepley       }
971d7ddef95SMatthew G. Knepley     }
972d7ddef95SMatthew G. Knepley     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
973d7ddef95SMatthew G. Knepley     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
974d7ddef95SMatthew G. Knepley   }
975e735a8a9SMatthew G. Knepley   cleanup:
976d7ddef95SMatthew G. Knepley   ierr = VecRestoreArray(locX, &x);CHKERRQ(ierr);
977d7ddef95SMatthew G. Knepley   if (Grad) {
97869291d52SBarry Smith     ierr = DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx);CHKERRQ(ierr);
979d7ddef95SMatthew G. Knepley     ierr = VecRestoreArrayRead(Grad, &grad);CHKERRQ(ierr);
980d7ddef95SMatthew G. Knepley   }
981e735a8a9SMatthew G. Knepley   if (cellGeometry) {ierr = VecRestoreArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);}
982d7ddef95SMatthew G. Knepley   ierr = VecRestoreArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
983e735a8a9SMatthew G. Knepley   CHKERRQ(ierru);
984d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
985d7ddef95SMatthew G. Knepley }
986d7ddef95SMatthew G. Knepley 
9870c364540SMatthew G. Knepley static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx)
9880c364540SMatthew G. Knepley {
9890c364540SMatthew G. Knepley   PetscInt c;
9900c364540SMatthew G. Knepley   for (c = 0; c < Nc; ++c) u[c] = 0.0;
9910c364540SMatthew G. Knepley   return 0;
9920c364540SMatthew G. Knepley }
9930c364540SMatthew G. Knepley 
994f1d73a7aSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
995d7ddef95SMatthew G. Knepley {
9960c364540SMatthew G. Knepley   PetscObject    isZero;
997e5e52638SMatthew G. Knepley   PetscDS        prob;
998d7ddef95SMatthew G. Knepley   PetscInt       numBd, b;
99955f2e967SMatthew G. Knepley   PetscErrorCode ierr;
100055f2e967SMatthew G. Knepley 
100155f2e967SMatthew G. Knepley   PetscFunctionBegin;
1002e5e52638SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
1003e5e52638SMatthew G. Knepley   ierr = PetscDSGetNumBoundary(prob, &numBd);CHKERRQ(ierr);
10040c364540SMatthew G. Knepley   ierr = PetscObjectQuery((PetscObject) locX, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
100555f2e967SMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
100645480ffeSMatthew G. Knepley     PetscWeakForm           wf;
1007f971fd6bSMatthew G. Knepley     DMBoundaryConditionType type;
100845480ffeSMatthew G. Knepley     const char             *name;
1009d7ddef95SMatthew G. Knepley     DMLabel                 label;
10101c531cf8SMatthew G. Knepley     PetscInt                field, Nc;
10111c531cf8SMatthew G. Knepley     const PetscInt         *comps;
1012d7ddef95SMatthew G. Knepley     PetscObject             obj;
1013d7ddef95SMatthew G. Knepley     PetscClassId            id;
101445480ffeSMatthew G. Knepley     void                  (*bvfunc)(void);
1015d7ddef95SMatthew G. Knepley     PetscInt                numids;
1016d7ddef95SMatthew G. Knepley     const PetscInt         *ids;
101755f2e967SMatthew G. Knepley     void                   *ctx;
101855f2e967SMatthew G. Knepley 
101945480ffeSMatthew G. Knepley     ierr = PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx);CHKERRQ(ierr);
1020f971fd6bSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
102144a7f3ddSMatthew G. Knepley     ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
1022d7ddef95SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
1023d7ddef95SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1024c60e475cSMatthew G. Knepley       switch (type) {
1025c60e475cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
1026c60e475cSMatthew G. Knepley       case DM_BC_ESSENTIAL:
102745480ffeSMatthew G. Knepley         {
102845480ffeSMatthew G. Knepley           PetscSimplePointFunc func = (PetscSimplePointFunc) bvfunc;
102945480ffeSMatthew G. Knepley 
103045480ffeSMatthew G. Knepley           if (isZero) func = zero;
1031092e5057SToby Isaac           ierr = DMPlexLabelAddCells(dm,label);CHKERRQ(ierr);
103245480ffeSMatthew G. Knepley           ierr = DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX);CHKERRQ(ierr);
1033092e5057SToby Isaac           ierr = DMPlexLabelClearCells(dm,label);CHKERRQ(ierr);
103445480ffeSMatthew G. Knepley         }
1035c60e475cSMatthew G. Knepley         break;
1036c60e475cSMatthew G. Knepley       case DM_BC_ESSENTIAL_FIELD:
103745480ffeSMatthew G. Knepley         {
103845480ffeSMatthew G. Knepley           PetscPointFunc func = (PetscPointFunc) bvfunc;
103945480ffeSMatthew G. Knepley 
1040c60e475cSMatthew G. Knepley           ierr = DMPlexLabelAddCells(dm,label);CHKERRQ(ierr);
104145480ffeSMatthew G. Knepley           ierr = DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX);CHKERRQ(ierr);
1042c60e475cSMatthew G. Knepley           ierr = DMPlexLabelClearCells(dm,label);CHKERRQ(ierr);
104345480ffeSMatthew G. Knepley         }
1044c60e475cSMatthew G. Knepley         break;
1045c60e475cSMatthew G. Knepley       default: break;
1046c60e475cSMatthew G. Knepley       }
1047d7ddef95SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
104845480ffeSMatthew G. Knepley       {
104945480ffeSMatthew G. Knepley         PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*) = (PetscErrorCode (*)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*)) bvfunc;
105045480ffeSMatthew G. Knepley 
105143ea7facSMatthew G. Knepley         if (!faceGeomFVM) continue;
105245480ffeSMatthew G. Knepley         ierr = DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX);CHKERRQ(ierr);
105345480ffeSMatthew G. Knepley       }
1054ff1e0c32SBarry Smith     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
105555f2e967SMatthew G. Knepley   }
105655f2e967SMatthew G. Knepley   PetscFunctionReturn(0);
105755f2e967SMatthew G. Knepley }
105855f2e967SMatthew G. Knepley 
105956cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
106056cf3b9cSMatthew G. Knepley {
106156cf3b9cSMatthew G. Knepley   PetscObject    isZero;
106256cf3b9cSMatthew G. Knepley   PetscDS        prob;
106356cf3b9cSMatthew G. Knepley   PetscInt       numBd, b;
106456cf3b9cSMatthew G. Knepley   PetscErrorCode ierr;
106556cf3b9cSMatthew G. Knepley 
106656cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
106756cf3b9cSMatthew G. Knepley   if (!locX) PetscFunctionReturn(0);
106856cf3b9cSMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
106956cf3b9cSMatthew G. Knepley   ierr = PetscDSGetNumBoundary(prob, &numBd);CHKERRQ(ierr);
107056cf3b9cSMatthew G. Knepley   ierr = PetscObjectQuery((PetscObject) locX, "__Vec_bc_zero__", &isZero);CHKERRQ(ierr);
107156cf3b9cSMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
107245480ffeSMatthew G. Knepley     PetscWeakForm           wf;
107356cf3b9cSMatthew G. Knepley     DMBoundaryConditionType type;
107445480ffeSMatthew G. Knepley     const char             *name;
107556cf3b9cSMatthew G. Knepley     DMLabel                 label;
107656cf3b9cSMatthew G. Knepley     PetscInt                field, Nc;
107756cf3b9cSMatthew G. Knepley     const PetscInt         *comps;
107856cf3b9cSMatthew G. Knepley     PetscObject             obj;
107956cf3b9cSMatthew G. Knepley     PetscClassId            id;
108056cf3b9cSMatthew G. Knepley     PetscInt                numids;
108156cf3b9cSMatthew G. Knepley     const PetscInt         *ids;
108245480ffeSMatthew G. Knepley     void                  (*bvfunc)(void);
108356cf3b9cSMatthew G. Knepley     void                   *ctx;
108456cf3b9cSMatthew G. Knepley 
108545480ffeSMatthew G. Knepley     ierr = PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx);CHKERRQ(ierr);
108656cf3b9cSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
108756cf3b9cSMatthew G. Knepley     ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
108856cf3b9cSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
108956cf3b9cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
109056cf3b9cSMatthew G. Knepley       switch (type) {
109156cf3b9cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
109256cf3b9cSMatthew G. Knepley       case DM_BC_ESSENTIAL:
109345480ffeSMatthew G. Knepley         {
109445480ffeSMatthew G. Knepley           PetscSimplePointFunc func_t = (PetscSimplePointFunc) bvfunc;
109545480ffeSMatthew G. Knepley 
109645480ffeSMatthew G. Knepley           if (isZero) func_t = zero;
109756cf3b9cSMatthew G. Knepley           ierr = DMPlexLabelAddCells(dm,label);CHKERRQ(ierr);
109845480ffeSMatthew G. Knepley           ierr = DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX);CHKERRQ(ierr);
109956cf3b9cSMatthew G. Knepley           ierr = DMPlexLabelClearCells(dm,label);CHKERRQ(ierr);
110045480ffeSMatthew G. Knepley         }
110156cf3b9cSMatthew G. Knepley         break;
110256cf3b9cSMatthew G. Knepley       case DM_BC_ESSENTIAL_FIELD:
110345480ffeSMatthew G. Knepley         {
110445480ffeSMatthew G. Knepley           PetscPointFunc func_t = (PetscPointFunc) bvfunc;
110545480ffeSMatthew G. Knepley 
110656cf3b9cSMatthew G. Knepley           ierr = DMPlexLabelAddCells(dm,label);CHKERRQ(ierr);
110745480ffeSMatthew G. Knepley           ierr = DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX);CHKERRQ(ierr);
110856cf3b9cSMatthew G. Knepley           ierr = DMPlexLabelClearCells(dm,label);CHKERRQ(ierr);
110945480ffeSMatthew G. Knepley         }
111056cf3b9cSMatthew G. Knepley         break;
111156cf3b9cSMatthew G. Knepley       default: break;
111256cf3b9cSMatthew G. Knepley       }
111356cf3b9cSMatthew G. Knepley     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
111456cf3b9cSMatthew G. Knepley   }
111556cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
111656cf3b9cSMatthew G. Knepley }
111756cf3b9cSMatthew G. Knepley 
1118f1d73a7aSMatthew G. Knepley /*@
1119f1d73a7aSMatthew G. Knepley   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector
1120f1d73a7aSMatthew G. Knepley 
1121f1d73a7aSMatthew G. Knepley   Input Parameters:
1122f1d73a7aSMatthew G. Knepley + dm - The DM
1123f1d73a7aSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
1124f1d73a7aSMatthew G. Knepley . time - The time
1125f1d73a7aSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
1126f1d73a7aSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
1127f1d73a7aSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
1128f1d73a7aSMatthew G. Knepley 
1129f1d73a7aSMatthew G. Knepley   Output Parameters:
1130f1d73a7aSMatthew G. Knepley . locX - Solution updated with boundary values
1131f1d73a7aSMatthew G. Knepley 
1132f1d73a7aSMatthew G. Knepley   Level: developer
1133f1d73a7aSMatthew G. Knepley 
1134f1d73a7aSMatthew G. Knepley .seealso: DMProjectFunctionLabelLocal()
1135f1d73a7aSMatthew G. Knepley @*/
1136f1d73a7aSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1137f1d73a7aSMatthew G. Knepley {
1138f1d73a7aSMatthew G. Knepley   PetscErrorCode ierr;
1139f1d73a7aSMatthew G. Knepley 
1140f1d73a7aSMatthew G. Knepley   PetscFunctionBegin;
1141f1d73a7aSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1142064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
1143064a246eSJacob Faibussowitsch   if (faceGeomFVM) {PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);}
1144064a246eSJacob Faibussowitsch   if (cellGeomFVM) {PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);}
1145064a246eSJacob Faibussowitsch   if (gradFVM)     {PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);}
1146f1d73a7aSMatthew G. Knepley   ierr = PetscTryMethod(dm,"DMPlexInsertBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX,time,faceGeomFVM,cellGeomFVM,gradFVM));CHKERRQ(ierr);
1147f1d73a7aSMatthew G. Knepley   PetscFunctionReturn(0);
1148f1d73a7aSMatthew G. Knepley }
1149f1d73a7aSMatthew G. Knepley 
115056cf3b9cSMatthew G. Knepley /*@
115156cf3b9cSMatthew G. Knepley   DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derviative into the local solution vector
115256cf3b9cSMatthew G. Knepley 
115356cf3b9cSMatthew G. Knepley   Input Parameters:
115456cf3b9cSMatthew G. Knepley + dm - The DM
115556cf3b9cSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
115656cf3b9cSMatthew G. Knepley . time - The time
115756cf3b9cSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
115856cf3b9cSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
115956cf3b9cSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
116056cf3b9cSMatthew G. Knepley 
116156cf3b9cSMatthew G. Knepley   Output Parameters:
116256cf3b9cSMatthew G. Knepley . locX_t - Solution updated with boundary values
116356cf3b9cSMatthew G. Knepley 
116456cf3b9cSMatthew G. Knepley   Level: developer
116556cf3b9cSMatthew G. Knepley 
116656cf3b9cSMatthew G. Knepley .seealso: DMProjectFunctionLabelLocal()
116756cf3b9cSMatthew G. Knepley @*/
116856cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
116956cf3b9cSMatthew G. Knepley {
117056cf3b9cSMatthew G. Knepley   PetscErrorCode ierr;
117156cf3b9cSMatthew G. Knepley 
117256cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
117356cf3b9cSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1174064a246eSJacob Faibussowitsch   if (locX_t)      {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3);}
1175064a246eSJacob Faibussowitsch   if (faceGeomFVM) {PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);}
1176064a246eSJacob Faibussowitsch   if (cellGeomFVM) {PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);}
1177064a246eSJacob Faibussowitsch   if (gradFVM)     {PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);}
117856cf3b9cSMatthew G. Knepley   ierr = PetscTryMethod(dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX_t,time,faceGeomFVM,cellGeomFVM,gradFVM));CHKERRQ(ierr);
117956cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
118056cf3b9cSMatthew G. Knepley }
118156cf3b9cSMatthew G. Knepley 
11820709b2feSToby Isaac PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1183cb1e1211SMatthew G Knepley {
1184574a98acSMatthew G. Knepley   Vec              localX;
1185574a98acSMatthew G. Knepley   PetscErrorCode   ierr;
1186574a98acSMatthew G. Knepley 
1187574a98acSMatthew G. Knepley   PetscFunctionBegin;
1188574a98acSMatthew G. Knepley   ierr = DMGetLocalVector(dm, &localX);CHKERRQ(ierr);
11895d42b983SMatthew G. Knepley   ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL);CHKERRQ(ierr);
1190574a98acSMatthew G. Knepley   ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
1191574a98acSMatthew G. Knepley   ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
1192574a98acSMatthew G. Knepley   ierr = DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff);CHKERRQ(ierr);
1193574a98acSMatthew G. Knepley   ierr = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr);
1194574a98acSMatthew G. Knepley   PetscFunctionReturn(0);
1195574a98acSMatthew G. Knepley }
1196574a98acSMatthew G. Knepley 
1197574a98acSMatthew G. Knepley /*@C
1198ca3d3a14SMatthew G. Knepley   DMComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.
1199574a98acSMatthew G. Knepley 
1200d083f849SBarry Smith   Collective on dm
1201c0f8e1fdSMatthew G. Knepley 
1202574a98acSMatthew G. Knepley   Input Parameters:
1203574a98acSMatthew G. Knepley + dm     - The DM
1204574a98acSMatthew G. Knepley . time   - The time
1205574a98acSMatthew G. Knepley . funcs  - The functions to evaluate for each field component
1206574a98acSMatthew G. Knepley . ctxs   - Optional array of contexts to pass to each function, or NULL.
1207574a98acSMatthew G. Knepley - localX - The coefficient vector u_h, a local vector
1208574a98acSMatthew G. Knepley 
1209574a98acSMatthew G. Knepley   Output Parameter:
1210574a98acSMatthew G. Knepley . diff - The diff ||u - u_h||_2
1211574a98acSMatthew G. Knepley 
1212574a98acSMatthew G. Knepley   Level: developer
1213574a98acSMatthew G. Knepley 
1214574a98acSMatthew G. Knepley .seealso: DMProjectFunction(), DMComputeL2FieldDiff(), DMComputeL2GradientDiff()
1215574a98acSMatthew G. Knepley @*/
1216574a98acSMatthew G. Knepley PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff)
1217574a98acSMatthew G. Knepley {
12180f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1219ca3d3a14SMatthew G. Knepley   DM               tdm;
1220ca3d3a14SMatthew G. Knepley   Vec              tv;
1221cb1e1211SMatthew G Knepley   PetscSection     section;
1222c5bbbd5bSMatthew G. Knepley   PetscQuadrature  quad;
12234bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
122415496722SMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
12254bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
1226cb1e1211SMatthew G Knepley   PetscReal        localDiff = 0.0;
12277318780aSToby Isaac   const PetscReal *quadWeights;
1228412e9a14SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset;
1229ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1230cb1e1211SMatthew G Knepley   PetscErrorCode   ierr;
1231cb1e1211SMatthew G Knepley 
1232cb1e1211SMatthew G Knepley   PetscFunctionBegin;
1233c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
12347318780aSToby Isaac   ierr = DMGetCoordinateDim(dm, &coordDim);CHKERRQ(ierr);
12352a4e142eSMatthew G. Knepley   fegeom.dimEmbed = coordDim;
123692fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1237cb1e1211SMatthew G Knepley   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
1238ca3d3a14SMatthew G. Knepley   ierr = DMGetBasisTransformDM_Internal(dm, &tdm);CHKERRQ(ierr);
1239ca3d3a14SMatthew G. Knepley   ierr = DMGetBasisTransformVec_Internal(dm, &tv);CHKERRQ(ierr);
1240ca3d3a14SMatthew G. Knepley   ierr = DMHasBasisTransform(dm, &transform);CHKERRQ(ierr);
1241cb1e1211SMatthew G Knepley   for (field = 0; field < numFields; ++field) {
124215496722SMatthew G. Knepley     PetscObject  obj;
124315496722SMatthew G. Knepley     PetscClassId id;
1244c5bbbd5bSMatthew G. Knepley     PetscInt     Nc;
1245c5bbbd5bSMatthew G. Knepley 
124644a7f3ddSMatthew G. Knepley     ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
124715496722SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
124815496722SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
124915496722SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
125015496722SMatthew G. Knepley 
12510f2d7e86SMatthew G. Knepley       ierr = PetscFEGetQuadrature(fe, &quad);CHKERRQ(ierr);
12520f2d7e86SMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
125315496722SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
125415496722SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
125515496722SMatthew G. Knepley 
125615496722SMatthew G. Knepley       ierr = PetscFVGetQuadrature(fv, &quad);CHKERRQ(ierr);
125715496722SMatthew G. Knepley       ierr = PetscFVGetNumComponents(fv, &Nc);CHKERRQ(ierr);
1258ff1e0c32SBarry Smith     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
1259c5bbbd5bSMatthew G. Knepley     numComponents += Nc;
1260cb1e1211SMatthew G Knepley   }
12619c3cf19fSMatthew G. Knepley   ierr = PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights);CHKERRQ(ierr);
1262beaa55a6SMatthew G. Knepley   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
12632a4e142eSMatthew G. Knepley   ierr = PetscMalloc6(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ);CHKERRQ(ierr);
1264aed3cbd0SMatthew G. Knepley   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1265412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
1266cb1e1211SMatthew G Knepley   for (c = cStart; c < cEnd; ++c) {
1267a1e44745SMatthew G. Knepley     PetscScalar *x = NULL;
1268cb1e1211SMatthew G Knepley     PetscReal    elemDiff = 0.0;
12699c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1270cb1e1211SMatthew G Knepley 
12712a4e142eSMatthew G. Knepley     ierr = DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ);CHKERRQ(ierr);
1272cb1e1211SMatthew G Knepley     ierr = DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr);
1273cb1e1211SMatthew G Knepley 
127415496722SMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
127515496722SMatthew G. Knepley       PetscObject  obj;
127615496722SMatthew G. Knepley       PetscClassId id;
1277c110b1eeSGeoffrey Irving       void * const ctx = ctxs ? ctxs[field] : NULL;
127815496722SMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1279cb1e1211SMatthew G Knepley 
128044a7f3ddSMatthew G. Knepley       ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
128115496722SMatthew G. Knepley       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
128215496722SMatthew G. Knepley       if (id == PETSCFE_CLASSID)      {ierr = PetscFEGetNumComponents((PetscFE) obj, &Nc);CHKERRQ(ierr);ierr = PetscFEGetDimension((PetscFE) obj, &Nb);CHKERRQ(ierr);}
128315496722SMatthew G. Knepley       else if (id == PETSCFV_CLASSID) {ierr = PetscFVGetNumComponents((PetscFV) obj, &Nc);CHKERRQ(ierr);Nb = 1;}
1284ff1e0c32SBarry Smith       else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
1285cb1e1211SMatthew G Knepley       if (debug) {
1286cb1e1211SMatthew G Knepley         char title[1024];
1287ff1e0c32SBarry Smith         ierr = PetscSNPrintf(title, 1023, "Solution for Field %D", field);CHKERRQ(ierr);
12884c848028SMatthew G. Knepley         ierr = DMPrintCellVector(c, title, Nb, &x[fieldOffset]);CHKERRQ(ierr);
1289cb1e1211SMatthew G Knepley       }
12907318780aSToby Isaac       for (q = 0; q < Nq; ++q) {
12912a4e142eSMatthew G. Knepley         PetscFEGeom qgeom;
12922a4e142eSMatthew G. Knepley 
12932a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
12942a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q*coordDim*coordDim];
12952a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
12962a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
1297ff1e0c32SBarry Smith         if (fegeom.detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, point %D", (double)fegeom.detJ[q], c, q);
1298d3a7d86cSMatthew G. Knepley         if (transform) {
1299d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*Nq];
1300d3a7d86cSMatthew G. Knepley           ierr = DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim*q], PETSC_TRUE, coordDim, &coords[coordDim*q], gcoords, dm->transformCtx);CHKERRQ(ierr);
1301d3a7d86cSMatthew G. Knepley         } else {
1302d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*q];
1303d3a7d86cSMatthew G. Knepley         }
1304ca3d3a14SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx);
1305e735a8a9SMatthew G. Knepley         if (ierr) {
1306e735a8a9SMatthew G. Knepley           PetscErrorCode ierr2;
1307e735a8a9SMatthew G. Knepley           ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
1308e735a8a9SMatthew G. Knepley           ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
13092a4e142eSMatthew G. Knepley           ierr2 = PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr2);
1310e735a8a9SMatthew G. Knepley           CHKERRQ(ierr);
1311e735a8a9SMatthew G. Knepley         }
1312ca3d3a14SMatthew G. Knepley         if (transform) {ierr = DMPlexBasisTransformApply_Internal(dm, &coords[coordDim*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx);CHKERRQ(ierr);}
13132a4e142eSMatthew G. Knepley         if (id == PETSCFE_CLASSID)      {ierr = PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], &qgeom, q, interpolant);CHKERRQ(ierr);}
131415496722SMatthew G. Knepley         else if (id == PETSCFV_CLASSID) {ierr = PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant);CHKERRQ(ierr);}
1315ff1e0c32SBarry Smith         else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
131615496722SMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1317beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
131880dbbc5dSMatthew G. Knepley           if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "    elem %D field %D,%D point %g %g %g diff %g\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim*q] : 0.), (double)(coordDim > 1 ? coords[coordDim*q+1] : 0.),(double)(coordDim > 2 ? coords[coordDim*q+2] : 0.), (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q]));CHKERRQ(ierr);}
13194bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
1320cb1e1211SMatthew G Knepley         }
1321cb1e1211SMatthew G Knepley       }
13229c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
1323beaa55a6SMatthew G. Knepley       qc += Nc;
1324cb1e1211SMatthew G Knepley     }
1325cb1e1211SMatthew G Knepley     ierr = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr);
1326ff1e0c32SBarry Smith     if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "  elem %D diff %g\n", c, (double)elemDiff);CHKERRQ(ierr);}
1327cb1e1211SMatthew G Knepley     localDiff += elemDiff;
1328cb1e1211SMatthew G Knepley   }
13292a4e142eSMatthew G. Knepley   ierr  = PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr);
1330820f2d46SBarry Smith   ierr  = MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1331cb1e1211SMatthew G Knepley   *diff = PetscSqrtReal(*diff);
1332cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1333cb1e1211SMatthew G Knepley }
1334cb1e1211SMatthew G Knepley 
1335b698f381SToby Isaac PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff)
1336cb1e1211SMatthew G Knepley {
13370f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1338ca3d3a14SMatthew G. Knepley   DM               tdm;
1339cb1e1211SMatthew G Knepley   PetscSection     section;
134040e14135SMatthew G. Knepley   PetscQuadrature  quad;
1341ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
13429c3cf19fSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
13432a4e142eSMatthew G. Knepley   const PetscReal *quadWeights;
13444bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
13454bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
134640e14135SMatthew G. Knepley   PetscReal        localDiff = 0.0;
1347485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset;
1348ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1349cb1e1211SMatthew G Knepley   PetscErrorCode   ierr;
1350cb1e1211SMatthew G Knepley 
1351cb1e1211SMatthew G Knepley   PetscFunctionBegin;
1352c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
13537318780aSToby Isaac   ierr = DMGetCoordinateDim(dm, &coordDim);CHKERRQ(ierr);
13544bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
135592fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
135640e14135SMatthew G. Knepley   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
135740e14135SMatthew G. Knepley   ierr = DMGetLocalVector(dm, &localX);CHKERRQ(ierr);
135840e14135SMatthew G. Knepley   ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
135940e14135SMatthew G. Knepley   ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
1360ca3d3a14SMatthew G. Knepley   ierr = DMGetBasisTransformDM_Internal(dm, &tdm);CHKERRQ(ierr);
1361ca3d3a14SMatthew G. Knepley   ierr = DMGetBasisTransformVec_Internal(dm, &tv);CHKERRQ(ierr);
1362ca3d3a14SMatthew G. Knepley   ierr = DMHasBasisTransform(dm, &transform);CHKERRQ(ierr);
1363652b88e8SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
13640f2d7e86SMatthew G. Knepley     PetscFE  fe;
136540e14135SMatthew G. Knepley     PetscInt Nc;
1366652b88e8SMatthew G. Knepley 
136744a7f3ddSMatthew G. Knepley     ierr = DMGetField(dm, field, NULL, (PetscObject *) &fe);CHKERRQ(ierr);
13680f2d7e86SMatthew G. Knepley     ierr = PetscFEGetQuadrature(fe, &quad);CHKERRQ(ierr);
13690f2d7e86SMatthew G. Knepley     ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
137040e14135SMatthew G. Knepley     numComponents += Nc;
1371652b88e8SMatthew G. Knepley   }
13722a4e142eSMatthew G. Knepley   ierr = PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights);CHKERRQ(ierr);
1373beaa55a6SMatthew G. Knepley   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
13744d6f44ffSToby Isaac   /* ierr = DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX);CHKERRQ(ierr); */
13754bee2e38SMatthew G. Knepley   ierr = PetscMalloc6(numComponents,&funcVal,coordDim*Nq,&coords,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ,numComponents*coordDim,&interpolant,Nq,&fegeom.detJ);CHKERRQ(ierr);
1376412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
137740e14135SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
137840e14135SMatthew G. Knepley     PetscScalar *x = NULL;
137940e14135SMatthew G. Knepley     PetscReal    elemDiff = 0.0;
13809c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1381652b88e8SMatthew G. Knepley 
13824bee2e38SMatthew G. Knepley     ierr = DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ);CHKERRQ(ierr);
138340e14135SMatthew G. Knepley     ierr = DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr);
138440e14135SMatthew G. Knepley 
13859c3cf19fSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
13860f2d7e86SMatthew G. Knepley       PetscFE          fe;
138751259fa3SMatthew G. Knepley       void * const     ctx = ctxs ? ctxs[field] : NULL;
13889c3cf19fSMatthew G. Knepley       PetscInt         Nb, Nc, q, fc;
138940e14135SMatthew G. Knepley 
139044a7f3ddSMatthew G. Knepley       ierr = DMGetField(dm, field, NULL, (PetscObject *) &fe);CHKERRQ(ierr);
13910f2d7e86SMatthew G. Knepley       ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
13929c3cf19fSMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
139340e14135SMatthew G. Knepley       if (debug) {
139440e14135SMatthew G. Knepley         char title[1024];
1395ff1e0c32SBarry Smith         ierr = PetscSNPrintf(title, 1023, "Solution for Field %D", field);CHKERRQ(ierr);
13969c3cf19fSMatthew G. Knepley         ierr = DMPrintCellVector(c, title, Nb, &x[fieldOffset]);CHKERRQ(ierr);
1397652b88e8SMatthew G. Knepley       }
13989c3cf19fSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
13992a4e142eSMatthew G. Knepley         PetscFEGeom qgeom;
14002a4e142eSMatthew G. Knepley 
14012a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
14022a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q*coordDim*coordDim];
14032a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
14042a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
1405ff1e0c32SBarry Smith         if (fegeom.detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature points %D", (double)fegeom.detJ[q], c, q);
1406d3a7d86cSMatthew G. Knepley         if (transform) {
1407d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*Nq];
1408d3a7d86cSMatthew G. Knepley           ierr = DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim*q], PETSC_TRUE, coordDim, &coords[coordDim*q], gcoords, dm->transformCtx);CHKERRQ(ierr);
1409d3a7d86cSMatthew G. Knepley         } else {
1410d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*q];
1411d3a7d86cSMatthew G. Knepley         }
14124bee2e38SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx);
1413e735a8a9SMatthew G. Knepley         if (ierr) {
1414e735a8a9SMatthew G. Knepley           PetscErrorCode ierr2;
1415e735a8a9SMatthew G. Knepley           ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
1416e735a8a9SMatthew G. Knepley           ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
14174bee2e38SMatthew G. Knepley           ierr2 = PetscFree6(funcVal,coords,fegeom.J,fegeom.invJ,interpolant,fegeom.detJ);CHKERRQ(ierr2);
1418e735a8a9SMatthew G. Knepley           CHKERRQ(ierr);
1419e735a8a9SMatthew G. Knepley         }
1420ca3d3a14SMatthew G. Knepley         if (transform) {ierr = DMPlexBasisTransformApply_Internal(dm, &coords[coordDim*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx);CHKERRQ(ierr);}
1421f9244615SMatthew G. Knepley         ierr = PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant);CHKERRQ(ierr);
14224bee2e38SMatthew G. Knepley         /* Overwrite with the dot product if the normal is given */
14234bee2e38SMatthew G. Knepley         if (n) {
14244bee2e38SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
14254bee2e38SMatthew G. Knepley             PetscScalar sum = 0.0;
14264bee2e38SMatthew G. Knepley             PetscInt    d;
14274bee2e38SMatthew G. Knepley             for (d = 0; d < dim; ++d) sum += interpolant[fc*dim+d]*n[d];
14284bee2e38SMatthew G. Knepley             interpolant[fc] = sum;
14294bee2e38SMatthew G. Knepley           }
14304bee2e38SMatthew G. Knepley         }
14319c3cf19fSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1432beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
1433ff1e0c32SBarry Smith           if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "    elem %D fieldDer %D,%D diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q]));CHKERRQ(ierr);}
14344bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
143540e14135SMatthew G. Knepley         }
143640e14135SMatthew G. Knepley       }
14379c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
14389c3cf19fSMatthew G. Knepley       qc          += Nc;
143940e14135SMatthew G. Knepley     }
144040e14135SMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr);
1441ff1e0c32SBarry Smith     if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "  elem %D diff %g\n", c, (double)elemDiff);CHKERRQ(ierr);}
144240e14135SMatthew G. Knepley     localDiff += elemDiff;
144340e14135SMatthew G. Knepley   }
14444bee2e38SMatthew G. Knepley   ierr  = PetscFree6(funcVal,coords,fegeom.J,fegeom.invJ,interpolant,fegeom.detJ);CHKERRQ(ierr);
144540e14135SMatthew G. Knepley   ierr  = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr);
1446820f2d46SBarry Smith   ierr  = MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
144740e14135SMatthew G. Knepley   *diff = PetscSqrtReal(*diff);
1448cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1449cb1e1211SMatthew G Knepley }
1450cb1e1211SMatthew G Knepley 
1451c6eecec3SToby Isaac PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
145273d901b8SMatthew G. Knepley {
14530f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1454ca3d3a14SMatthew G. Knepley   DM               tdm;
1455083401c6SMatthew G. Knepley   DMLabel          depthLabel;
145673d901b8SMatthew G. Knepley   PetscSection     section;
1457ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
145873d901b8SMatthew G. Knepley   PetscReal       *localDiff;
1459083401c6SMatthew G. Knepley   PetscInt         dim, depth, dE, Nf, f, Nds, s;
1460ca3d3a14SMatthew G. Knepley   PetscBool        transform;
146173d901b8SMatthew G. Knepley   PetscErrorCode   ierr;
146273d901b8SMatthew G. Knepley 
146373d901b8SMatthew G. Knepley   PetscFunctionBegin;
1464c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1465083401c6SMatthew G. Knepley   ierr = DMGetCoordinateDim(dm, &dE);CHKERRQ(ierr);
146692fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
146773d901b8SMatthew G. Knepley   ierr = DMGetLocalVector(dm, &localX);CHKERRQ(ierr);
1468083401c6SMatthew G. Knepley   ierr = DMGetBasisTransformDM_Internal(dm, &tdm);CHKERRQ(ierr);
1469083401c6SMatthew G. Knepley   ierr = DMGetBasisTransformVec_Internal(dm, &tv);CHKERRQ(ierr);
1470083401c6SMatthew G. Knepley   ierr = DMHasBasisTransform(dm, &transform);CHKERRQ(ierr);
1471083401c6SMatthew G. Knepley   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
1472083401c6SMatthew G. Knepley   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1473083401c6SMatthew G. Knepley   ierr = DMLabelGetNumValues(depthLabel, &depth);CHKERRQ(ierr);
1474083401c6SMatthew G. Knepley 
1475ca3d3a14SMatthew G. Knepley   ierr = VecSet(localX, 0.0);CHKERRQ(ierr);
147673d901b8SMatthew G. Knepley   ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
147773d901b8SMatthew G. Knepley   ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
1478ca3d3a14SMatthew G. Knepley   ierr = DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX);CHKERRQ(ierr);
1479083401c6SMatthew G. Knepley   ierr = DMGetNumDS(dm, &Nds);CHKERRQ(ierr);
1480083401c6SMatthew G. Knepley   ierr = PetscCalloc1(Nf, &localDiff);CHKERRQ(ierr);
1481083401c6SMatthew G. Knepley   for (s = 0; s < Nds; ++s) {
1482083401c6SMatthew G. Knepley     PetscDS          ds;
1483083401c6SMatthew G. Knepley     DMLabel          label;
1484083401c6SMatthew G. Knepley     IS               fieldIS, pointIS;
1485083401c6SMatthew G. Knepley     const PetscInt  *fields, *points = NULL;
1486083401c6SMatthew G. Knepley     PetscQuadrature  quad;
1487083401c6SMatthew G. Knepley     const PetscReal *quadPoints, *quadWeights;
1488083401c6SMatthew G. Knepley     PetscFEGeom      fegeom;
1489083401c6SMatthew G. Knepley     PetscReal       *coords, *gcoords;
1490083401c6SMatthew G. Knepley     PetscScalar     *funcVal, *interpolant;
149196959cd1SMatthew G. Knepley     PetscBool        isHybrid;
1492083401c6SMatthew G. Knepley     PetscInt         qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf;
149373d901b8SMatthew G. Knepley 
1494083401c6SMatthew G. Knepley     ierr = DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds);CHKERRQ(ierr);
1495083401c6SMatthew G. Knepley     ierr = ISGetIndices(fieldIS, &fields);CHKERRQ(ierr);
149696959cd1SMatthew G. Knepley     ierr = PetscDSGetHybrid(ds, &isHybrid);CHKERRQ(ierr);
1497083401c6SMatthew G. Knepley     ierr = PetscDSGetNumFields(ds, &dsNf);CHKERRQ(ierr);
1498083401c6SMatthew G. Knepley     ierr = PetscDSGetTotalComponents(ds, &totNc);CHKERRQ(ierr);
1499083401c6SMatthew G. Knepley     ierr = PetscDSGetQuadrature(ds, &quad);CHKERRQ(ierr);
15009c3cf19fSMatthew G. Knepley     ierr = PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);CHKERRQ(ierr);
1501083401c6SMatthew G. Knepley     if ((qNc != 1) && (qNc != totNc)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, totNc);
1502083401c6SMatthew G. Knepley     ierr = PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE*(Nq+1), &coords,Nq, &fegeom.detJ, dE*dE*Nq, &fegeom.J, dE*dE*Nq, &fegeom.invJ);CHKERRQ(ierr);
1503083401c6SMatthew G. Knepley     if (!label) {
1504412e9a14SMatthew G. Knepley       ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1505083401c6SMatthew G. Knepley     } else {
1506083401c6SMatthew G. Knepley       ierr = DMLabelGetStratumIS(label, 1, &pointIS);CHKERRQ(ierr);
1507083401c6SMatthew G. Knepley       ierr = ISGetLocalSize(pointIS, &cEnd);CHKERRQ(ierr);
1508083401c6SMatthew G. Knepley       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
1509083401c6SMatthew G. Knepley     }
151073d901b8SMatthew G. Knepley     for (c = cStart; c < cEnd; ++c) {
1511083401c6SMatthew G. Knepley       const PetscInt cell = points ? points[c] : c;
151273d901b8SMatthew G. Knepley       PetscScalar   *x    = NULL;
151396959cd1SMatthew G. Knepley       PetscInt       qc   = 0, fOff = 0, dep, fStart = isHybrid ? dsNf-1 : 0;
151473d901b8SMatthew G. Knepley 
1515083401c6SMatthew G. Knepley       ierr = DMLabelGetValue(depthLabel, cell, &dep);CHKERRQ(ierr);
1516083401c6SMatthew G. Knepley       if (dep != depth-1) continue;
151796959cd1SMatthew G. Knepley       if (isHybrid) {
151896959cd1SMatthew G. Knepley         const PetscInt *cone;
151996959cd1SMatthew G. Knepley 
152096959cd1SMatthew G. Knepley         ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
152196959cd1SMatthew G. Knepley         ierr = DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ);CHKERRQ(ierr);
152296959cd1SMatthew G. Knepley       } else {
1523083401c6SMatthew G. Knepley         ierr = DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ);CHKERRQ(ierr);
152496959cd1SMatthew G. Knepley       }
1525083401c6SMatthew G. Knepley       ierr = DMPlexVecGetClosure(dm, NULL, localX, cell, NULL, &x);CHKERRQ(ierr);
152696959cd1SMatthew G. Knepley       for (f = fStart; f < dsNf; ++f) {
152715496722SMatthew G. Knepley         PetscObject  obj;
152815496722SMatthew G. Knepley         PetscClassId id;
1529083401c6SMatthew G. Knepley         void * const ctx = ctxs ? ctxs[fields[f]] : NULL;
153015496722SMatthew G. Knepley         PetscInt     Nb, Nc, q, fc;
153115496722SMatthew G. Knepley         PetscReal    elemDiff = 0.0;
153215496722SMatthew G. Knepley 
1533083401c6SMatthew G. Knepley         ierr = PetscDSGetDiscretization(ds, f, &obj);CHKERRQ(ierr);
153415496722SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
153515496722SMatthew G. Knepley         if (id == PETSCFE_CLASSID)      {ierr = PetscFEGetNumComponents((PetscFE) obj, &Nc);CHKERRQ(ierr);ierr = PetscFEGetDimension((PetscFE) obj, &Nb);CHKERRQ(ierr);}
153615496722SMatthew G. Knepley         else if (id == PETSCFV_CLASSID) {ierr = PetscFVGetNumComponents((PetscFV) obj, &Nc);CHKERRQ(ierr);Nb = 1;}
1537083401c6SMatthew G. Knepley         else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", fields[f]);
153873d901b8SMatthew G. Knepley         if (debug) {
153973d901b8SMatthew G. Knepley           char title[1024];
1540083401c6SMatthew G. Knepley           ierr = PetscSNPrintf(title, 1023, "Solution for Field %D", fields[f]);CHKERRQ(ierr);
1541083401c6SMatthew G. Knepley           ierr = DMPrintCellVector(cell, title, Nb, &x[fOff]);CHKERRQ(ierr);
154273d901b8SMatthew G. Knepley         }
15437318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
15442a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
15452a4e142eSMatthew G. Knepley 
15462a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
1547083401c6SMatthew G. Knepley           qgeom.J        = &fegeom.J[q*dE*dE];
1548083401c6SMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*dE*dE];
15492a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
1550083401c6SMatthew G. Knepley           if (fegeom.detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %D, quadrature point %D", (double)fegeom.detJ[q], cell, q);
1551d3a7d86cSMatthew G. Knepley           if (transform) {
1552083401c6SMatthew G. Knepley             gcoords = &coords[dE*Nq];
1553083401c6SMatthew G. Knepley             ierr = DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE*q], PETSC_TRUE, dE, &coords[dE*q], gcoords, dm->transformCtx);CHKERRQ(ierr);
1554d3a7d86cSMatthew G. Knepley           } else {
1555083401c6SMatthew G. Knepley             gcoords = &coords[dE*q];
1556d3a7d86cSMatthew G. Knepley           }
1557083401c6SMatthew G. Knepley           ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx);
1558e735a8a9SMatthew G. Knepley           if (ierr) {
1559e735a8a9SMatthew G. Knepley             PetscErrorCode ierr2;
1560083401c6SMatthew G. Knepley             ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x);CHKERRQ(ierr2);
1561e735a8a9SMatthew G. Knepley             ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
1562083401c6SMatthew G. Knepley             ierr2 = PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr2);
1563e735a8a9SMatthew G. Knepley             CHKERRQ(ierr);
1564e735a8a9SMatthew G. Knepley           }
1565083401c6SMatthew G. Knepley           if (transform) {ierr = DMPlexBasisTransformApply_Internal(dm, &coords[dE*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx);CHKERRQ(ierr);}
156696959cd1SMatthew G. Knepley           /* Call once for each face, except for lagrange field */
1567083401c6SMatthew G. Knepley           if (id == PETSCFE_CLASSID)      {ierr = PetscFEInterpolate_Static((PetscFE) obj, &x[fOff], &qgeom, q, interpolant);CHKERRQ(ierr);}
1568083401c6SMatthew G. Knepley           else if (id == PETSCFV_CLASSID) {ierr = PetscFVInterpolate_Static((PetscFV) obj, &x[fOff], q, interpolant);CHKERRQ(ierr);}
1569083401c6SMatthew G. Knepley           else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", fields[f]);
157015496722SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1571beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
1572083401c6SMatthew G. Knepley             if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "    cell %D field %D,%D point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE*q] : 0.), (double)(dE > 1 ? coords[dE*q+1] : 0.),(double)(dE > 2 ? coords[dE*q+2] : 0.), (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q]));CHKERRQ(ierr);}
15734bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
157473d901b8SMatthew G. Knepley           }
157573d901b8SMatthew G. Knepley         }
1576083401c6SMatthew G. Knepley         fOff += Nb;
15779c3cf19fSMatthew G. Knepley         qc   += Nc;
1578083401c6SMatthew G. Knepley         localDiff[fields[f]] += elemDiff;
1579083401c6SMatthew G. Knepley         if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "  cell %D field %D cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]]);CHKERRQ(ierr);}
158073d901b8SMatthew G. Knepley       }
1581083401c6SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x);CHKERRQ(ierr);
1582083401c6SMatthew G. Knepley     }
1583083401c6SMatthew G. Knepley     if (label) {
1584083401c6SMatthew G. Knepley       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
1585083401c6SMatthew G. Knepley       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1586083401c6SMatthew G. Knepley     }
1587083401c6SMatthew G. Knepley     ierr = ISRestoreIndices(fieldIS, &fields);CHKERRQ(ierr);
1588083401c6SMatthew G. Knepley     ierr = PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ);CHKERRQ(ierr);
158973d901b8SMatthew G. Knepley   }
159073d901b8SMatthew G. Knepley   ierr = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr);
1591820f2d46SBarry Smith   ierr = MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm));CHKERRMPI(ierr);
1592083401c6SMatthew G. Knepley   ierr = PetscFree(localDiff);CHKERRQ(ierr);
1593083401c6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]);
159473d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
159573d901b8SMatthew G. Knepley }
159673d901b8SMatthew G. Knepley 
1597e729f68cSMatthew G. Knepley /*@C
1598e729f68cSMatthew G. Knepley   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.
1599e729f68cSMatthew G. Knepley 
1600d083f849SBarry Smith   Collective on dm
1601c0f8e1fdSMatthew G. Knepley 
1602e729f68cSMatthew G. Knepley   Input Parameters:
1603e729f68cSMatthew G. Knepley + dm    - The DM
16040163fd50SMatthew G. Knepley . time  - The time
1605ca3eba1bSToby Isaac . funcs - The functions to evaluate for each field component: NULL means that component does not contribute to error calculation
1606e729f68cSMatthew G. Knepley . ctxs  - Optional array of contexts to pass to each function, or NULL.
1607e729f68cSMatthew G. Knepley - X     - The coefficient vector u_h
1608e729f68cSMatthew G. Knepley 
1609e729f68cSMatthew G. Knepley   Output Parameter:
1610e729f68cSMatthew G. Knepley . D - A Vec which holds the difference ||u - u_h||_2 for each cell
1611e729f68cSMatthew G. Knepley 
1612e729f68cSMatthew G. Knepley   Level: developer
1613e729f68cSMatthew G. Knepley 
1614b698f381SToby Isaac .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
1615e729f68cSMatthew G. Knepley @*/
16160163fd50SMatthew G. Knepley PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D)
1617e729f68cSMatthew G. Knepley {
1618e729f68cSMatthew G. Knepley   PetscSection     section;
1619e729f68cSMatthew G. Knepley   PetscQuadrature  quad;
1620e729f68cSMatthew G. Knepley   Vec              localX;
16214bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
1622e729f68cSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
16234bee2e38SMatthew G. Knepley   PetscReal       *coords;
1624e729f68cSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1625485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset;
1626e729f68cSMatthew G. Knepley   PetscErrorCode   ierr;
1627e729f68cSMatthew G. Knepley 
1628e729f68cSMatthew G. Knepley   PetscFunctionBegin;
1629e729f68cSMatthew G. Knepley   ierr = VecSet(D, 0.0);CHKERRQ(ierr);
1630e729f68cSMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
16317318780aSToby Isaac   ierr = DMGetCoordinateDim(dm, &coordDim);CHKERRQ(ierr);
163292fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1633e729f68cSMatthew G. Knepley   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
1634e729f68cSMatthew G. Knepley   ierr = DMGetLocalVector(dm, &localX);CHKERRQ(ierr);
1635bdd6f66aSToby Isaac   ierr = DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX);CHKERRQ(ierr);
1636e729f68cSMatthew G. Knepley   ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
1637e729f68cSMatthew G. Knepley   ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);CHKERRQ(ierr);
1638e729f68cSMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
1639e729f68cSMatthew G. Knepley     PetscObject  obj;
1640e729f68cSMatthew G. Knepley     PetscClassId id;
1641e729f68cSMatthew G. Knepley     PetscInt     Nc;
1642e729f68cSMatthew G. Knepley 
164344a7f3ddSMatthew G. Knepley     ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
1644e729f68cSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
1645e729f68cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1646e729f68cSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
1647e729f68cSMatthew G. Knepley 
1648e729f68cSMatthew G. Knepley       ierr = PetscFEGetQuadrature(fe, &quad);CHKERRQ(ierr);
1649e729f68cSMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
1650e729f68cSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
1651e729f68cSMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
1652e729f68cSMatthew G. Knepley 
1653e729f68cSMatthew G. Knepley       ierr = PetscFVGetQuadrature(fv, &quad);CHKERRQ(ierr);
1654e729f68cSMatthew G. Knepley       ierr = PetscFVGetNumComponents(fv, &Nc);CHKERRQ(ierr);
1655ff1e0c32SBarry Smith     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
1656e729f68cSMatthew G. Knepley     numComponents += Nc;
1657e729f68cSMatthew G. Knepley   }
16589c3cf19fSMatthew G. Knepley   ierr = PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);CHKERRQ(ierr);
1659beaa55a6SMatthew G. Knepley   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
16602a4e142eSMatthew G. Knepley   ierr = PetscMalloc6(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ);CHKERRQ(ierr);
1661412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1662e729f68cSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
1663e729f68cSMatthew G. Knepley     PetscScalar *x = NULL;
16646f288a59SMatthew G. Knepley     PetscScalar  elemDiff = 0.0;
16659c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1666e729f68cSMatthew G. Knepley 
16672a4e142eSMatthew G. Knepley     ierr = DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ);CHKERRQ(ierr);
1668e729f68cSMatthew G. Knepley     ierr = DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr);
1669e729f68cSMatthew G. Knepley 
1670e729f68cSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1671e729f68cSMatthew G. Knepley       PetscObject  obj;
1672e729f68cSMatthew G. Knepley       PetscClassId id;
1673e729f68cSMatthew G. Knepley       void * const ctx = ctxs ? ctxs[field] : NULL;
1674e729f68cSMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1675e729f68cSMatthew G. Knepley 
167644a7f3ddSMatthew G. Knepley       ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
1677e729f68cSMatthew G. Knepley       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
1678e729f68cSMatthew G. Knepley       if (id == PETSCFE_CLASSID)      {ierr = PetscFEGetNumComponents((PetscFE) obj, &Nc);CHKERRQ(ierr);ierr = PetscFEGetDimension((PetscFE) obj, &Nb);CHKERRQ(ierr);}
1679e729f68cSMatthew G. Knepley       else if (id == PETSCFV_CLASSID) {ierr = PetscFVGetNumComponents((PetscFV) obj, &Nc);CHKERRQ(ierr);Nb = 1;}
1680ff1e0c32SBarry Smith       else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
168123f34ed2SToby Isaac       if (funcs[field]) {
16827318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
16832a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
16842a4e142eSMatthew G. Knepley 
16852a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
16862a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*coordDim*coordDim];
16872a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
16882a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
16894bee2e38SMatthew G. Knepley           if (fegeom.detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature points %D", (double)fegeom.detJ[q], c, q);
1690274e8aaeSMatthew G. Knepley           ierr = (*funcs[field])(coordDim, time, &coords[q*coordDim], Nc, funcVal, ctx);
1691e735a8a9SMatthew G. Knepley           if (ierr) {
1692e735a8a9SMatthew G. Knepley             PetscErrorCode ierr2;
1693e735a8a9SMatthew G. Knepley             ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
16942a4e142eSMatthew G. Knepley             ierr2 = PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr2);
1695e735a8a9SMatthew G. Knepley             ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
1696e735a8a9SMatthew G. Knepley             CHKERRQ(ierr);
1697e735a8a9SMatthew G. Knepley           }
16982a4e142eSMatthew G. Knepley           if (id == PETSCFE_CLASSID)      {ierr = PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], &qgeom, q, interpolant);CHKERRQ(ierr);}
1699e729f68cSMatthew G. Knepley           else if (id == PETSCFV_CLASSID) {ierr = PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant);CHKERRQ(ierr);}
1700ff1e0c32SBarry Smith           else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
1701e729f68cSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1702beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
17034bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
1704e729f68cSMatthew G. Knepley           }
1705e729f68cSMatthew G. Knepley         }
170623f34ed2SToby Isaac       }
1707beaa55a6SMatthew G. Knepley       fieldOffset += Nb;
17089c3cf19fSMatthew G. Knepley       qc          += Nc;
1709e729f68cSMatthew G. Knepley     }
1710e729f68cSMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr);
171123f34ed2SToby Isaac     ierr = VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES);CHKERRQ(ierr);
1712e729f68cSMatthew G. Knepley   }
17132a4e142eSMatthew G. Knepley   ierr = PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr);
1714e729f68cSMatthew G. Knepley   ierr = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr);
1715e729f68cSMatthew G. Knepley   ierr = VecSqrtAbs(D);CHKERRQ(ierr);
1716e729f68cSMatthew G. Knepley   PetscFunctionReturn(0);
1717e729f68cSMatthew G. Knepley }
1718e729f68cSMatthew G. Knepley 
17191555c271SMatthew G. Knepley /*@C
17201555c271SMatthew G. Knepley   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1, and stores it in a Vec.
17211555c271SMatthew G. Knepley 
1722d083f849SBarry Smith   Collective on dm
1723c0f8e1fdSMatthew G. Knepley 
17241555c271SMatthew G. Knepley   Input Parameters:
17251555c271SMatthew G. Knepley + dm - The DM
17261555c271SMatthew G. Knepley - LocX  - The coefficient vector u_h
17271555c271SMatthew G. Knepley 
17281555c271SMatthew G. Knepley   Output Parameter:
17291555c271SMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the gradient
17301555c271SMatthew G. Knepley 
173195452b02SPatrick Sanan   Notes:
173295452b02SPatrick Sanan     Add citation to (Clement, 1975) and definition of the interpolant
17331555c271SMatthew G. Knepley   \nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
17341555c271SMatthew G. Knepley 
17351555c271SMatthew G. Knepley   Level: developer
17361555c271SMatthew G. Knepley 
17371555c271SMatthew G. Knepley .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
17381555c271SMatthew G. Knepley @*/
17391555c271SMatthew G. Knepley PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC)
17401555c271SMatthew G. Knepley {
1741db1066baSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *) dm->data;
1742db1066baSMatthew G. Knepley   PetscInt         debug = mesh->printFEM;
17431555c271SMatthew G. Knepley   DM               dmC;
17441555c271SMatthew G. Knepley   PetscSection     section;
17451555c271SMatthew G. Knepley   PetscQuadrature  quad;
17461555c271SMatthew G. Knepley   PetscScalar     *interpolant, *gradsum;
17474bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
17484bee2e38SMatthew G. Knepley   PetscReal       *coords;
17491555c271SMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1750485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset;
17511555c271SMatthew G. Knepley   PetscErrorCode   ierr;
17521555c271SMatthew G. Knepley 
17531555c271SMatthew G. Knepley   PetscFunctionBegin;
17541555c271SMatthew G. Knepley   ierr = VecGetDM(locC, &dmC);CHKERRQ(ierr);
17551555c271SMatthew G. Knepley   ierr = VecSet(locC, 0.0);CHKERRQ(ierr);
17561555c271SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
17571555c271SMatthew G. Knepley   ierr = DMGetCoordinateDim(dm, &coordDim);CHKERRQ(ierr);
17584bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
175992fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
17601555c271SMatthew G. Knepley   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
17611555c271SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
17621555c271SMatthew G. Knepley     PetscObject  obj;
17631555c271SMatthew G. Knepley     PetscClassId id;
17641555c271SMatthew G. Knepley     PetscInt     Nc;
17651555c271SMatthew G. Knepley 
176644a7f3ddSMatthew G. Knepley     ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
17671555c271SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
17681555c271SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
17691555c271SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
17701555c271SMatthew G. Knepley 
17711555c271SMatthew G. Knepley       ierr = PetscFEGetQuadrature(fe, &quad);CHKERRQ(ierr);
17721555c271SMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
17731555c271SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
17741555c271SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
17751555c271SMatthew G. Knepley 
17761555c271SMatthew G. Knepley       ierr = PetscFVGetQuadrature(fv, &quad);CHKERRQ(ierr);
17771555c271SMatthew G. Knepley       ierr = PetscFVGetNumComponents(fv, &Nc);CHKERRQ(ierr);
1778ff1e0c32SBarry Smith     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
17791555c271SMatthew G. Knepley     numComponents += Nc;
17801555c271SMatthew G. Knepley   }
17811555c271SMatthew G. Knepley   ierr = PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);CHKERRQ(ierr);
17821555c271SMatthew G. Knepley   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
17834bee2e38SMatthew G. Knepley   ierr = PetscMalloc6(coordDim*numComponents*2,&gradsum,coordDim*numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ);CHKERRQ(ierr);
17841555c271SMatthew G. Knepley   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1785412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
17861555c271SMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
17871555c271SMatthew G. Knepley     PetscScalar volsum = 0.0;
17881555c271SMatthew G. Knepley     PetscInt   *star = NULL;
17891555c271SMatthew G. Knepley     PetscInt    starSize, st, d, fc;
17901555c271SMatthew G. Knepley 
1791580bdb30SBarry Smith     ierr = PetscArrayzero(gradsum, coordDim*numComponents);CHKERRQ(ierr);
17921555c271SMatthew G. Knepley     ierr = DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
17931555c271SMatthew G. Knepley     for (st = 0; st < starSize*2; st += 2) {
17941555c271SMatthew G. Knepley       const PetscInt cell = star[st];
17951555c271SMatthew G. Knepley       PetscScalar   *grad = &gradsum[coordDim*numComponents];
17961555c271SMatthew G. Knepley       PetscScalar   *x    = NULL;
17971555c271SMatthew G. Knepley       PetscReal      vol  = 0.0;
17981555c271SMatthew G. Knepley 
17991555c271SMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
18004bee2e38SMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ);CHKERRQ(ierr);
18011555c271SMatthew G. Knepley       ierr = DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x);CHKERRQ(ierr);
18021555c271SMatthew G. Knepley       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
18031555c271SMatthew G. Knepley         PetscObject  obj;
18041555c271SMatthew G. Knepley         PetscClassId id;
18051555c271SMatthew G. Knepley         PetscInt     Nb, Nc, q, qc = 0;
18061555c271SMatthew G. Knepley 
1807580bdb30SBarry Smith         ierr = PetscArrayzero(grad, coordDim*numComponents);CHKERRQ(ierr);
180844a7f3ddSMatthew G. Knepley         ierr = DMGetField(dm, field, NULL, &obj);CHKERRQ(ierr);
18091555c271SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
18101555c271SMatthew G. Knepley         if (id == PETSCFE_CLASSID)      {ierr = PetscFEGetNumComponents((PetscFE) obj, &Nc);CHKERRQ(ierr);ierr = PetscFEGetDimension((PetscFE) obj, &Nb);CHKERRQ(ierr);}
18111555c271SMatthew G. Knepley         else if (id == PETSCFV_CLASSID) {ierr = PetscFVGetNumComponents((PetscFV) obj, &Nc);CHKERRQ(ierr);Nb = 1;}
1812ff1e0c32SBarry Smith         else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
18131555c271SMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
18142a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
18152a4e142eSMatthew G. Knepley 
18162a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
18172a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*coordDim*coordDim];
18182a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
18192a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
18204bee2e38SMatthew G. Knepley           if (fegeom.detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature points %D", (double)fegeom.detJ[q], cell, q);
18211555c271SMatthew G. Knepley           if (ierr) {
18221555c271SMatthew G. Knepley             PetscErrorCode ierr2;
18231555c271SMatthew G. Knepley             ierr2 = DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x);CHKERRQ(ierr2);
18241555c271SMatthew G. Knepley             ierr2 = DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr2);
18254bee2e38SMatthew G. Knepley             ierr2 = PetscFree6(gradsum,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr2);
18261555c271SMatthew G. Knepley             CHKERRQ(ierr);
18271555c271SMatthew G. Knepley           }
1828f9244615SMatthew G. Knepley           if (id == PETSCFE_CLASSID)      {ierr = PetscFEInterpolateGradient_Static((PetscFE) obj, 1, &x[fieldOffset], &qgeom, q, interpolant);CHKERRQ(ierr);}
1829ff1e0c32SBarry Smith           else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", field);
18301555c271SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
18311555c271SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+qc+fc];
18321555c271SMatthew G. Knepley 
18334bee2e38SMatthew G. Knepley             for (d = 0; d < coordDim; ++d) grad[fc*coordDim+d] += interpolant[fc*dim+d]*wt*fegeom.detJ[q];
18341555c271SMatthew G. Knepley           }
18354bee2e38SMatthew G. Knepley           vol += quadWeights[q*qNc]*fegeom.detJ[q];
18361555c271SMatthew G. Knepley         }
18371555c271SMatthew G. Knepley         fieldOffset += Nb;
18381555c271SMatthew G. Knepley         qc          += Nc;
18391555c271SMatthew G. Knepley       }
18401555c271SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x);CHKERRQ(ierr);
1841f8527842SMatthew G. Knepley       for (fc = 0; fc < numComponents; ++fc) {
1842f8527842SMatthew G. Knepley         for (d = 0; d < coordDim; ++d) {
1843f8527842SMatthew G. Knepley           gradsum[fc*coordDim+d] += grad[fc*coordDim+d];
1844f8527842SMatthew G. Knepley         }
1845f8527842SMatthew G. Knepley       }
1846f8527842SMatthew G. Knepley       volsum += vol;
1847db1066baSMatthew G. Knepley       if (debug) {
18486d20ae03SJed Brown         ierr = PetscPrintf(PETSC_COMM_SELF, "Cell %D gradient: [", cell);CHKERRQ(ierr);
18491555c271SMatthew G. Knepley         for (fc = 0; fc < numComponents; ++fc) {
18501555c271SMatthew G. Knepley           for (d = 0; d < coordDim; ++d) {
18511555c271SMatthew G. Knepley             if (fc || d > 0) {ierr = PetscPrintf(PETSC_COMM_SELF, ", ");CHKERRQ(ierr);}
18526d20ae03SJed Brown             ierr = PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc*coordDim+d]));CHKERRQ(ierr);
18531555c271SMatthew G. Knepley           }
18541555c271SMatthew G. Knepley         }
18551555c271SMatthew G. Knepley         ierr = PetscPrintf(PETSC_COMM_SELF, "]\n");CHKERRQ(ierr);
1856db1066baSMatthew G. Knepley       }
18571555c271SMatthew G. Knepley     }
18581555c271SMatthew G. Knepley     for (fc = 0; fc < numComponents; ++fc) {
18591555c271SMatthew G. Knepley       for (d = 0; d < coordDim; ++d) gradsum[fc*coordDim+d] /= volsum;
18601555c271SMatthew G. Knepley     }
18611555c271SMatthew G. Knepley     ierr = DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
18621555c271SMatthew G. Knepley     ierr = DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES);CHKERRQ(ierr);
18631555c271SMatthew G. Knepley   }
18644bee2e38SMatthew G. Knepley   ierr = PetscFree6(gradsum,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ);CHKERRQ(ierr);
18651555c271SMatthew G. Knepley   PetscFunctionReturn(0);
18661555c271SMatthew G. Knepley }
18671555c271SMatthew G. Knepley 
1868338f77d5SMatthew G. Knepley static PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec X, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user)
186973d901b8SMatthew G. Knepley {
1870338f77d5SMatthew G. Knepley   DM                 dmAux = NULL;
187161aaff12SToby Isaac   PetscDS            prob,    probAux = NULL;
187273d901b8SMatthew G. Knepley   PetscSection       section, sectionAux;
1873338f77d5SMatthew G. Knepley   Vec                locX,    locA;
1874c330f8ffSToby Isaac   PetscInt           dim, numCells = cEnd - cStart, c, f;
1875c330f8ffSToby Isaac   PetscBool          useFVM = PETSC_FALSE;
1876338f77d5SMatthew G. Knepley   /* DS */
1877338f77d5SMatthew G. Knepley   PetscInt           Nf,    totDim,    *uOff, *uOff_x, numConstants;
1878338f77d5SMatthew G. Knepley   PetscInt           NfAux, totDimAux, *aOff;
1879338f77d5SMatthew G. Knepley   PetscScalar       *u, *a;
1880338f77d5SMatthew G. Knepley   const PetscScalar *constants;
1881338f77d5SMatthew G. Knepley   /* Geometry */
1882c330f8ffSToby Isaac   PetscFEGeom       *cgeomFEM;
1883338f77d5SMatthew G. Knepley   DM                 dmGrad;
1884c330f8ffSToby Isaac   PetscQuadrature    affineQuad = NULL;
1885338f77d5SMatthew G. Knepley   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
1886b5a3613cSMatthew G. Knepley   PetscFVCellGeom   *cgeomFVM;
1887338f77d5SMatthew G. Knepley   const PetscScalar *lgrad;
1888b7260050SToby Isaac   PetscInt           maxDegree;
1889c330f8ffSToby Isaac   DMField            coordField;
1890c330f8ffSToby Isaac   IS                 cellIS;
189173d901b8SMatthew G. Knepley   PetscErrorCode     ierr;
189273d901b8SMatthew G. Knepley 
189373d901b8SMatthew G. Knepley   PetscFunctionBegin;
1894338f77d5SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
1895c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
189692fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
189773d901b8SMatthew G. Knepley   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
1898338f77d5SMatthew G. Knepley   /* Determine which discretizations we have */
1899b5a3613cSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
1900b5a3613cSMatthew G. Knepley     PetscObject  obj;
1901b5a3613cSMatthew G. Knepley     PetscClassId id;
1902b5a3613cSMatthew G. Knepley 
1903b5a3613cSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
1904b5a3613cSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
1905338f77d5SMatthew G. Knepley     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
1906338f77d5SMatthew G. Knepley   }
1907338f77d5SMatthew G. Knepley   /* Get local solution with boundary values */
1908338f77d5SMatthew G. Knepley   ierr = DMGetLocalVector(dm, &locX);CHKERRQ(ierr);
1909338f77d5SMatthew G. Knepley   ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL);CHKERRQ(ierr);
1910338f77d5SMatthew G. Knepley   ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
1911338f77d5SMatthew G. Knepley   ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
1912338f77d5SMatthew G. Knepley   /* Read DS information */
1913338f77d5SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
1914338f77d5SMatthew G. Knepley   ierr = PetscDSGetComponentOffsets(prob, &uOff);CHKERRQ(ierr);
1915338f77d5SMatthew G. Knepley   ierr = PetscDSGetComponentDerivativeOffsets(prob, &uOff_x);CHKERRQ(ierr);
1916c330f8ffSToby Isaac   ierr = ISCreateStride(PETSC_COMM_SELF,numCells,cStart,1,&cellIS);CHKERRQ(ierr);
1917338f77d5SMatthew G. Knepley   ierr = PetscDSGetConstants(prob, &numConstants, &constants);CHKERRQ(ierr);
1918338f77d5SMatthew G. Knepley   /* Read Auxiliary DS information */
19199a2a23afSMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, NULL, 0, &locA);CHKERRQ(ierr);
19209a2a23afSMatthew G. Knepley   if (locA) {
19219a2a23afSMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
1922338f77d5SMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
1923338f77d5SMatthew G. Knepley     ierr = PetscDSGetNumFields(probAux, &NfAux);CHKERRQ(ierr);
192492fd8e1eSJed Brown     ierr = DMGetLocalSection(dmAux, &sectionAux);CHKERRQ(ierr);
1925338f77d5SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
1926338f77d5SMatthew G. Knepley     ierr = PetscDSGetComponentOffsets(probAux, &aOff);CHKERRQ(ierr);
1927338f77d5SMatthew G. Knepley   }
1928338f77d5SMatthew G. Knepley   /* Allocate data  arrays */
1929338f77d5SMatthew G. Knepley   ierr = PetscCalloc1(numCells*totDim, &u);CHKERRQ(ierr);
1930338f77d5SMatthew G. Knepley   if (dmAux) {ierr = PetscMalloc1(numCells*totDimAux, &a);CHKERRQ(ierr);}
1931338f77d5SMatthew G. Knepley   /* Read out geometry */
1932c330f8ffSToby Isaac   ierr = DMGetCoordinateField(dm,&coordField);CHKERRQ(ierr);
1933b7260050SToby Isaac   ierr = DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);CHKERRQ(ierr);
1934b7260050SToby Isaac   if (maxDegree <= 1) {
1935c330f8ffSToby Isaac     ierr = DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad);CHKERRQ(ierr);
1936c330f8ffSToby Isaac     if (affineQuad) {
1937c330f8ffSToby Isaac       ierr = DMFieldCreateFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&cgeomFEM);CHKERRQ(ierr);
1938338f77d5SMatthew G. Knepley     }
1939b5a3613cSMatthew G. Knepley   }
1940b5a3613cSMatthew G. Knepley   if (useFVM) {
1941338f77d5SMatthew G. Knepley     PetscFV   fv = NULL;
1942b5a3613cSMatthew G. Knepley     Vec       grad;
1943b5a3613cSMatthew G. Knepley     PetscInt  fStart, fEnd;
1944b5a3613cSMatthew G. Knepley     PetscBool compGrad;
1945b5a3613cSMatthew G. Knepley 
1946338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
1947338f77d5SMatthew G. Knepley       PetscObject  obj;
1948338f77d5SMatthew G. Knepley       PetscClassId id;
1949338f77d5SMatthew G. Knepley 
1950338f77d5SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
1951338f77d5SMatthew G. Knepley       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
1952338f77d5SMatthew G. Knepley       if (id == PETSCFV_CLASSID) {fv = (PetscFV) obj; break;}
1953338f77d5SMatthew G. Knepley     }
1954338f77d5SMatthew G. Knepley     ierr = PetscFVGetComputeGradients(fv, &compGrad);CHKERRQ(ierr);
1955338f77d5SMatthew G. Knepley     ierr = PetscFVSetComputeGradients(fv, PETSC_TRUE);CHKERRQ(ierr);
1956b5a3613cSMatthew G. Knepley     ierr = DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM);CHKERRQ(ierr);
1957338f77d5SMatthew G. Knepley     ierr = DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad);CHKERRQ(ierr);
1958338f77d5SMatthew G. Knepley     ierr = PetscFVSetComputeGradients(fv, compGrad);CHKERRQ(ierr);
1959b5a3613cSMatthew G. Knepley     ierr = VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM);CHKERRQ(ierr);
1960b5a3613cSMatthew G. Knepley     /* Reconstruct and limit cell gradients */
1961b5a3613cSMatthew G. Knepley     ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
1962b5a3613cSMatthew G. Knepley     ierr = DMGetGlobalVector(dmGrad, &grad);CHKERRQ(ierr);
1963338f77d5SMatthew G. Knepley     ierr = DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad);CHKERRQ(ierr);
1964b5a3613cSMatthew G. Knepley     /* Communicate gradient values */
1965b5a3613cSMatthew G. Knepley     ierr = DMGetLocalVector(dmGrad, &locGrad);CHKERRQ(ierr);
1966b5a3613cSMatthew G. Knepley     ierr = DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
1967b5a3613cSMatthew G. Knepley     ierr = DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
1968b5a3613cSMatthew G. Knepley     ierr = DMRestoreGlobalVector(dmGrad, &grad);CHKERRQ(ierr);
1969b5a3613cSMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
1970338f77d5SMatthew G. Knepley     ierr = DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad);CHKERRQ(ierr);
1971b5a3613cSMatthew G. Knepley     ierr = VecGetArrayRead(locGrad, &lgrad);CHKERRQ(ierr);
1972b5a3613cSMatthew G. Knepley   }
1973338f77d5SMatthew G. Knepley   /* Read out data from inputs */
197473d901b8SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
197573d901b8SMatthew G. Knepley     PetscScalar *x = NULL;
197673d901b8SMatthew G. Knepley     PetscInt     i;
197773d901b8SMatthew G. Knepley 
1978338f77d5SMatthew G. Knepley     ierr = DMPlexVecGetClosure(dm, section, locX, c, NULL, &x);CHKERRQ(ierr);
19790f2d7e86SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
1980338f77d5SMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x);CHKERRQ(ierr);
198173d901b8SMatthew G. Knepley     if (dmAux) {
1982338f77d5SMatthew G. Knepley       ierr = DMPlexVecGetClosure(dmAux, sectionAux, locA, c, NULL, &x);CHKERRQ(ierr);
19830f2d7e86SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
1984338f77d5SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(dmAux, sectionAux, locA, c, NULL, &x);CHKERRQ(ierr);
198573d901b8SMatthew G. Knepley     }
198673d901b8SMatthew G. Knepley   }
1987338f77d5SMatthew G. Knepley   /* Do integration for each field */
198873d901b8SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
1989c1f031eeSMatthew G. Knepley     PetscObject  obj;
1990c1f031eeSMatthew G. Knepley     PetscClassId id;
1991c1f031eeSMatthew G. Knepley     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
199273d901b8SMatthew G. Knepley 
1993c1f031eeSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
1994c1f031eeSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
1995c1f031eeSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1996c1f031eeSMatthew G. Knepley       PetscFE         fe = (PetscFE) obj;
1997c1f031eeSMatthew G. Knepley       PetscQuadrature q;
1998c330f8ffSToby Isaac       PetscFEGeom     *chunkGeom = NULL;
1999c1f031eeSMatthew G. Knepley       PetscInt        Nq, Nb;
2000c1f031eeSMatthew G. Knepley 
20010f2d7e86SMatthew G. Knepley       ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
2002c1f031eeSMatthew G. Knepley       ierr = PetscFEGetQuadrature(fe, &q);CHKERRQ(ierr);
20039c3cf19fSMatthew G. Knepley       ierr = PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
2004c1f031eeSMatthew G. Knepley       ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
2005c1f031eeSMatthew G. Knepley       blockSize = Nb*Nq;
200673d901b8SMatthew G. Knepley       batchSize = numBlocks * blockSize;
20070f2d7e86SMatthew G. Knepley       ierr = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
200873d901b8SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
200973d901b8SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
201073d901b8SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
201173d901b8SMatthew G. Knepley       offset    = numCells - Nr;
2012c330f8ffSToby Isaac       if (!affineQuad) {
2013c330f8ffSToby Isaac         ierr = DMFieldCreateFEGeom(coordField,cellIS,q,PETSC_FALSE,&cgeomFEM);CHKERRQ(ierr);
2014c330f8ffSToby Isaac       }
2015c330f8ffSToby Isaac       ierr = PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);CHKERRQ(ierr);
20164bee2e38SMatthew G. Knepley       ierr = PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral);CHKERRQ(ierr);
2017c330f8ffSToby Isaac       ierr = PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&chunkGeom);CHKERRQ(ierr);
20184bee2e38SMatthew G. Knepley       ierr = PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset*totDim], probAux, &a[offset*totDimAux], &cintegral[offset*Nf]);CHKERRQ(ierr);
2019c330f8ffSToby Isaac       ierr = PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&chunkGeom);CHKERRQ(ierr);
2020c330f8ffSToby Isaac       if (!affineQuad) {
2021c330f8ffSToby Isaac         ierr = PetscFEGeomDestroy(&cgeomFEM);CHKERRQ(ierr);
2022c330f8ffSToby Isaac       }
2023c1f031eeSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
2024c1f031eeSMatthew G. Knepley       PetscInt       foff;
2025420e96edSMatthew G. Knepley       PetscPointFunc obj_func;
2026b69edc29SMatthew G. Knepley       PetscScalar    lint;
2027c1f031eeSMatthew G. Knepley 
2028c1f031eeSMatthew G. Knepley       ierr = PetscDSGetObjective(prob, f, &obj_func);CHKERRQ(ierr);
2029c1f031eeSMatthew G. Knepley       ierr = PetscDSGetFieldOffset(prob, f, &foff);CHKERRQ(ierr);
2030c1f031eeSMatthew G. Knepley       if (obj_func) {
2031c1f031eeSMatthew G. Knepley         for (c = 0; c < numCells; ++c) {
2032b5a3613cSMatthew G. Knepley           PetscScalar *u_x;
2033b5a3613cSMatthew G. Knepley 
2034b5a3613cSMatthew G. Knepley           ierr = DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x);CHKERRQ(ierr);
2035338f77d5SMatthew G. Knepley           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim*c+foff], NULL, u_x, aOff, NULL, &a[totDimAux*c], NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
2036338f77d5SMatthew G. Knepley           cintegral[c*Nf+f] += PetscRealPart(lint)*cgeomFVM[c].volume;
203773d901b8SMatthew G. Knepley         }
2038c1f031eeSMatthew G. Knepley       }
2039ff1e0c32SBarry Smith     } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", f);
2040c1f031eeSMatthew G. Knepley   }
2041338f77d5SMatthew G. Knepley   /* Cleanup data arrays */
2042b5a3613cSMatthew G. Knepley   if (useFVM) {
2043b5a3613cSMatthew G. Knepley     ierr = VecRestoreArrayRead(locGrad, &lgrad);CHKERRQ(ierr);
2044b5a3613cSMatthew G. Knepley     ierr = VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM);CHKERRQ(ierr);
2045b5a3613cSMatthew G. Knepley     ierr = DMRestoreLocalVector(dmGrad, &locGrad);CHKERRQ(ierr);
2046b5a3613cSMatthew G. Knepley     ierr = VecDestroy(&faceGeometryFVM);CHKERRQ(ierr);
2047b5a3613cSMatthew G. Knepley     ierr = VecDestroy(&cellGeometryFVM);CHKERRQ(ierr);
2048b5a3613cSMatthew G. Knepley     ierr = DMDestroy(&dmGrad);CHKERRQ(ierr);
2049b5a3613cSMatthew G. Knepley   }
205073d901b8SMatthew G. Knepley   if (dmAux) {ierr = PetscFree(a);CHKERRQ(ierr);}
2051338f77d5SMatthew G. Knepley   ierr = PetscFree(u);CHKERRQ(ierr);
2052338f77d5SMatthew G. Knepley   /* Cleanup */
2053f99c8401SToby Isaac   if (affineQuad) {
2054f99c8401SToby Isaac     ierr = PetscFEGeomDestroy(&cgeomFEM);CHKERRQ(ierr);
2055f99c8401SToby Isaac   }
2056f99c8401SToby Isaac   ierr = PetscQuadratureDestroy(&affineQuad);CHKERRQ(ierr);
2057c330f8ffSToby Isaac   ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
2058338f77d5SMatthew G. Knepley   ierr = DMRestoreLocalVector(dm, &locX);CHKERRQ(ierr);
2059338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2060338f77d5SMatthew G. Knepley }
2061338f77d5SMatthew G. Knepley 
2062338f77d5SMatthew G. Knepley /*@
2063338f77d5SMatthew G. Knepley   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user
2064338f77d5SMatthew G. Knepley 
2065338f77d5SMatthew G. Knepley   Input Parameters:
2066338f77d5SMatthew G. Knepley + dm - The mesh
2067338f77d5SMatthew G. Knepley . X  - Global input vector
2068338f77d5SMatthew G. Knepley - user - The user context
2069338f77d5SMatthew G. Knepley 
2070338f77d5SMatthew G. Knepley   Output Parameter:
2071338f77d5SMatthew G. Knepley . integral - Integral for each field
2072338f77d5SMatthew G. Knepley 
2073338f77d5SMatthew G. Knepley   Level: developer
2074338f77d5SMatthew G. Knepley 
2075446c23c1SPierre Jolivet .seealso: DMPlexSNESComputeResidualFEM()
2076338f77d5SMatthew G. Knepley @*/
2077b8feb594SMatthew G. Knepley PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user)
2078338f77d5SMatthew G. Knepley {
2079338f77d5SMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dm->data;
2080b8feb594SMatthew G. Knepley   PetscScalar   *cintegral, *lintegral;
2081412e9a14SMatthew G. Knepley   PetscInt       Nf, f, cellHeight, cStart, cEnd, cell;
2082338f77d5SMatthew G. Knepley   PetscErrorCode ierr;
2083338f77d5SMatthew G. Knepley 
2084338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2085338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2086338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2087338f77d5SMatthew G. Knepley   PetscValidPointer(integral, 3);
2088338f77d5SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0);CHKERRQ(ierr);
2089338f77d5SMatthew G. Knepley   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
2090338f77d5SMatthew G. Knepley   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
2091412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
2092338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
2093338f77d5SMatthew G. Knepley   ierr = PetscCalloc2(Nf, &lintegral, (cEnd-cStart)*Nf, &cintegral);CHKERRQ(ierr);
2094338f77d5SMatthew G. Knepley   ierr = DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user);CHKERRQ(ierr);
2095338f77d5SMatthew G. Knepley   /* Sum up values */
2096338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2097338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2098338f77d5SMatthew G. Knepley 
2099338f77d5SMatthew G. Knepley     if (mesh->printFEM > 1) {ierr = DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]);CHKERRQ(ierr);}
2100b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c*Nf+f];
2101338f77d5SMatthew G. Knepley   }
2102820f2d46SBarry Smith   ierr = MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject) dm));CHKERRMPI(ierr);
210373d901b8SMatthew G. Knepley   if (mesh->printFEM) {
2104338f77d5SMatthew G. Knepley     ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "Integral:");CHKERRQ(ierr);
2105b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), " %g", (double) PetscRealPart(integral[f]));CHKERRQ(ierr);}
210673d901b8SMatthew G. Knepley     ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "\n");CHKERRQ(ierr);
210773d901b8SMatthew G. Knepley   }
2108338f77d5SMatthew G. Knepley   ierr = PetscFree2(lintegral, cintegral);CHKERRQ(ierr);
2109338f77d5SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0);CHKERRQ(ierr);
2110338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2111338f77d5SMatthew G. Knepley }
2112338f77d5SMatthew G. Knepley 
2113338f77d5SMatthew G. Knepley /*@
2114338f77d5SMatthew G. Knepley   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user
2115338f77d5SMatthew G. Knepley 
2116338f77d5SMatthew G. Knepley   Input Parameters:
2117338f77d5SMatthew G. Knepley + dm - The mesh
2118338f77d5SMatthew G. Knepley . X  - Global input vector
2119338f77d5SMatthew G. Knepley - user - The user context
2120338f77d5SMatthew G. Knepley 
2121338f77d5SMatthew G. Knepley   Output Parameter:
2122338f77d5SMatthew G. Knepley . integral - Cellwise integrals for each field
2123338f77d5SMatthew G. Knepley 
2124338f77d5SMatthew G. Knepley   Level: developer
2125338f77d5SMatthew G. Knepley 
2126446c23c1SPierre Jolivet .seealso: DMPlexSNESComputeResidualFEM()
2127338f77d5SMatthew G. Knepley @*/
2128338f77d5SMatthew G. Knepley PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user)
2129338f77d5SMatthew G. Knepley {
2130338f77d5SMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dm->data;
2131338f77d5SMatthew G. Knepley   DM             dmF;
2132338f77d5SMatthew G. Knepley   PetscSection   sectionF;
2133338f77d5SMatthew G. Knepley   PetscScalar   *cintegral, *af;
2134412e9a14SMatthew G. Knepley   PetscInt       Nf, f, cellHeight, cStart, cEnd, cell;
2135338f77d5SMatthew G. Knepley   PetscErrorCode ierr;
2136338f77d5SMatthew G. Knepley 
2137338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2138338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2139338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2140338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(F, VEC_CLASSID, 3);
2141338f77d5SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0);CHKERRQ(ierr);
2142338f77d5SMatthew G. Knepley   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
2143338f77d5SMatthew G. Knepley   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
2144412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
2145338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
2146338f77d5SMatthew G. Knepley   ierr = PetscCalloc1((cEnd-cStart)*Nf, &cintegral);CHKERRQ(ierr);
2147338f77d5SMatthew G. Knepley   ierr = DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user);CHKERRQ(ierr);
2148338f77d5SMatthew G. Knepley   /* Put values in F*/
2149338f77d5SMatthew G. Knepley   ierr = VecGetDM(F, &dmF);CHKERRQ(ierr);
215092fd8e1eSJed Brown   ierr = DMGetLocalSection(dmF, &sectionF);CHKERRQ(ierr);
2151338f77d5SMatthew G. Knepley   ierr = VecGetArray(F, &af);CHKERRQ(ierr);
2152338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2153338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2154338f77d5SMatthew G. Knepley     PetscInt       dof, off;
2155338f77d5SMatthew G. Knepley 
2156338f77d5SMatthew G. Knepley     if (mesh->printFEM > 1) {ierr = DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]);CHKERRQ(ierr);}
2157338f77d5SMatthew G. Knepley     ierr = PetscSectionGetDof(sectionF, cell, &dof);CHKERRQ(ierr);
2158338f77d5SMatthew G. Knepley     ierr = PetscSectionGetOffset(sectionF, cell, &off);CHKERRQ(ierr);
2159338f77d5SMatthew G. Knepley     if (dof != Nf) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %D != %D", dof, Nf);
2160338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) af[off+f] = cintegral[c*Nf+f];
2161338f77d5SMatthew G. Knepley   }
2162338f77d5SMatthew G. Knepley   ierr = VecRestoreArray(F, &af);CHKERRQ(ierr);
2163338f77d5SMatthew G. Knepley   ierr = PetscFree(cintegral);CHKERRQ(ierr);
2164c1f031eeSMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0);CHKERRQ(ierr);
216573d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
216673d901b8SMatthew G. Knepley }
216773d901b8SMatthew G. Knepley 
21689b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS,
21699b6f715bSMatthew G. Knepley                                                        void (*func)(PetscInt, PetscInt, PetscInt,
217064c72086SMatthew G. Knepley                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
217164c72086SMatthew G. Knepley                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
217264c72086SMatthew G. Knepley                                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
217364c72086SMatthew G. Knepley                                                        PetscScalar *fintegral, void *user)
217464c72086SMatthew G. Knepley {
21759b6f715bSMatthew G. Knepley   DM                 plex = NULL, plexA = NULL;
2176a6e0b375SMatthew G. Knepley   DMEnclosureType    encAux;
21779b6f715bSMatthew G. Knepley   PetscDS            prob, probAux = NULL;
21789b6f715bSMatthew G. Knepley   PetscSection       section, sectionAux = NULL;
21799b6f715bSMatthew G. Knepley   Vec                locA = NULL;
21809b6f715bSMatthew G. Knepley   DMField            coordField;
21819b6f715bSMatthew G. Knepley   PetscInt           Nf,        totDim,        *uOff, *uOff_x;
21829b6f715bSMatthew G. Knepley   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
21839b6f715bSMatthew G. Knepley   PetscScalar       *u, *a = NULL;
218464c72086SMatthew G. Knepley   const PetscScalar *constants;
21859b6f715bSMatthew G. Knepley   PetscInt           numConstants, f;
218664c72086SMatthew G. Knepley   PetscErrorCode     ierr;
218764c72086SMatthew G. Knepley 
218864c72086SMatthew G. Knepley   PetscFunctionBegin;
21899b6f715bSMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
21909b6f715bSMatthew G. Knepley   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
219164c72086SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
219292fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
219364c72086SMatthew G. Knepley   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
219464c72086SMatthew G. Knepley   /* Determine which discretizations we have */
21959b6f715bSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
219664c72086SMatthew G. Knepley     PetscObject  obj;
219764c72086SMatthew G. Knepley     PetscClassId id;
219864c72086SMatthew G. Knepley 
21999b6f715bSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
220064c72086SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
22019b6f715bSMatthew G. Knepley     if (id == PETSCFV_CLASSID) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Not supported for FVM (field %D)", f);
220264c72086SMatthew G. Knepley   }
220364c72086SMatthew G. Knepley   /* Read DS information */
220464c72086SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
220564c72086SMatthew G. Knepley   ierr = PetscDSGetComponentOffsets(prob, &uOff);CHKERRQ(ierr);
220664c72086SMatthew G. Knepley   ierr = PetscDSGetComponentDerivativeOffsets(prob, &uOff_x);CHKERRQ(ierr);
220764c72086SMatthew G. Knepley   ierr = PetscDSGetConstants(prob, &numConstants, &constants);CHKERRQ(ierr);
220864c72086SMatthew G. Knepley   /* Read Auxiliary DS information */
22099a2a23afSMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, NULL, 0, &locA);CHKERRQ(ierr);
22109b6f715bSMatthew G. Knepley   if (locA) {
22119b6f715bSMatthew G. Knepley     DM dmAux;
22129b6f715bSMatthew G. Knepley 
22139b6f715bSMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
2214a6e0b375SMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux, dm, &encAux);CHKERRQ(ierr);
22159b6f715bSMatthew G. Knepley     ierr = DMConvert(dmAux, DMPLEX, &plexA);CHKERRQ(ierr);
221664c72086SMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
221764c72086SMatthew G. Knepley     ierr = PetscDSGetNumFields(probAux, &NfAux);CHKERRQ(ierr);
221892fd8e1eSJed Brown     ierr = DMGetLocalSection(dmAux, &sectionAux);CHKERRQ(ierr);
221964c72086SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
222064c72086SMatthew G. Knepley     ierr = PetscDSGetComponentOffsets(probAux, &aOff);CHKERRQ(ierr);
222164c72086SMatthew G. Knepley   }
22229b6f715bSMatthew G. Knepley   /* Integrate over points */
22239b6f715bSMatthew G. Knepley   {
22249b6f715bSMatthew G. Knepley     PetscFEGeom    *fgeom, *chunkGeom = NULL;
2225b7260050SToby Isaac     PetscInt        maxDegree;
22269b6f715bSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
22279b6f715bSMatthew G. Knepley     const PetscInt *points;
22289b6f715bSMatthew G. Knepley     PetscInt        numFaces, face, Nq, field;
22299b6f715bSMatthew G. Knepley     PetscInt        numChunks, chunkSize, chunk, Nr, offset;
223064c72086SMatthew G. Knepley 
22319b6f715bSMatthew G. Knepley     ierr = ISGetLocalSize(pointIS, &numFaces);CHKERRQ(ierr);
22329b6f715bSMatthew G. Knepley     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
22339b6f715bSMatthew G. Knepley     ierr = PetscCalloc2(numFaces*totDim, &u, locA ? numFaces*totDimAux : 0, &a);CHKERRQ(ierr);
2234b7260050SToby Isaac     ierr = DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree);CHKERRQ(ierr);
223564c72086SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
223664c72086SMatthew G. Knepley       PetscFE fe;
223764c72086SMatthew G. Knepley 
223864c72086SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, field, (PetscObject *) &fe);CHKERRQ(ierr);
2239b7260050SToby Isaac       if (maxDegree <= 1) {ierr = DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom);CHKERRQ(ierr);}
22409b6f715bSMatthew G. Knepley       if (!qGeom) {
22419b6f715bSMatthew G. Knepley         ierr = PetscFEGetFaceQuadrature(fe, &qGeom);CHKERRQ(ierr);
22429b6f715bSMatthew G. Knepley         ierr = PetscObjectReference((PetscObject) qGeom);CHKERRQ(ierr);
22439b6f715bSMatthew G. Knepley       }
22449b6f715bSMatthew G. Knepley       ierr = PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
22459b6f715bSMatthew G. Knepley       ierr = DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom);CHKERRQ(ierr);
22469b6f715bSMatthew G. Knepley       for (face = 0; face < numFaces; ++face) {
2247f15274beSMatthew Knepley         const PetscInt point = points[face], *support;
22489b6f715bSMatthew G. Knepley         PetscScalar    *x    = NULL;
2249f15274beSMatthew Knepley         PetscInt       i;
22509b6f715bSMatthew G. Knepley 
22519b6f715bSMatthew G. Knepley         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
22529b6f715bSMatthew G. Knepley         ierr = DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x);CHKERRQ(ierr);
22539b6f715bSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
22549b6f715bSMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x);CHKERRQ(ierr);
22559b6f715bSMatthew G. Knepley         if (locA) {
22569b6f715bSMatthew G. Knepley           PetscInt subp;
2257a6e0b375SMatthew G. Knepley           ierr = DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp);CHKERRQ(ierr);
22589b6f715bSMatthew G. Knepley           ierr = DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x);CHKERRQ(ierr);
22599b6f715bSMatthew G. Knepley           for (i = 0; i < totDimAux; ++i) a[f*totDimAux+i] = x[i];
22609b6f715bSMatthew G. Knepley           ierr = DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x);CHKERRQ(ierr);
22619b6f715bSMatthew G. Knepley         }
22629b6f715bSMatthew G. Knepley       }
22639b6f715bSMatthew G. Knepley       /* Get blocking */
22649b6f715bSMatthew G. Knepley       {
22659b6f715bSMatthew G. Knepley         PetscQuadrature q;
22669b6f715bSMatthew G. Knepley         PetscInt        numBatches, batchSize, numBlocks, blockSize;
22679b6f715bSMatthew G. Knepley         PetscInt        Nq, Nb;
22689b6f715bSMatthew G. Knepley 
226964c72086SMatthew G. Knepley         ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
227064c72086SMatthew G. Knepley         ierr = PetscFEGetQuadrature(fe, &q);CHKERRQ(ierr);
227164c72086SMatthew G. Knepley         ierr = PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
227264c72086SMatthew G. Knepley         ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
227364c72086SMatthew G. Knepley         blockSize = Nb*Nq;
227464c72086SMatthew G. Knepley         batchSize = numBlocks * blockSize;
22759b6f715bSMatthew G. Knepley         chunkSize = numBatches*batchSize;
227664c72086SMatthew G. Knepley         ierr = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
22779b6f715bSMatthew G. Knepley         numChunks = numFaces / chunkSize;
22789b6f715bSMatthew G. Knepley         Nr        = numFaces % chunkSize;
227964c72086SMatthew G. Knepley         offset    = numFaces - Nr;
228064c72086SMatthew G. Knepley       }
22819b6f715bSMatthew G. Knepley       /* Do integration for each field */
22829b6f715bSMatthew G. Knepley       for (chunk = 0; chunk < numChunks; ++chunk) {
22839b6f715bSMatthew G. Knepley         ierr = PetscFEGeomGetChunk(fgeom, chunk*chunkSize, (chunk+1)*chunkSize, &chunkGeom);CHKERRQ(ierr);
22844bee2e38SMatthew G. Knepley         ierr = PetscFEIntegrateBd(prob, field, func, chunkSize, chunkGeom, u, probAux, a, fintegral);CHKERRQ(ierr);
22859b6f715bSMatthew G. Knepley         ierr = PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom);CHKERRQ(ierr);
228664c72086SMatthew G. Knepley       }
22879b6f715bSMatthew G. Knepley       ierr = PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom);CHKERRQ(ierr);
22884bee2e38SMatthew G. Knepley       ierr = PetscFEIntegrateBd(prob, field, func, Nr, chunkGeom, &u[offset*totDim], probAux, a ? &a[offset*totDimAux] : NULL, &fintegral[offset*Nf]);CHKERRQ(ierr);
22899b6f715bSMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom);CHKERRQ(ierr);
229064c72086SMatthew G. Knepley       /* Cleanup data arrays */
22919b6f715bSMatthew G. Knepley       ierr = DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom);CHKERRQ(ierr);
22929b6f715bSMatthew G. Knepley       ierr = PetscQuadratureDestroy(&qGeom);CHKERRQ(ierr);
22939b6f715bSMatthew G. Knepley       ierr = PetscFree2(u, a);CHKERRQ(ierr);
22949b6f715bSMatthew G. Knepley       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
229564c72086SMatthew G. Knepley     }
229664c72086SMatthew G. Knepley   }
22979b6f715bSMatthew G. Knepley   if (plex)  {ierr = DMDestroy(&plex);CHKERRQ(ierr);}
22989b6f715bSMatthew G. Knepley   if (plexA) {ierr = DMDestroy(&plexA);CHKERRQ(ierr);}
22999b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
23009b6f715bSMatthew G. Knepley }
23019b6f715bSMatthew G. Knepley 
23029b6f715bSMatthew G. Knepley /*@
23039b6f715bSMatthew G. Knepley   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user
23049b6f715bSMatthew G. Knepley 
23059b6f715bSMatthew G. Knepley   Input Parameters:
23069b6f715bSMatthew G. Knepley + dm      - The mesh
23079b6f715bSMatthew G. Knepley . X       - Global input vector
23089b6f715bSMatthew G. Knepley . label   - The boundary DMLabel
23099b6f715bSMatthew G. Knepley . numVals - The number of label values to use, or PETSC_DETERMINE for all values
23109b6f715bSMatthew G. Knepley . vals    - The label values to use, or PETSC_NULL for all values
23119b6f715bSMatthew G. Knepley . func    = The function to integrate along the boundary
23129b6f715bSMatthew G. Knepley - user    - The user context
23139b6f715bSMatthew G. Knepley 
23149b6f715bSMatthew G. Knepley   Output Parameter:
23159b6f715bSMatthew G. Knepley . integral - Integral for each field
23169b6f715bSMatthew G. Knepley 
23179b6f715bSMatthew G. Knepley   Level: developer
23189b6f715bSMatthew G. Knepley 
23199b6f715bSMatthew G. Knepley .seealso: DMPlexComputeIntegralFEM(), DMPlexComputeBdResidualFEM()
23209b6f715bSMatthew G. Knepley @*/
23219b6f715bSMatthew G. Knepley PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[],
23229b6f715bSMatthew G. Knepley                                        void (*func)(PetscInt, PetscInt, PetscInt,
23239b6f715bSMatthew G. Knepley                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
23249b6f715bSMatthew G. Knepley                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
23259b6f715bSMatthew G. Knepley                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
23269b6f715bSMatthew G. Knepley                                        PetscScalar *integral, void *user)
23279b6f715bSMatthew G. Knepley {
23289b6f715bSMatthew G. Knepley   Vec            locX;
23299b6f715bSMatthew G. Knepley   PetscSection   section;
23309b6f715bSMatthew G. Knepley   DMLabel        depthLabel;
23319b6f715bSMatthew G. Knepley   IS             facetIS;
23329b6f715bSMatthew G. Knepley   PetscInt       dim, Nf, f, v;
23339b6f715bSMatthew G. Knepley   PetscErrorCode ierr;
23349b6f715bSMatthew G. Knepley 
23359b6f715bSMatthew G. Knepley   PetscFunctionBegin;
23369b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
23379b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
23389b6f715bSMatthew G. Knepley   PetscValidPointer(label, 3);
23399b6f715bSMatthew G. Knepley   if (vals) PetscValidPointer(vals, 5);
2340064a246eSJacob Faibussowitsch   PetscValidPointer(integral, 7);
23419b6f715bSMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0);CHKERRQ(ierr);
23429b6f715bSMatthew G. Knepley   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
23439b6f715bSMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
23449b6f715bSMatthew G. Knepley   ierr = DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);CHKERRQ(ierr);
234592fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
23469b6f715bSMatthew G. Knepley   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
23479b6f715bSMatthew G. Knepley   /* Get local solution with boundary values */
23489b6f715bSMatthew G. Knepley   ierr = DMGetLocalVector(dm, &locX);CHKERRQ(ierr);
23499b6f715bSMatthew G. Knepley   ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL);CHKERRQ(ierr);
23509b6f715bSMatthew G. Knepley   ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
23519b6f715bSMatthew G. Knepley   ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
23529b6f715bSMatthew G. Knepley   /* Loop over label values */
2353580bdb30SBarry Smith   ierr = PetscArrayzero(integral, Nf);CHKERRQ(ierr);
23549b6f715bSMatthew G. Knepley   for (v = 0; v < numVals; ++v) {
23559b6f715bSMatthew G. Knepley     IS           pointIS;
23569b6f715bSMatthew G. Knepley     PetscInt     numFaces, face;
23579b6f715bSMatthew G. Knepley     PetscScalar *fintegral;
23589b6f715bSMatthew G. Knepley 
23599b6f715bSMatthew G. Knepley     ierr = DMLabelGetStratumIS(label, vals[v], &pointIS);CHKERRQ(ierr);
23609b6f715bSMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
23619b6f715bSMatthew G. Knepley     {
23629b6f715bSMatthew G. Knepley       IS isectIS;
23639b6f715bSMatthew G. Knepley 
23649b6f715bSMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
23659b6f715bSMatthew G. Knepley       ierr = ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS);CHKERRQ(ierr);
23669b6f715bSMatthew G. Knepley       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
23679b6f715bSMatthew G. Knepley       pointIS = isectIS;
23689b6f715bSMatthew G. Knepley     }
23699b6f715bSMatthew G. Knepley     ierr = ISGetLocalSize(pointIS, &numFaces);CHKERRQ(ierr);
23709b6f715bSMatthew G. Knepley     ierr = PetscCalloc1(numFaces*Nf, &fintegral);CHKERRQ(ierr);
23719b6f715bSMatthew G. Knepley     ierr = DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, func, fintegral, user);CHKERRQ(ierr);
23729b6f715bSMatthew G. Knepley     /* Sum point contributions into integral */
23739b6f715bSMatthew G. Knepley     for (f = 0; f < Nf; ++f) for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face*Nf+f];
23749b6f715bSMatthew G. Knepley     ierr = PetscFree(fintegral);CHKERRQ(ierr);
23759b6f715bSMatthew G. Knepley     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
23769b6f715bSMatthew G. Knepley   }
237764c72086SMatthew G. Knepley   ierr = DMRestoreLocalVector(dm, &locX);CHKERRQ(ierr);
23789b6f715bSMatthew G. Knepley   ierr = ISDestroy(&facetIS);CHKERRQ(ierr);
23799b6f715bSMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0);CHKERRQ(ierr);
238064c72086SMatthew G. Knepley   PetscFunctionReturn(0);
238164c72086SMatthew G. Knepley }
238264c72086SMatthew G. Knepley 
2383d69c5d34SMatthew G. Knepley /*@
2384cf51de39SMatthew G. Knepley   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix I from the coarse DM to a uniformly refined DM.
2385d69c5d34SMatthew G. Knepley 
2386d69c5d34SMatthew G. Knepley   Input Parameters:
2387cf51de39SMatthew G. Knepley + dmc  - The coarse mesh
2388cf51de39SMatthew G. Knepley . dmf  - The fine mesh
2389cf51de39SMatthew G. Knepley . isRefined - Flag indicating regular refinement, rather than the same topology
2390d69c5d34SMatthew G. Knepley - user - The user context
2391d69c5d34SMatthew G. Knepley 
2392d69c5d34SMatthew G. Knepley   Output Parameter:
2393934789fcSMatthew G. Knepley . In  - The interpolation matrix
2394d69c5d34SMatthew G. Knepley 
2395d69c5d34SMatthew G. Knepley   Level: developer
2396d69c5d34SMatthew G. Knepley 
239768132eb9SMatthew G. Knepley .seealso: DMPlexComputeInterpolatorGeneral(), DMPlexComputeJacobianFEM()
2398d69c5d34SMatthew G. Knepley @*/
2399cf51de39SMatthew G. Knepley PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user)
2400d69c5d34SMatthew G. Knepley {
2401d69c5d34SMatthew G. Knepley   DM_Plex          *mesh  = (DM_Plex *) dmc->data;
2402d69c5d34SMatthew G. Knepley   const char       *name  = "Interpolator";
2403d69c5d34SMatthew G. Knepley   PetscFE          *feRef;
240497c42addSMatthew G. Knepley   PetscFV          *fvRef;
2405d69c5d34SMatthew G. Knepley   PetscSection      fsection, fglobalSection;
2406d69c5d34SMatthew G. Knepley   PetscSection      csection, cglobalSection;
2407d69c5d34SMatthew G. Knepley   PetscScalar      *elemMat;
2408485ad865SMatthew G. Knepley   PetscInt          dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c;
24092ea9c922SToby Isaac   PetscInt          cTotDim=0, rTotDim = 0;
2410d69c5d34SMatthew G. Knepley   PetscErrorCode    ierr;
2411d69c5d34SMatthew G. Knepley 
2412d69c5d34SMatthew G. Knepley   PetscFunctionBegin;
2413d69c5d34SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);CHKERRQ(ierr);
2414c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dmf, &dim);CHKERRQ(ierr);
241592fd8e1eSJed Brown   ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);
2416e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmf, &fglobalSection);CHKERRQ(ierr);
241792fd8e1eSJed Brown   ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);
2418e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmc, &cglobalSection);CHKERRQ(ierr);
2419d69c5d34SMatthew G. Knepley   ierr = PetscSectionGetNumFields(fsection, &Nf);CHKERRQ(ierr);
2420412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd);CHKERRQ(ierr);
242197c42addSMatthew G. Knepley   ierr = PetscCalloc2(Nf, &feRef, Nf, &fvRef);CHKERRQ(ierr);
2422d69c5d34SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
24232ea9c922SToby Isaac     PetscObject  obj, objc;
24242ea9c922SToby Isaac     PetscClassId id, idc;
24252ea9c922SToby Isaac     PetscInt     rNb = 0, Nc = 0, cNb = 0;
2426d69c5d34SMatthew G. Knepley 
24272ea9c922SToby Isaac     ierr = DMGetField(dmf, f, NULL, &obj);CHKERRQ(ierr);
242897c42addSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
242997c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
243097c42addSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
243197c42addSMatthew G. Knepley 
2432cf51de39SMatthew G. Knepley       if (isRefined) {
24330f2d7e86SMatthew G. Knepley         ierr = PetscFERefine(fe, &feRef[f]);CHKERRQ(ierr);
2434cf51de39SMatthew G. Knepley       } else {
2435cf51de39SMatthew G. Knepley         ierr = PetscObjectReference((PetscObject) fe);CHKERRQ(ierr);
2436cf51de39SMatthew G. Knepley         feRef[f] = fe;
2437cf51de39SMatthew G. Knepley       }
2438d69c5d34SMatthew G. Knepley       ierr = PetscFEGetDimension(feRef[f], &rNb);CHKERRQ(ierr);
24390f2d7e86SMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
244097c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
244197c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV) obj;
244297c42addSMatthew G. Knepley       PetscDualSpace Q;
244397c42addSMatthew G. Knepley 
2444cf51de39SMatthew G. Knepley       if (isRefined) {
244597c42addSMatthew G. Knepley         ierr = PetscFVRefine(fv, &fvRef[f]);CHKERRQ(ierr);
2446cf51de39SMatthew G. Knepley       } else {
2447cf51de39SMatthew G. Knepley         ierr = PetscObjectReference((PetscObject) fv);CHKERRQ(ierr);
2448cf51de39SMatthew G. Knepley         fvRef[f] = fv;
2449cf51de39SMatthew G. Knepley       }
245097c42addSMatthew G. Knepley       ierr = PetscFVGetDualSpace(fvRef[f], &Q);CHKERRQ(ierr);
245197c42addSMatthew G. Knepley       ierr = PetscDualSpaceGetDimension(Q, &rNb);CHKERRQ(ierr);
24522ea9c922SToby Isaac       ierr = PetscFVGetDualSpace(fv, &Q);CHKERRQ(ierr);
245397c42addSMatthew G. Knepley       ierr = PetscFVGetNumComponents(fv, &Nc);CHKERRQ(ierr);
245497c42addSMatthew G. Knepley     }
24552ea9c922SToby Isaac     ierr = DMGetField(dmc, f, NULL, &objc);CHKERRQ(ierr);
24562ea9c922SToby Isaac     ierr = PetscObjectGetClassId(objc, &idc);CHKERRQ(ierr);
24572ea9c922SToby Isaac     if (idc == PETSCFE_CLASSID) {
24582ea9c922SToby Isaac       PetscFE fe = (PetscFE) objc;
24592ea9c922SToby Isaac 
24602ea9c922SToby Isaac       ierr = PetscFEGetDimension(fe, &cNb);CHKERRQ(ierr);
24612ea9c922SToby Isaac     } else if (id == PETSCFV_CLASSID) {
24622ea9c922SToby Isaac       PetscFV        fv = (PetscFV) obj;
24632ea9c922SToby Isaac       PetscDualSpace Q;
24642ea9c922SToby Isaac 
24652ea9c922SToby Isaac       ierr = PetscFVGetDualSpace(fv, &Q);CHKERRQ(ierr);
24662ea9c922SToby Isaac       ierr = PetscDualSpaceGetDimension(Q, &cNb);CHKERRQ(ierr);
2467d69c5d34SMatthew G. Knepley     }
24682ea9c922SToby Isaac     rTotDim += rNb;
24692ea9c922SToby Isaac     cTotDim += cNb;
24702ea9c922SToby Isaac   }
24710f2d7e86SMatthew G. Knepley   ierr = PetscMalloc1(rTotDim*cTotDim,&elemMat);CHKERRQ(ierr);
2472580bdb30SBarry Smith   ierr = PetscArrayzero(elemMat, rTotDim*cTotDim);CHKERRQ(ierr);
2473d69c5d34SMatthew G. Knepley   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
2474d69c5d34SMatthew G. Knepley     PetscDualSpace   Qref;
2475d69c5d34SMatthew G. Knepley     PetscQuadrature  f;
2476d69c5d34SMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2477d69c5d34SMatthew G. Knepley     PetscReal       *points;
2478d69c5d34SMatthew G. Knepley     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;
2479d69c5d34SMatthew G. Knepley 
2480d69c5d34SMatthew G. Knepley     /* Compose points from all dual basis functionals */
248197c42addSMatthew G. Knepley     if (feRef[fieldI]) {
2482d69c5d34SMatthew G. Knepley       ierr = PetscFEGetDualSpace(feRef[fieldI], &Qref);CHKERRQ(ierr);
24830f2d7e86SMatthew G. Knepley       ierr = PetscFEGetNumComponents(feRef[fieldI], &Nc);CHKERRQ(ierr);
248497c42addSMatthew G. Knepley     } else {
248597c42addSMatthew G. Knepley       ierr = PetscFVGetDualSpace(fvRef[fieldI], &Qref);CHKERRQ(ierr);
248697c42addSMatthew G. Knepley       ierr = PetscFVGetNumComponents(fvRef[fieldI], &Nc);CHKERRQ(ierr);
248797c42addSMatthew G. Knepley     }
2488d69c5d34SMatthew G. Knepley     ierr = PetscDualSpaceGetDimension(Qref, &fpdim);CHKERRQ(ierr);
2489d69c5d34SMatthew G. Knepley     for (i = 0; i < fpdim; ++i) {
2490d69c5d34SMatthew G. Knepley       ierr = PetscDualSpaceGetFunctional(Qref, i, &f);CHKERRQ(ierr);
24919c3cf19fSMatthew G. Knepley       ierr = PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL);CHKERRQ(ierr);
2492d69c5d34SMatthew G. Knepley       npoints += Np;
2493d69c5d34SMatthew G. Knepley     }
2494d69c5d34SMatthew G. Knepley     ierr = PetscMalloc1(npoints*dim,&points);CHKERRQ(ierr);
2495d69c5d34SMatthew G. Knepley     for (i = 0, k = 0; i < fpdim; ++i) {
2496d69c5d34SMatthew G. Knepley       ierr = PetscDualSpaceGetFunctional(Qref, i, &f);CHKERRQ(ierr);
24979c3cf19fSMatthew G. Knepley       ierr = PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL);CHKERRQ(ierr);
2498d69c5d34SMatthew G. Knepley       for (p = 0; p < Np; ++p, ++k) for (d = 0; d < dim; ++d) points[k*dim+d] = qpoints[p*dim+d];
2499d69c5d34SMatthew G. Knepley     }
2500d69c5d34SMatthew G. Knepley 
2501d69c5d34SMatthew G. Knepley     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
250297c42addSMatthew G. Knepley       PetscObject  obj;
250397c42addSMatthew G. Knepley       PetscClassId id;
25049c3cf19fSMatthew G. Knepley       PetscInt     NcJ = 0, cpdim = 0, j, qNc;
2505d69c5d34SMatthew G. Knepley 
25062ea9c922SToby Isaac       ierr = DMGetField(dmc, fieldJ, NULL, &obj);CHKERRQ(ierr);
250797c42addSMatthew G. Knepley       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
250897c42addSMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
250997c42addSMatthew G. Knepley         PetscFE           fe = (PetscFE) obj;
2510ef0bb6c7SMatthew G. Knepley         PetscTabulation T  = NULL;
2511d69c5d34SMatthew G. Knepley 
2512d69c5d34SMatthew G. Knepley         /* Evaluate basis at points */
25130f2d7e86SMatthew G. Knepley         ierr = PetscFEGetNumComponents(fe, &NcJ);CHKERRQ(ierr);
25140f2d7e86SMatthew G. Knepley         ierr = PetscFEGetDimension(fe, &cpdim);CHKERRQ(ierr);
2515ffe73a53SMatthew G. Knepley         /* For now, fields only interpolate themselves */
2516ffe73a53SMatthew G. Knepley         if (fieldI == fieldJ) {
25179c3cf19fSMatthew G. Knepley           if (Nc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %D does not match coarse field %D", Nc, NcJ);
2518ef0bb6c7SMatthew G. Knepley           ierr = PetscFECreateTabulation(fe, 1, npoints, points, 0, &T);CHKERRQ(ierr);
2519d69c5d34SMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
2520d69c5d34SMatthew G. Knepley             ierr = PetscDualSpaceGetFunctional(Qref, i, &f);CHKERRQ(ierr);
25219c3cf19fSMatthew G. Knepley             ierr = PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights);CHKERRQ(ierr);
25229c3cf19fSMatthew G. Knepley             if (qNc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %D does not match coarse field %D", qNc, NcJ);
2523d69c5d34SMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
252436a6d9c0SMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2525d172c84bSMatthew G. Knepley                 /*
2526d172c84bSMatthew G. Knepley                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
2527d172c84bSMatthew G. Knepley                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
2528d172c84bSMatthew G. Knepley                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
2529d172c84bSMatthew G. Knepley                    qNC, Nc, Ncj, c:    Number of components in this field
2530d172c84bSMatthew G. Knepley                    Np, p:              Number of quad points in the fine grid functional i
2531d172c84bSMatthew G. Knepley                    k:                  i*Np + p, overall point number for the interpolation
2532d172c84bSMatthew G. Knepley                 */
2533ef0bb6c7SMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += T->T[0][k*cpdim*NcJ+j*Nc+c]*qweights[p*qNc+c];
253436a6d9c0SMatthew G. Knepley               }
2535d69c5d34SMatthew G. Knepley             }
2536d69c5d34SMatthew G. Knepley           }
2537ef0bb6c7SMatthew G. Knepley           ierr = PetscTabulationDestroy(&T);CHKERRQ(ierr);CHKERRQ(ierr);
2538ffe73a53SMatthew G. Knepley         }
253997c42addSMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
254097c42addSMatthew G. Knepley         PetscFV        fv = (PetscFV) obj;
254197c42addSMatthew G. Knepley 
254297c42addSMatthew G. Knepley         /* Evaluate constant function at points */
254397c42addSMatthew G. Knepley         ierr = PetscFVGetNumComponents(fv, &NcJ);CHKERRQ(ierr);
254497c42addSMatthew G. Knepley         cpdim = 1;
254597c42addSMatthew G. Knepley         /* For now, fields only interpolate themselves */
254697c42addSMatthew G. Knepley         if (fieldI == fieldJ) {
2547ff1e0c32SBarry Smith           if (Nc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %D does not match coarse field %D", Nc, NcJ);
254897c42addSMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
254997c42addSMatthew G. Knepley             ierr = PetscDualSpaceGetFunctional(Qref, i, &f);CHKERRQ(ierr);
25509c3cf19fSMatthew G. Knepley             ierr = PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights);CHKERRQ(ierr);
25519c3cf19fSMatthew G. Knepley             if (qNc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %D does not match coarse field %D", qNc, NcJ);
255297c42addSMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
255397c42addSMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2554458eb97cSMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += 1.0*qweights[p*qNc+c];
255597c42addSMatthew G. Knepley               }
255697c42addSMatthew G. Knepley             }
255797c42addSMatthew G. Knepley           }
255897c42addSMatthew G. Knepley         }
255997c42addSMatthew G. Knepley       }
2560d172c84bSMatthew G. Knepley       offsetJ += cpdim;
2561d69c5d34SMatthew G. Knepley     }
2562d172c84bSMatthew G. Knepley     offsetI += fpdim;
2563549a8adaSMatthew G. Knepley     ierr = PetscFree(points);CHKERRQ(ierr);
2564d69c5d34SMatthew G. Knepley   }
25650f2d7e86SMatthew G. Knepley   if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat);CHKERRQ(ierr);}
25667f5b169aSMatthew G. Knepley   /* Preallocate matrix */
25677f5b169aSMatthew G. Knepley   {
2568c094ef40SMatthew G. Knepley     Mat          preallocator;
2569c094ef40SMatthew G. Knepley     PetscScalar *vals;
2570c094ef40SMatthew G. Knepley     PetscInt    *cellCIndices, *cellFIndices;
2571c094ef40SMatthew G. Knepley     PetscInt     locRows, locCols, cell;
25727f5b169aSMatthew G. Knepley 
2573c094ef40SMatthew G. Knepley     ierr = MatGetLocalSize(In, &locRows, &locCols);CHKERRQ(ierr);
2574c094ef40SMatthew G. Knepley     ierr = MatCreate(PetscObjectComm((PetscObject) In), &preallocator);CHKERRQ(ierr);
2575c094ef40SMatthew G. Knepley     ierr = MatSetType(preallocator, MATPREALLOCATOR);CHKERRQ(ierr);
2576c094ef40SMatthew G. Knepley     ierr = MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
2577c094ef40SMatthew G. Knepley     ierr = MatSetUp(preallocator);CHKERRQ(ierr);
2578c094ef40SMatthew G. Knepley     ierr = PetscCalloc3(rTotDim*cTotDim, &vals,cTotDim,&cellCIndices,rTotDim,&cellFIndices);CHKERRQ(ierr);
25797f5b169aSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2580cf51de39SMatthew G. Knepley       if (isRefined) {
25817f5b169aSMatthew G. Knepley         ierr = DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices);CHKERRQ(ierr);
2582c094ef40SMatthew G. Knepley         ierr = MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES);CHKERRQ(ierr);
2583cf51de39SMatthew G. Knepley       } else {
2584cf51de39SMatthew G. Knepley         ierr = DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, preallocator, cell, vals, INSERT_VALUES);CHKERRQ(ierr);
2585cf51de39SMatthew G. Knepley       }
25867f5b169aSMatthew G. Knepley     }
2587c094ef40SMatthew G. Knepley     ierr = PetscFree3(vals,cellCIndices,cellFIndices);CHKERRQ(ierr);
2588c094ef40SMatthew G. Knepley     ierr = MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2589c094ef40SMatthew G. Knepley     ierr = MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2590c094ef40SMatthew G. Knepley     ierr = MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In);CHKERRQ(ierr);
2591c094ef40SMatthew G. Knepley     ierr = MatDestroy(&preallocator);CHKERRQ(ierr);
25927f5b169aSMatthew G. Knepley   }
25937f5b169aSMatthew G. Knepley   /* Fill matrix */
25947f5b169aSMatthew G. Knepley   ierr = MatZeroEntries(In);CHKERRQ(ierr);
2595d69c5d34SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
2596cf51de39SMatthew G. Knepley     if (isRefined) {
2597934789fcSMatthew G. Knepley       ierr = DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES);CHKERRQ(ierr);
2598cf51de39SMatthew G. Knepley     } else {
2599cf51de39SMatthew G. Knepley       ierr = DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES);CHKERRQ(ierr);
2600cf51de39SMatthew G. Knepley     }
2601d69c5d34SMatthew G. Knepley   }
2602549a8adaSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {ierr = PetscFEDestroy(&feRef[f]);CHKERRQ(ierr);}
260397c42addSMatthew G. Knepley   ierr = PetscFree2(feRef,fvRef);CHKERRQ(ierr);
2604549a8adaSMatthew G. Knepley   ierr = PetscFree(elemMat);CHKERRQ(ierr);
2605934789fcSMatthew G. Knepley   ierr = MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
2606934789fcSMatthew G. Knepley   ierr = MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
26079318fe57SMatthew G. Knepley   if (mesh->printFEM > 1) {
2608825f8a23SLisandro Dalcin     ierr = PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name);CHKERRQ(ierr);
2609934789fcSMatthew G. Knepley     ierr = MatChop(In, 1.0e-10);CHKERRQ(ierr);
2610825f8a23SLisandro Dalcin     ierr = MatView(In, NULL);CHKERRQ(ierr);
2611d69c5d34SMatthew G. Knepley   }
2612d69c5d34SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);CHKERRQ(ierr);
2613d69c5d34SMatthew G. Knepley   PetscFunctionReturn(0);
2614d69c5d34SMatthew G. Knepley }
26156c73c22cSMatthew G. Knepley 
2616bd041c0cSMatthew G. Knepley PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user)
2617bd041c0cSMatthew G. Knepley {
2618bd041c0cSMatthew G. Knepley   SETERRQ(PetscObjectComm((PetscObject) dmc), PETSC_ERR_SUP, "Laziness");
2619bd041c0cSMatthew G. Knepley }
2620bd041c0cSMatthew G. Knepley 
262168132eb9SMatthew G. Knepley /*@
262268132eb9SMatthew G. Knepley   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix I from the coarse DM to a non-nested fine DM.
262368132eb9SMatthew G. Knepley 
262468132eb9SMatthew G. Knepley   Input Parameters:
262568132eb9SMatthew G. Knepley + dmf  - The fine mesh
262668132eb9SMatthew G. Knepley . dmc  - The coarse mesh
262768132eb9SMatthew G. Knepley - user - The user context
262868132eb9SMatthew G. Knepley 
262968132eb9SMatthew G. Knepley   Output Parameter:
263068132eb9SMatthew G. Knepley . In  - The interpolation matrix
263168132eb9SMatthew G. Knepley 
263268132eb9SMatthew G. Knepley   Level: developer
263368132eb9SMatthew G. Knepley 
263468132eb9SMatthew G. Knepley .seealso: DMPlexComputeInterpolatorNested(), DMPlexComputeJacobianFEM()
263568132eb9SMatthew G. Knepley @*/
263668132eb9SMatthew G. Knepley PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user)
26374ef9d792SMatthew G. Knepley {
263864e98e1dSMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dmf->data;
263964e98e1dSMatthew G. Knepley   const char    *name = "Interpolator";
26404ef9d792SMatthew G. Knepley   PetscDS        prob;
26414ef9d792SMatthew G. Knepley   PetscSection   fsection, csection, globalFSection, globalCSection;
2642e8f14785SLisandro Dalcin   PetscHSetIJ    ht;
26434ef9d792SMatthew G. Knepley   PetscLayout    rLayout;
26444ef9d792SMatthew G. Knepley   PetscInt      *dnz, *onz;
26454ef9d792SMatthew G. Knepley   PetscInt       locRows, rStart, rEnd;
26464ef9d792SMatthew G. Knepley   PetscReal     *x, *v0, *J, *invJ, detJ;
26474ef9d792SMatthew G. Knepley   PetscReal     *v0c, *Jc, *invJc, detJc;
26484ef9d792SMatthew G. Knepley   PetscScalar   *elemMat;
26494ef9d792SMatthew G. Knepley   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
26504ef9d792SMatthew G. Knepley   PetscErrorCode ierr;
26514ef9d792SMatthew G. Knepley 
26524ef9d792SMatthew G. Knepley   PetscFunctionBegin;
265377711781SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);CHKERRQ(ierr);
26544ef9d792SMatthew G. Knepley   ierr = DMGetCoordinateDim(dmc, &dim);CHKERRQ(ierr);
26554ef9d792SMatthew G. Knepley   ierr = DMGetDS(dmc, &prob);CHKERRQ(ierr);
26564bee2e38SMatthew G. Knepley   ierr = PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
26574ef9d792SMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
26584ef9d792SMatthew G. Knepley   ierr = PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ);CHKERRQ(ierr);
26594ef9d792SMatthew G. Knepley   ierr = PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc);CHKERRQ(ierr);
266092fd8e1eSJed Brown   ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);
2661e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);
266292fd8e1eSJed Brown   ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);
2663e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);
26644ef9d792SMatthew G. Knepley   ierr = DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd);CHKERRQ(ierr);
26654ef9d792SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
26669c3cf19fSMatthew G. Knepley   ierr = PetscMalloc1(totDim, &elemMat);CHKERRQ(ierr);
26674ef9d792SMatthew G. Knepley 
26684ef9d792SMatthew G. Knepley   ierr = MatGetLocalSize(In, &locRows, NULL);CHKERRQ(ierr);
26694ef9d792SMatthew G. Knepley   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) In), &rLayout);CHKERRQ(ierr);
26704ef9d792SMatthew G. Knepley   ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
26714ef9d792SMatthew G. Knepley   ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
26724ef9d792SMatthew G. Knepley   ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
26734ef9d792SMatthew G. Knepley   ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
26744ef9d792SMatthew G. Knepley   ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
26754ef9d792SMatthew G. Knepley   ierr = PetscCalloc2(locRows,&dnz,locRows,&onz);CHKERRQ(ierr);
2676e8f14785SLisandro Dalcin   ierr = PetscHSetIJCreate(&ht);CHKERRQ(ierr);
26774ef9d792SMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
26784ef9d792SMatthew G. Knepley     PetscObject      obj;
26794ef9d792SMatthew G. Knepley     PetscClassId     id;
2680c0d7054bSMatthew G. Knepley     PetscDualSpace   Q = NULL;
26814ef9d792SMatthew G. Knepley     PetscQuadrature  f;
268217f047d8SMatthew G. Knepley     const PetscReal *qpoints;
268317f047d8SMatthew G. Knepley     PetscInt         Nc, Np, fpdim, i, d;
26844ef9d792SMatthew G. Knepley 
26854ef9d792SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, field, &obj);CHKERRQ(ierr);
26864ef9d792SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
26874ef9d792SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
26884ef9d792SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
26894ef9d792SMatthew G. Knepley 
26904ef9d792SMatthew G. Knepley       ierr = PetscFEGetDualSpace(fe, &Q);CHKERRQ(ierr);
26914ef9d792SMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
26924ef9d792SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
26934ef9d792SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
26944ef9d792SMatthew G. Knepley 
26954ef9d792SMatthew G. Knepley       ierr = PetscFVGetDualSpace(fv, &Q);CHKERRQ(ierr);
26964ef9d792SMatthew G. Knepley       Nc   = 1;
26974ef9d792SMatthew G. Knepley     }
26984ef9d792SMatthew G. Knepley     ierr = PetscDualSpaceGetDimension(Q, &fpdim);CHKERRQ(ierr);
26994ef9d792SMatthew G. Knepley     /* For each fine grid cell */
27004ef9d792SMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
27014ef9d792SMatthew G. Knepley       PetscInt *findices,   *cindices;
27024ef9d792SMatthew G. Knepley       PetscInt  numFIndices, numCIndices;
27034ef9d792SMatthew G. Knepley 
270471f0bbf9SMatthew G. Knepley       ierr = DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
27054ef9d792SMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);CHKERRQ(ierr);
2706ff1e0c32SBarry Smith       if (numFIndices != fpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %D != %D dual basis vecs", numFIndices, fpdim);
27074ef9d792SMatthew G. Knepley       for (i = 0; i < fpdim; ++i) {
27084ef9d792SMatthew G. Knepley         Vec             pointVec;
27094ef9d792SMatthew G. Knepley         PetscScalar    *pV;
27103a93e3b7SToby Isaac         PetscSF         coarseCellSF = NULL;
27113a93e3b7SToby Isaac         const PetscSFNode *coarseCells;
27129c3cf19fSMatthew G. Knepley         PetscInt        numCoarseCells, q, c;
27134ef9d792SMatthew G. Knepley 
27144ef9d792SMatthew G. Knepley         /* Get points from the dual basis functional quadrature */
27154ef9d792SMatthew G. Knepley         ierr = PetscDualSpaceGetFunctional(Q, i, &f);CHKERRQ(ierr);
27169c3cf19fSMatthew G. Knepley         ierr = PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL);CHKERRQ(ierr);
27174ef9d792SMatthew G. Knepley         ierr = VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec);CHKERRQ(ierr);
27184ef9d792SMatthew G. Knepley         ierr = VecSetBlockSize(pointVec, dim);CHKERRQ(ierr);
27194ef9d792SMatthew G. Knepley         ierr = VecGetArray(pointVec, &pV);CHKERRQ(ierr);
27204ef9d792SMatthew G. Knepley         for (q = 0; q < Np; ++q) {
2721c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2722c330f8ffSToby Isaac 
27234ef9d792SMatthew G. Knepley           /* Transform point to real space */
2724c330f8ffSToby Isaac           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
27254ef9d792SMatthew G. Knepley           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
27264ef9d792SMatthew G. Knepley         }
27274ef9d792SMatthew G. Knepley         ierr = VecRestoreArray(pointVec, &pV);CHKERRQ(ierr);
27284ef9d792SMatthew G. Knepley         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
27291555c271SMatthew G. Knepley         /* OPT: Pack all quad points from fine cell */
273062a38674SMatthew G. Knepley         ierr = DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);CHKERRQ(ierr);
27313a93e3b7SToby Isaac         ierr = PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view");CHKERRQ(ierr);
27324ef9d792SMatthew G. Knepley         /* Update preallocation info */
27333a93e3b7SToby Isaac         ierr = PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);CHKERRQ(ierr);
27343a93e3b7SToby Isaac         if (numCoarseCells != Np) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
27359c3cf19fSMatthew G. Knepley         {
2736e8f14785SLisandro Dalcin           PetscHashIJKey key;
2737e8f14785SLisandro Dalcin           PetscBool      missing;
27384ef9d792SMatthew G. Knepley 
2739e8f14785SLisandro Dalcin           key.i = findices[i];
2740e8f14785SLisandro Dalcin           if (key.i >= 0) {
27414ef9d792SMatthew G. Knepley             /* Get indices for coarse elements */
27424ef9d792SMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
274371f0bbf9SMatthew G. Knepley               ierr = DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
27444ef9d792SMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
2745e8f14785SLisandro Dalcin                 key.j = cindices[c];
2746e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
2747e8f14785SLisandro Dalcin                 ierr = PetscHSetIJQueryAdd(ht, key, &missing);CHKERRQ(ierr);
27484ef9d792SMatthew G. Knepley                 if (missing) {
2749e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
2750e8f14785SLisandro Dalcin                   else                                     ++onz[key.i-rStart];
27514ef9d792SMatthew G. Knepley                 }
27524ef9d792SMatthew G. Knepley               }
275371f0bbf9SMatthew G. Knepley               ierr = DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
27544ef9d792SMatthew G. Knepley             }
27554ef9d792SMatthew G. Knepley           }
27568c543595SMatthew G. Knepley         }
27573a93e3b7SToby Isaac         ierr = PetscSFDestroy(&coarseCellSF);CHKERRQ(ierr);
27584ef9d792SMatthew G. Knepley         ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
27594ef9d792SMatthew G. Knepley       }
276071f0bbf9SMatthew G. Knepley       ierr = DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
27614ef9d792SMatthew G. Knepley     }
27624ef9d792SMatthew G. Knepley   }
2763e8f14785SLisandro Dalcin   ierr = PetscHSetIJDestroy(&ht);CHKERRQ(ierr);
27644ef9d792SMatthew G. Knepley   ierr = MatXAIJSetPreallocation(In, 1, dnz, onz, NULL, NULL);CHKERRQ(ierr);
27654ef9d792SMatthew G. Knepley   ierr = MatSetOption(In, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
27664ef9d792SMatthew G. Knepley   ierr = PetscFree2(dnz,onz);CHKERRQ(ierr);
27674ef9d792SMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
27684ef9d792SMatthew G. Knepley     PetscObject       obj;
27694ef9d792SMatthew G. Knepley     PetscClassId      id;
2770c0d7054bSMatthew G. Knepley     PetscDualSpace    Q = NULL;
2771ef0bb6c7SMatthew G. Knepley     PetscTabulation T = NULL;
27724ef9d792SMatthew G. Knepley     PetscQuadrature   f;
27734ef9d792SMatthew G. Knepley     const PetscReal  *qpoints, *qweights;
27749c3cf19fSMatthew G. Knepley     PetscInt          Nc, qNc, Np, fpdim, i, d;
27754ef9d792SMatthew G. Knepley 
27764ef9d792SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, field, &obj);CHKERRQ(ierr);
27774ef9d792SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
27784ef9d792SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
27794ef9d792SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
27804ef9d792SMatthew G. Knepley 
27814ef9d792SMatthew G. Knepley       ierr = PetscFEGetDualSpace(fe, &Q);CHKERRQ(ierr);
27824ef9d792SMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
2783ef0bb6c7SMatthew G. Knepley       ierr = PetscFECreateTabulation(fe, 1, 1, x, 0, &T);CHKERRQ(ierr);
27844ef9d792SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
27854ef9d792SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
27864ef9d792SMatthew G. Knepley 
27874ef9d792SMatthew G. Knepley       ierr = PetscFVGetDualSpace(fv, &Q);CHKERRQ(ierr);
27884ef9d792SMatthew G. Knepley       Nc   = 1;
2789ff1e0c32SBarry Smith     } else SETERRQ1(PetscObjectComm((PetscObject)dmc),PETSC_ERR_ARG_WRONG,"Unknown discretization type for field %D",field);
27904ef9d792SMatthew G. Knepley     ierr = PetscDualSpaceGetDimension(Q, &fpdim);CHKERRQ(ierr);
27914ef9d792SMatthew G. Knepley     /* For each fine grid cell */
27924ef9d792SMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
27934ef9d792SMatthew G. Knepley       PetscInt *findices,   *cindices;
27944ef9d792SMatthew G. Knepley       PetscInt  numFIndices, numCIndices;
27954ef9d792SMatthew G. Knepley 
279671f0bbf9SMatthew G. Knepley       ierr = DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
27974ef9d792SMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);CHKERRQ(ierr);
2798ff1e0c32SBarry Smith       if (numFIndices != fpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %D != %D dual basis vecs", numFIndices, fpdim);
27994ef9d792SMatthew G. Knepley       for (i = 0; i < fpdim; ++i) {
28004ef9d792SMatthew G. Knepley         Vec             pointVec;
28014ef9d792SMatthew G. Knepley         PetscScalar    *pV;
280212111d7cSToby Isaac         PetscSF         coarseCellSF = NULL;
28033a93e3b7SToby Isaac         const PetscSFNode *coarseCells;
280417f047d8SMatthew G. Knepley         PetscInt        numCoarseCells, cpdim, q, c, j;
28054ef9d792SMatthew G. Knepley 
28064ef9d792SMatthew G. Knepley         /* Get points from the dual basis functional quadrature */
28074ef9d792SMatthew G. Knepley         ierr = PetscDualSpaceGetFunctional(Q, i, &f);CHKERRQ(ierr);
28089c3cf19fSMatthew G. Knepley         ierr = PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights);CHKERRQ(ierr);
28099c3cf19fSMatthew G. Knepley         if (qNc != Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %D does not match coarse field %D", qNc, Nc);
28104ef9d792SMatthew G. Knepley         ierr = VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec);CHKERRQ(ierr);
28114ef9d792SMatthew G. Knepley         ierr = VecSetBlockSize(pointVec, dim);CHKERRQ(ierr);
28124ef9d792SMatthew G. Knepley         ierr = VecGetArray(pointVec, &pV);CHKERRQ(ierr);
28134ef9d792SMatthew G. Knepley         for (q = 0; q < Np; ++q) {
2814c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2815c330f8ffSToby Isaac 
28164ef9d792SMatthew G. Knepley           /* Transform point to real space */
2817c330f8ffSToby Isaac           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
28184ef9d792SMatthew G. Knepley           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
28194ef9d792SMatthew G. Knepley         }
28204ef9d792SMatthew G. Knepley         ierr = VecRestoreArray(pointVec, &pV);CHKERRQ(ierr);
28214ef9d792SMatthew G. Knepley         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
28221555c271SMatthew G. Knepley         /* OPT: Read this out from preallocation information */
282362a38674SMatthew G. Knepley         ierr = DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);CHKERRQ(ierr);
28244ef9d792SMatthew G. Knepley         /* Update preallocation info */
28253a93e3b7SToby Isaac         ierr = PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);CHKERRQ(ierr);
28263a93e3b7SToby Isaac         if (numCoarseCells != Np) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
28274ef9d792SMatthew G. Knepley         ierr = VecGetArray(pointVec, &pV);CHKERRQ(ierr);
28284ef9d792SMatthew G. Knepley         for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2829826eb36dSMatthew G. Knepley           PetscReal pVReal[3];
2830c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2831826eb36dSMatthew G. Knepley 
283271f0bbf9SMatthew G. Knepley           ierr = DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
28334ef9d792SMatthew G. Knepley           /* Transform points from real space to coarse reference space */
28343a93e3b7SToby Isaac           ierr = DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc);CHKERRQ(ierr);
2835e2d86523SMatthew G. Knepley           for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
2836c330f8ffSToby Isaac           CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
28374ef9d792SMatthew G. Knepley 
28384ef9d792SMatthew G. Knepley           if (id == PETSCFE_CLASSID) {
28394ef9d792SMatthew G. Knepley             PetscFE fe = (PetscFE) obj;
28404ef9d792SMatthew G. Knepley 
28414ef9d792SMatthew G. Knepley             /* Evaluate coarse basis on contained point */
28424ef9d792SMatthew G. Knepley             ierr = PetscFEGetDimension(fe, &cpdim);CHKERRQ(ierr);
2843ef0bb6c7SMatthew G. Knepley             ierr = PetscFEComputeTabulation(fe, 1, x, 0, T);CHKERRQ(ierr);
2844580bdb30SBarry Smith             ierr = PetscArrayzero(elemMat, cpdim);CHKERRQ(ierr);
28454ef9d792SMatthew G. Knepley             /* Get elemMat entries by multiplying by weight */
28464ef9d792SMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
2847ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j*Nc + c]*qweights[ccell*qNc + c];
28484ef9d792SMatthew G. Knepley             }
28494ef9d792SMatthew G. Knepley           } else {
28504ef9d792SMatthew G. Knepley             cpdim = 1;
28514ef9d792SMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
28529c3cf19fSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*qweights[ccell*qNc + c];
28534ef9d792SMatthew G. Knepley             }
28544ef9d792SMatthew G. Knepley           }
28554ef9d792SMatthew G. Knepley           /* Update interpolator */
28569c3cf19fSMatthew G. Knepley           if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);CHKERRQ(ierr);}
28579c3cf19fSMatthew G. Knepley           if (numCIndices != cpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %D != %D", numCIndices, cpdim);
28589c3cf19fSMatthew G. Knepley           ierr = MatSetValues(In, 1, &findices[i], numCIndices, cindices, elemMat, INSERT_VALUES);CHKERRQ(ierr);
285971f0bbf9SMatthew G. Knepley           ierr = DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
28604ef9d792SMatthew G. Knepley         }
28614ef9d792SMatthew G. Knepley         ierr = VecRestoreArray(pointVec, &pV);CHKERRQ(ierr);
28623a93e3b7SToby Isaac         ierr = PetscSFDestroy(&coarseCellSF);CHKERRQ(ierr);
28634ef9d792SMatthew G. Knepley         ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
28644ef9d792SMatthew G. Knepley       }
286571f0bbf9SMatthew G. Knepley       ierr = DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
28664ef9d792SMatthew G. Knepley     }
2867ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {ierr = PetscTabulationDestroy(&T);CHKERRQ(ierr);}
28684ef9d792SMatthew G. Knepley   }
28694ef9d792SMatthew G. Knepley   ierr = PetscFree3(v0,J,invJ);CHKERRQ(ierr);
28704ef9d792SMatthew G. Knepley   ierr = PetscFree3(v0c,Jc,invJc);CHKERRQ(ierr);
28714ef9d792SMatthew G. Knepley   ierr = PetscFree(elemMat);CHKERRQ(ierr);
28724ef9d792SMatthew G. Knepley   ierr = MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
28734ef9d792SMatthew G. Knepley   ierr = MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
287477711781SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);CHKERRQ(ierr);
28754ef9d792SMatthew G. Knepley   PetscFunctionReturn(0);
28764ef9d792SMatthew G. Knepley }
28774ef9d792SMatthew G. Knepley 
287846fa42a0SMatthew G. Knepley /*@
2879bd041c0cSMatthew G. Knepley   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix M from the coarse DM to a non-nested fine DM.
2880bd041c0cSMatthew G. Knepley 
2881bd041c0cSMatthew G. Knepley   Input Parameters:
2882bd041c0cSMatthew G. Knepley + dmf  - The fine mesh
2883bd041c0cSMatthew G. Knepley . dmc  - The coarse mesh
2884bd041c0cSMatthew G. Knepley - user - The user context
2885bd041c0cSMatthew G. Knepley 
2886bd041c0cSMatthew G. Knepley   Output Parameter:
2887bd041c0cSMatthew G. Knepley . mass  - The mass matrix
2888bd041c0cSMatthew G. Knepley 
2889bd041c0cSMatthew G. Knepley   Level: developer
2890bd041c0cSMatthew G. Knepley 
2891bd041c0cSMatthew G. Knepley .seealso: DMPlexComputeMassMatrixNested(), DMPlexComputeInterpolatorNested(), DMPlexComputeInterpolatorGeneral(), DMPlexComputeJacobianFEM()
2892bd041c0cSMatthew G. Knepley @*/
2893bd041c0cSMatthew G. Knepley PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user)
2894bd041c0cSMatthew G. Knepley {
2895bd041c0cSMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dmf->data;
2896bd041c0cSMatthew G. Knepley   const char    *name = "Mass Matrix";
2897bd041c0cSMatthew G. Knepley   PetscDS        prob;
2898bd041c0cSMatthew G. Knepley   PetscSection   fsection, csection, globalFSection, globalCSection;
2899e8f14785SLisandro Dalcin   PetscHSetIJ    ht;
2900bd041c0cSMatthew G. Knepley   PetscLayout    rLayout;
2901bd041c0cSMatthew G. Knepley   PetscInt      *dnz, *onz;
2902bd041c0cSMatthew G. Knepley   PetscInt       locRows, rStart, rEnd;
2903bd041c0cSMatthew G. Knepley   PetscReal     *x, *v0, *J, *invJ, detJ;
2904bd041c0cSMatthew G. Knepley   PetscReal     *v0c, *Jc, *invJc, detJc;
2905bd041c0cSMatthew G. Knepley   PetscScalar   *elemMat;
2906bd041c0cSMatthew G. Knepley   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
2907bd041c0cSMatthew G. Knepley   PetscErrorCode ierr;
2908bd041c0cSMatthew G. Knepley 
2909bd041c0cSMatthew G. Knepley   PetscFunctionBegin;
2910bd041c0cSMatthew G. Knepley   ierr = DMGetCoordinateDim(dmc, &dim);CHKERRQ(ierr);
2911bd041c0cSMatthew G. Knepley   ierr = DMGetDS(dmc, &prob);CHKERRQ(ierr);
29124bee2e38SMatthew G. Knepley   ierr = PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
2913bd041c0cSMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
2914bd041c0cSMatthew G. Knepley   ierr = PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ);CHKERRQ(ierr);
2915bd041c0cSMatthew G. Knepley   ierr = PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc);CHKERRQ(ierr);
291692fd8e1eSJed Brown   ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);
2917e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);
291892fd8e1eSJed Brown   ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);
2919e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);
2920bd041c0cSMatthew G. Knepley   ierr = DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd);CHKERRQ(ierr);
2921bd041c0cSMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
2922bd041c0cSMatthew G. Knepley   ierr = PetscMalloc1(totDim, &elemMat);CHKERRQ(ierr);
2923bd041c0cSMatthew G. Knepley 
2924bd041c0cSMatthew G. Knepley   ierr = MatGetLocalSize(mass, &locRows, NULL);CHKERRQ(ierr);
2925bd041c0cSMatthew G. Knepley   ierr = PetscLayoutCreate(PetscObjectComm((PetscObject) mass), &rLayout);CHKERRQ(ierr);
2926bd041c0cSMatthew G. Knepley   ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
2927bd041c0cSMatthew G. Knepley   ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
2928bd041c0cSMatthew G. Knepley   ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
2929bd041c0cSMatthew G. Knepley   ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
2930bd041c0cSMatthew G. Knepley   ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
2931bd041c0cSMatthew G. Knepley   ierr = PetscCalloc2(locRows,&dnz,locRows,&onz);CHKERRQ(ierr);
2932e8f14785SLisandro Dalcin   ierr = PetscHSetIJCreate(&ht);CHKERRQ(ierr);
2933bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
2934bd041c0cSMatthew G. Knepley     PetscObject      obj;
2935bd041c0cSMatthew G. Knepley     PetscClassId     id;
2936bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
2937bd041c0cSMatthew G. Knepley     const PetscReal *qpoints;
2938bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
2939bd041c0cSMatthew G. Knepley 
2940bd041c0cSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, field, &obj);CHKERRQ(ierr);
2941bd041c0cSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
2942bd041c0cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {ierr = PetscFEGetQuadrature((PetscFE) obj, &quad);CHKERRQ(ierr);}
2943bd041c0cSMatthew G. Knepley     else                       {ierr = PetscFVGetQuadrature((PetscFV) obj, &quad);CHKERRQ(ierr);}
2944bd041c0cSMatthew G. Knepley     ierr = PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL);CHKERRQ(ierr);
2945bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
2946bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2947bd041c0cSMatthew G. Knepley       Vec                pointVec;
2948bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
2949bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
2950bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
2951bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, q, c;
2952bd041c0cSMatthew G. Knepley       PetscInt          *findices,   *cindices;
2953bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
2954bd041c0cSMatthew G. Knepley 
295571f0bbf9SMatthew G. Knepley       ierr = DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
2956bd041c0cSMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);CHKERRQ(ierr);
2957bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
2958bd041c0cSMatthew G. Knepley       ierr = VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec);CHKERRQ(ierr);
2959bd041c0cSMatthew G. Knepley       ierr = VecSetBlockSize(pointVec, dim);CHKERRQ(ierr);
2960bd041c0cSMatthew G. Knepley       ierr = VecGetArray(pointVec, &pV);CHKERRQ(ierr);
2961bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
2962c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
2963c330f8ffSToby Isaac 
2964bd041c0cSMatthew G. Knepley         /* Transform point to real space */
2965c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
2966bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
2967bd041c0cSMatthew G. Knepley       }
2968bd041c0cSMatthew G. Knepley       ierr = VecRestoreArray(pointVec, &pV);CHKERRQ(ierr);
2969bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
2970bd041c0cSMatthew G. Knepley       ierr = DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);CHKERRQ(ierr);
2971bd041c0cSMatthew G. Knepley       ierr = PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view");CHKERRQ(ierr);
2972bd041c0cSMatthew G. Knepley       /* Update preallocation info */
2973bd041c0cSMatthew G. Knepley       ierr = PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);CHKERRQ(ierr);
2974bd041c0cSMatthew G. Knepley       if (numCoarseCells != Nq) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
2975bd041c0cSMatthew G. Knepley       {
2976e8f14785SLisandro Dalcin         PetscHashIJKey key;
2977e8f14785SLisandro Dalcin         PetscBool      missing;
2978bd041c0cSMatthew G. Knepley 
2979bd041c0cSMatthew G. Knepley         for (i = 0; i < numFIndices; ++i) {
2980e8f14785SLisandro Dalcin           key.i = findices[i];
2981e8f14785SLisandro Dalcin           if (key.i >= 0) {
2982bd041c0cSMatthew G. Knepley             /* Get indices for coarse elements */
2983bd041c0cSMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
298471f0bbf9SMatthew G. Knepley               ierr = DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
2985bd041c0cSMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
2986e8f14785SLisandro Dalcin                 key.j = cindices[c];
2987e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
2988e8f14785SLisandro Dalcin                 ierr = PetscHSetIJQueryAdd(ht, key, &missing);CHKERRQ(ierr);
2989bd041c0cSMatthew G. Knepley                 if (missing) {
2990e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
2991e8f14785SLisandro Dalcin                   else                                     ++onz[key.i-rStart];
2992bd041c0cSMatthew G. Knepley                 }
2993bd041c0cSMatthew G. Knepley               }
299471f0bbf9SMatthew G. Knepley               ierr = DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
2995bd041c0cSMatthew G. Knepley             }
2996bd041c0cSMatthew G. Knepley           }
2997bd041c0cSMatthew G. Knepley         }
2998bd041c0cSMatthew G. Knepley       }
2999bd041c0cSMatthew G. Knepley       ierr = PetscSFDestroy(&coarseCellSF);CHKERRQ(ierr);
3000bd041c0cSMatthew G. Knepley       ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
300171f0bbf9SMatthew G. Knepley       ierr = DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
3002bd041c0cSMatthew G. Knepley     }
3003bd041c0cSMatthew G. Knepley   }
3004e8f14785SLisandro Dalcin   ierr = PetscHSetIJDestroy(&ht);CHKERRQ(ierr);
3005bd041c0cSMatthew G. Knepley   ierr = MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL);CHKERRQ(ierr);
3006bd041c0cSMatthew G. Knepley   ierr = MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
3007bd041c0cSMatthew G. Knepley   ierr = PetscFree2(dnz,onz);CHKERRQ(ierr);
3008bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3009bd041c0cSMatthew G. Knepley     PetscObject       obj;
3010bd041c0cSMatthew G. Knepley     PetscClassId      id;
3011ef0bb6c7SMatthew G. Knepley     PetscTabulation T, Tfine;
3012bd041c0cSMatthew G. Knepley     PetscQuadrature   quad;
3013bd041c0cSMatthew G. Knepley     const PetscReal  *qpoints, *qweights;
3014bd041c0cSMatthew G. Knepley     PetscInt          Nq, Nc, i, d;
3015bd041c0cSMatthew G. Knepley 
3016bd041c0cSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, field, &obj);CHKERRQ(ierr);
3017bd041c0cSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
3018ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
3019ef0bb6c7SMatthew G. Knepley       ierr = PetscFEGetQuadrature((PetscFE) obj, &quad);CHKERRQ(ierr);
3020f9244615SMatthew G. Knepley       ierr = PetscFEGetCellTabulation((PetscFE) obj, 1, &Tfine);CHKERRQ(ierr);
3021ef0bb6c7SMatthew G. Knepley       ierr = PetscFECreateTabulation((PetscFE) obj, 1, 1, x, 0, &T);CHKERRQ(ierr);
3022ef0bb6c7SMatthew G. Knepley     } else {
3023ef0bb6c7SMatthew G. Knepley       ierr = PetscFVGetQuadrature((PetscFV) obj, &quad);CHKERRQ(ierr);
3024ef0bb6c7SMatthew G. Knepley     }
3025bd041c0cSMatthew G. Knepley     ierr = PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights);CHKERRQ(ierr);
3026bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3027bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3028bd041c0cSMatthew G. Knepley       Vec                pointVec;
3029bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3030bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3031bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3032bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, cpdim, q, c, j;
3033bd041c0cSMatthew G. Knepley       PetscInt          *findices,   *cindices;
3034bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3035bd041c0cSMatthew G. Knepley 
303671f0bbf9SMatthew G. Knepley       ierr = DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
3037bd041c0cSMatthew G. Knepley       ierr = DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);CHKERRQ(ierr);
3038bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
3039bd041c0cSMatthew G. Knepley       ierr = VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec);CHKERRQ(ierr);
3040bd041c0cSMatthew G. Knepley       ierr = VecSetBlockSize(pointVec, dim);CHKERRQ(ierr);
3041bd041c0cSMatthew G. Knepley       ierr = VecGetArray(pointVec, &pV);CHKERRQ(ierr);
3042bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3043c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3044c330f8ffSToby Isaac 
3045bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3046c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
3047bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
3048bd041c0cSMatthew G. Knepley       }
3049bd041c0cSMatthew G. Knepley       ierr = VecRestoreArray(pointVec, &pV);CHKERRQ(ierr);
3050bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
3051bd041c0cSMatthew G. Knepley       ierr = DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);CHKERRQ(ierr);
3052bd041c0cSMatthew G. Knepley       /* Update matrix */
3053bd041c0cSMatthew G. Knepley       ierr = PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);CHKERRQ(ierr);
3054bd041c0cSMatthew G. Knepley       if (numCoarseCells != Nq) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
3055bd041c0cSMatthew G. Knepley       ierr = VecGetArray(pointVec, &pV);CHKERRQ(ierr);
3056bd041c0cSMatthew G. Knepley       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3057bd041c0cSMatthew G. Knepley         PetscReal pVReal[3];
3058c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3059c330f8ffSToby Isaac 
306071f0bbf9SMatthew G. Knepley         ierr = DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
3061bd041c0cSMatthew G. Knepley         /* Transform points from real space to coarse reference space */
3062bd041c0cSMatthew G. Knepley         ierr = DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc);CHKERRQ(ierr);
3063bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
3064c330f8ffSToby Isaac         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
3065bd041c0cSMatthew G. Knepley 
3066bd041c0cSMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
3067bd041c0cSMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
3068bd041c0cSMatthew G. Knepley 
3069bd041c0cSMatthew G. Knepley           /* Evaluate coarse basis on contained point */
3070bd041c0cSMatthew G. Knepley           ierr = PetscFEGetDimension(fe, &cpdim);CHKERRQ(ierr);
3071ef0bb6c7SMatthew G. Knepley           ierr = PetscFEComputeTabulation(fe, 1, x, 0, T);CHKERRQ(ierr);
3072bd041c0cSMatthew G. Knepley           /* Get elemMat entries by multiplying by weight */
3073bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
3074580bdb30SBarry Smith             ierr = PetscArrayzero(elemMat, cpdim);CHKERRQ(ierr);
3075bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3076ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j*Nc + c]*Tfine->T[0][(ccell*numFIndices + i)*Nc + c]*qweights[ccell*Nc + c]*detJ;
3077bd041c0cSMatthew G. Knepley             }
3078bd041c0cSMatthew G. Knepley             /* Update interpolator */
3079bd041c0cSMatthew G. Knepley             if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);CHKERRQ(ierr);}
3080bd041c0cSMatthew G. Knepley             if (numCIndices != cpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %D != %D", numCIndices, cpdim);
3081bd041c0cSMatthew G. Knepley             ierr = MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES);CHKERRQ(ierr);
3082bd041c0cSMatthew G. Knepley           }
3083bd041c0cSMatthew G. Knepley         } else {
3084bd041c0cSMatthew G. Knepley           cpdim = 1;
3085bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
3086580bdb30SBarry Smith             ierr = PetscArrayzero(elemMat, cpdim);CHKERRQ(ierr);
3087bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3088bd041c0cSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*1.0*qweights[ccell*Nc + c]*detJ;
3089bd041c0cSMatthew G. Knepley             }
3090bd041c0cSMatthew G. Knepley             /* Update interpolator */
3091bd041c0cSMatthew G. Knepley             if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);CHKERRQ(ierr);}
3092ff1e0c32SBarry Smith             ierr = PetscPrintf(PETSC_COMM_SELF, "Nq: %D %D Nf: %D %D Nc: %D %D\n", ccell, Nq, i, numFIndices, j, numCIndices);CHKERRQ(ierr);
3093bd041c0cSMatthew G. Knepley             if (numCIndices != cpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %D != %D", numCIndices, cpdim);
3094bd041c0cSMatthew G. Knepley             ierr = MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES);CHKERRQ(ierr);
3095bd041c0cSMatthew G. Knepley           }
3096bd041c0cSMatthew G. Knepley         }
309771f0bbf9SMatthew G. Knepley         ierr = DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL);CHKERRQ(ierr);
3098bd041c0cSMatthew G. Knepley       }
3099bd041c0cSMatthew G. Knepley       ierr = VecRestoreArray(pointVec, &pV);CHKERRQ(ierr);
3100bd041c0cSMatthew G. Knepley       ierr = PetscSFDestroy(&coarseCellSF);CHKERRQ(ierr);
3101bd041c0cSMatthew G. Knepley       ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
310271f0bbf9SMatthew G. Knepley       ierr = DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL);CHKERRQ(ierr);
3103bd041c0cSMatthew G. Knepley     }
3104ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {ierr = PetscTabulationDestroy(&T);CHKERRQ(ierr);}
3105bd041c0cSMatthew G. Knepley   }
3106bd041c0cSMatthew G. Knepley   ierr = PetscFree3(v0,J,invJ);CHKERRQ(ierr);
3107bd041c0cSMatthew G. Knepley   ierr = PetscFree3(v0c,Jc,invJc);CHKERRQ(ierr);
3108bd041c0cSMatthew G. Knepley   ierr = PetscFree(elemMat);CHKERRQ(ierr);
3109bd041c0cSMatthew G. Knepley   ierr = MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
3110bd041c0cSMatthew G. Knepley   ierr = MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
3111bd041c0cSMatthew G. Knepley   PetscFunctionReturn(0);
3112bd041c0cSMatthew G. Knepley }
3113bd041c0cSMatthew G. Knepley 
3114bd041c0cSMatthew G. Knepley /*@
311546fa42a0SMatthew G. Knepley   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns
311646fa42a0SMatthew G. Knepley 
311746fa42a0SMatthew G. Knepley   Input Parameters:
311846fa42a0SMatthew G. Knepley + dmc  - The coarse mesh
311946fa42a0SMatthew G. Knepley - dmf  - The fine mesh
312046fa42a0SMatthew G. Knepley - user - The user context
312146fa42a0SMatthew G. Knepley 
312246fa42a0SMatthew G. Knepley   Output Parameter:
312346fa42a0SMatthew G. Knepley . sc   - The mapping
312446fa42a0SMatthew G. Knepley 
312546fa42a0SMatthew G. Knepley   Level: developer
312646fa42a0SMatthew G. Knepley 
312746fa42a0SMatthew G. Knepley .seealso: DMPlexComputeInterpolatorNested(), DMPlexComputeJacobianFEM()
312846fa42a0SMatthew G. Knepley @*/
31297c927364SMatthew G. Knepley PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user)
31307c927364SMatthew G. Knepley {
3131e9d4ef1bSMatthew G. Knepley   PetscDS        prob;
31327c927364SMatthew G. Knepley   PetscFE       *feRef;
313397c42addSMatthew G. Knepley   PetscFV       *fvRef;
31347c927364SMatthew G. Knepley   Vec            fv, cv;
31357c927364SMatthew G. Knepley   IS             fis, cis;
31367c927364SMatthew G. Knepley   PetscSection   fsection, fglobalSection, csection, cglobalSection;
31377c927364SMatthew G. Knepley   PetscInt      *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
3138485ad865SMatthew G. Knepley   PetscInt       cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m;
31396f3d3cbcSMatthew G. Knepley   PetscBool     *needAvg;
31407c927364SMatthew G. Knepley   PetscErrorCode ierr;
31417c927364SMatthew G. Knepley 
31427c927364SMatthew G. Knepley   PetscFunctionBegin;
314375a69067SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_InjectorFEM,dmc,dmf,0,0);CHKERRQ(ierr);
3144c73cfb54SMatthew G. Knepley   ierr = DMGetDimension(dmf, &dim);CHKERRQ(ierr);
314592fd8e1eSJed Brown   ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);
3146e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmf, &fglobalSection);CHKERRQ(ierr);
314792fd8e1eSJed Brown   ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);
3148e87a4003SBarry Smith   ierr = DMGetGlobalSection(dmc, &cglobalSection);CHKERRQ(ierr);
31497c927364SMatthew G. Knepley   ierr = PetscSectionGetNumFields(fsection, &Nf);CHKERRQ(ierr);
3150412e9a14SMatthew G. Knepley   ierr = DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd);CHKERRQ(ierr);
3151e9d4ef1bSMatthew G. Knepley   ierr = DMGetDS(dmc, &prob);CHKERRQ(ierr);
31526f3d3cbcSMatthew G. Knepley   ierr = PetscCalloc3(Nf,&feRef,Nf,&fvRef,Nf,&needAvg);CHKERRQ(ierr);
31537c927364SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
315497c42addSMatthew G. Knepley     PetscObject  obj;
315597c42addSMatthew G. Knepley     PetscClassId id;
3156aa7890ccSMatthew G. Knepley     PetscInt     fNb = 0, Nc = 0;
31577c927364SMatthew G. Knepley 
315897c42addSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
315997c42addSMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
316097c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
316197c42addSMatthew G. Knepley       PetscFE    fe = (PetscFE) obj;
31626f3d3cbcSMatthew G. Knepley       PetscSpace sp;
31639b2fc754SMatthew G. Knepley       PetscInt   maxDegree;
316497c42addSMatthew G. Knepley 
31657c927364SMatthew G. Knepley       ierr = PetscFERefine(fe, &feRef[f]);CHKERRQ(ierr);
31667c927364SMatthew G. Knepley       ierr = PetscFEGetDimension(feRef[f], &fNb);CHKERRQ(ierr);
31677c927364SMatthew G. Knepley       ierr = PetscFEGetNumComponents(fe, &Nc);CHKERRQ(ierr);
31686f3d3cbcSMatthew G. Knepley       ierr = PetscFEGetBasisSpace(fe, &sp);CHKERRQ(ierr);
31699b2fc754SMatthew G. Knepley       ierr = PetscSpaceGetDegree(sp, NULL, &maxDegree);CHKERRQ(ierr);
31709b2fc754SMatthew G. Knepley       if (!maxDegree) needAvg[f] = PETSC_TRUE;
317197c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
317297c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV) obj;
317397c42addSMatthew G. Knepley       PetscDualSpace Q;
317497c42addSMatthew G. Knepley 
317597c42addSMatthew G. Knepley       ierr = PetscFVRefine(fv, &fvRef[f]);CHKERRQ(ierr);
317697c42addSMatthew G. Knepley       ierr = PetscFVGetDualSpace(fvRef[f], &Q);CHKERRQ(ierr);
317797c42addSMatthew G. Knepley       ierr = PetscDualSpaceGetDimension(Q, &fNb);CHKERRQ(ierr);
317897c42addSMatthew G. Knepley       ierr = PetscFVGetNumComponents(fv, &Nc);CHKERRQ(ierr);
31796f3d3cbcSMatthew G. Knepley       needAvg[f] = PETSC_TRUE;
318097c42addSMatthew G. Knepley     }
3181d172c84bSMatthew G. Knepley     fTotDim += fNb;
31827c927364SMatthew G. Knepley   }
3183e9d4ef1bSMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &cTotDim);CHKERRQ(ierr);
31847c927364SMatthew G. Knepley   ierr = PetscMalloc1(cTotDim,&cmap);CHKERRQ(ierr);
31857c927364SMatthew G. Knepley   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
31867c927364SMatthew G. Knepley     PetscFE        feC;
318797c42addSMatthew G. Knepley     PetscFV        fvC;
31887c927364SMatthew G. Knepley     PetscDualSpace QF, QC;
3189d172c84bSMatthew G. Knepley     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;
31907c927364SMatthew G. Knepley 
319197c42addSMatthew G. Knepley     if (feRef[field]) {
3192e9d4ef1bSMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, field, (PetscObject *) &feC);CHKERRQ(ierr);
31937c927364SMatthew G. Knepley       ierr = PetscFEGetNumComponents(feC, &NcC);CHKERRQ(ierr);
31947c927364SMatthew G. Knepley       ierr = PetscFEGetNumComponents(feRef[field], &NcF);CHKERRQ(ierr);
31957c927364SMatthew G. Knepley       ierr = PetscFEGetDualSpace(feRef[field], &QF);CHKERRQ(ierr);
3196d172c84bSMatthew G. Knepley       ierr = PetscDualSpaceGetOrder(QF, &order);CHKERRQ(ierr);
31977c927364SMatthew G. Knepley       ierr = PetscDualSpaceGetDimension(QF, &fpdim);CHKERRQ(ierr);
31987c927364SMatthew G. Knepley       ierr = PetscFEGetDualSpace(feC, &QC);CHKERRQ(ierr);
31997c927364SMatthew G. Knepley       ierr = PetscDualSpaceGetDimension(QC, &cpdim);CHKERRQ(ierr);
320097c42addSMatthew G. Knepley     } else {
320197c42addSMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, field, (PetscObject *) &fvC);CHKERRQ(ierr);
320297c42addSMatthew G. Knepley       ierr = PetscFVGetNumComponents(fvC, &NcC);CHKERRQ(ierr);
320397c42addSMatthew G. Knepley       ierr = PetscFVGetNumComponents(fvRef[field], &NcF);CHKERRQ(ierr);
320497c42addSMatthew G. Knepley       ierr = PetscFVGetDualSpace(fvRef[field], &QF);CHKERRQ(ierr);
320597c42addSMatthew G. Knepley       ierr = PetscDualSpaceGetDimension(QF, &fpdim);CHKERRQ(ierr);
320697c42addSMatthew G. Knepley       ierr = PetscFVGetDualSpace(fvC, &QC);CHKERRQ(ierr);
320797c42addSMatthew G. Knepley       ierr = PetscDualSpaceGetDimension(QC, &cpdim);CHKERRQ(ierr);
320897c42addSMatthew G. Knepley     }
3209ff1e0c32SBarry Smith     if (NcF != NcC) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %D does not match coarse field %D", NcF, NcC);
32107c927364SMatthew G. Knepley     for (c = 0; c < cpdim; ++c) {
32117c927364SMatthew G. Knepley       PetscQuadrature  cfunc;
3212d172c84bSMatthew G. Knepley       const PetscReal *cqpoints, *cqweights;
3213d172c84bSMatthew G. Knepley       PetscInt         NqcC, NpC;
321497c42addSMatthew G. Knepley       PetscBool        found = PETSC_FALSE;
32157c927364SMatthew G. Knepley 
32167c927364SMatthew G. Knepley       ierr = PetscDualSpaceGetFunctional(QC, c, &cfunc);CHKERRQ(ierr);
3217d172c84bSMatthew G. Knepley       ierr = PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights);CHKERRQ(ierr);
3218ff1e0c32SBarry Smith       if (NqcC != NcC) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %D must match number of field components %D", NqcC, NcC);
321997c42addSMatthew G. Knepley       if (NpC != 1 && feRef[field]) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
32207c927364SMatthew G. Knepley       for (f = 0; f < fpdim; ++f) {
32217c927364SMatthew G. Knepley         PetscQuadrature  ffunc;
3222d172c84bSMatthew G. Knepley         const PetscReal *fqpoints, *fqweights;
32237c927364SMatthew G. Knepley         PetscReal        sum = 0.0;
3224d172c84bSMatthew G. Knepley         PetscInt         NqcF, NpF;
32257c927364SMatthew G. Knepley 
32267c927364SMatthew G. Knepley         ierr = PetscDualSpaceGetFunctional(QF, f, &ffunc);CHKERRQ(ierr);
3227d172c84bSMatthew G. Knepley         ierr = PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights);CHKERRQ(ierr);
3228ff1e0c32SBarry Smith         if (NqcF != NcF) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %D must match number of field components %D", NqcF, NcF);
32297c927364SMatthew G. Knepley         if (NpC != NpF) continue;
32307c927364SMatthew G. Knepley         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
32317c927364SMatthew G. Knepley         if (sum > 1.0e-9) continue;
3232d172c84bSMatthew G. Knepley         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d]*fqweights[d]);
3233d172c84bSMatthew G. Knepley         if (sum < 1.0e-9) continue;
3234d172c84bSMatthew G. Knepley         cmap[offsetC+c] = offsetF+f;
323597c42addSMatthew G. Knepley         found = PETSC_TRUE;
32367c927364SMatthew G. Knepley         break;
32377c927364SMatthew G. Knepley       }
323897c42addSMatthew G. Knepley       if (!found) {
323997c42addSMatthew G. Knepley         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
3240d172c84bSMatthew G. Knepley         if (fvRef[field] || (feRef[field] && order == 0)) {
3241d172c84bSMatthew G. Knepley           cmap[offsetC+c] = offsetF+0;
324297c42addSMatthew G. Knepley         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
324397c42addSMatthew G. Knepley       }
32447c927364SMatthew G. Knepley     }
3245d172c84bSMatthew G. Knepley     offsetC += cpdim;
3246d172c84bSMatthew G. Knepley     offsetF += fpdim;
32477c927364SMatthew G. Knepley   }
324897c42addSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {ierr = PetscFEDestroy(&feRef[f]);CHKERRQ(ierr);ierr = PetscFVDestroy(&fvRef[f]);CHKERRQ(ierr);}
32496f3d3cbcSMatthew G. Knepley   ierr = PetscFree3(feRef,fvRef,needAvg);CHKERRQ(ierr);
32507c927364SMatthew G. Knepley 
32517c927364SMatthew G. Knepley   ierr = DMGetGlobalVector(dmf, &fv);CHKERRQ(ierr);
32527c927364SMatthew G. Knepley   ierr = DMGetGlobalVector(dmc, &cv);CHKERRQ(ierr);
32530bd915a7SMatthew G. Knepley   ierr = VecGetOwnershipRange(cv, &startC, &endC);CHKERRQ(ierr);
32547c927364SMatthew G. Knepley   ierr = PetscSectionGetConstrainedStorageSize(cglobalSection, &m);CHKERRQ(ierr);
32557c927364SMatthew G. Knepley   ierr = PetscMalloc2(cTotDim,&cellCIndices,fTotDim,&cellFIndices);CHKERRQ(ierr);
3256aa7da3c4SMatthew G. Knepley   ierr = PetscMalloc1(m,&cindices);CHKERRQ(ierr);
3257aa7da3c4SMatthew G. Knepley   ierr = PetscMalloc1(m,&findices);CHKERRQ(ierr);
32587c927364SMatthew G. Knepley   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
32597c927364SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
32607c927364SMatthew G. Knepley     ierr = DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices);CHKERRQ(ierr);
32617c927364SMatthew G. Knepley     for (d = 0; d < cTotDim; ++d) {
32620bd915a7SMatthew G. Knepley       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
3263ff1e0c32SBarry Smith       if ((findices[cellCIndices[d]-startC] >= 0) && (findices[cellCIndices[d]-startC] != cellFIndices[cmap[d]])) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Coarse dof %D maps to both %D and %D", cindices[cellCIndices[d]-startC], findices[cellCIndices[d]-startC], cellFIndices[cmap[d]]);
32647c927364SMatthew G. Knepley       cindices[cellCIndices[d]-startC] = cellCIndices[d];
32657c927364SMatthew G. Knepley       findices[cellCIndices[d]-startC] = cellFIndices[cmap[d]];
32667c927364SMatthew G. Knepley     }
32677c927364SMatthew G. Knepley   }
32687c927364SMatthew G. Knepley   ierr = PetscFree(cmap);CHKERRQ(ierr);
32697c927364SMatthew G. Knepley   ierr = PetscFree2(cellCIndices,cellFIndices);CHKERRQ(ierr);
32707c927364SMatthew G. Knepley 
32717c927364SMatthew G. Knepley   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);CHKERRQ(ierr);
32727c927364SMatthew G. Knepley   ierr = ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);CHKERRQ(ierr);
32739448b7f1SJunchao Zhang   ierr = VecScatterCreate(cv, cis, fv, fis, sc);CHKERRQ(ierr);
32747c927364SMatthew G. Knepley   ierr = ISDestroy(&cis);CHKERRQ(ierr);
32757c927364SMatthew G. Knepley   ierr = ISDestroy(&fis);CHKERRQ(ierr);
32767c927364SMatthew G. Knepley   ierr = DMRestoreGlobalVector(dmf, &fv);CHKERRQ(ierr);
32777c927364SMatthew G. Knepley   ierr = DMRestoreGlobalVector(dmc, &cv);CHKERRQ(ierr);
327875a69067SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_InjectorFEM,dmc,dmf,0,0);CHKERRQ(ierr);
3279cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
3280cb1e1211SMatthew G Knepley }
3281a1cf66bbSMatthew G. Knepley 
32822f856554SMatthew G. Knepley /*@C
32832f856554SMatthew G. Knepley   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells
32842f856554SMatthew G. Knepley 
32852f856554SMatthew G. Knepley   Input Parameters:
32862f856554SMatthew G. Knepley + dm     - The DM
32872f856554SMatthew G. Knepley . cellIS - The cells to include
32882f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
32892f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
32902f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
32912f856554SMatthew G. Knepley 
32922f856554SMatthew G. Knepley   Output Parameters:
32932f856554SMatthew G. Knepley + u   - The field coefficients
32942f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
32952f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
32962f856554SMatthew G. Knepley 
32972f856554SMatthew G. Knepley   Level: developer
32982f856554SMatthew G. Knepley 
32992f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
33002f856554SMatthew G. Knepley @*/
33012f856554SMatthew G. Knepley PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
33022f856554SMatthew G. Knepley {
33032f856554SMatthew G. Knepley   DM              plex, plexA = NULL;
3304a6e0b375SMatthew G. Knepley   DMEnclosureType encAux;
33052f856554SMatthew G. Knepley   PetscSection    section, sectionAux;
33062f856554SMatthew G. Knepley   PetscDS         prob;
33072f856554SMatthew G. Knepley   const PetscInt *cells;
33082f856554SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
33092f856554SMatthew G. Knepley   PetscErrorCode  ierr;
33102f856554SMatthew G. Knepley 
33112f856554SMatthew G. Knepley   PetscFunctionBegin;
33122f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3313064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
3314064a246eSJacob Faibussowitsch   if (locX_t) {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4);}
3315064a246eSJacob Faibussowitsch   if (locA)   {PetscValidHeaderSpecific(locA, VEC_CLASSID, 5);}
3316064a246eSJacob Faibussowitsch   PetscValidPointer(u, 6);
3317064a246eSJacob Faibussowitsch   PetscValidPointer(u_t, 7);
3318064a246eSJacob Faibussowitsch   PetscValidPointer(a, 8);
33192f856554SMatthew G. Knepley   ierr = DMPlexConvertPlex(dm, &plex, PETSC_FALSE);CHKERRQ(ierr);
33202f856554SMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
332192fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
3322083401c6SMatthew G. Knepley   ierr = DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob);CHKERRQ(ierr);
33232f856554SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
33242f856554SMatthew G. Knepley   if (locA) {
33252f856554SMatthew G. Knepley     DM      dmAux;
33262f856554SMatthew G. Knepley     PetscDS probAux;
33272f856554SMatthew G. Knepley 
33282f856554SMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
3329a6e0b375SMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux, dm, &encAux);CHKERRQ(ierr);
33302f856554SMatthew G. Knepley     ierr = DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE);CHKERRQ(ierr);
333192fd8e1eSJed Brown     ierr = DMGetLocalSection(dmAux, &sectionAux);CHKERRQ(ierr);
33322f856554SMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
33332f856554SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
33342f856554SMatthew G. Knepley   }
33352f856554SMatthew G. Knepley   numCells = cEnd - cStart;
33362f856554SMatthew G. Knepley   ierr = DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u);CHKERRQ(ierr);
33372f856554SMatthew G. Knepley   if (locX_t) {ierr = DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u_t);CHKERRQ(ierr);} else {*u_t = NULL;}
33382f856554SMatthew G. Knepley   if (locA)   {ierr = DMGetWorkArray(dm, numCells*totDimAux, MPIU_SCALAR, a);CHKERRQ(ierr);} else {*a = NULL;}
33392f856554SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
33402f856554SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
33412f856554SMatthew G. Knepley     const PetscInt cind = c - cStart;
33422f856554SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
33432f856554SMatthew G. Knepley     PetscInt       i;
33442f856554SMatthew G. Knepley 
33452f856554SMatthew G. Knepley     ierr = DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x);CHKERRQ(ierr);
33462f856554SMatthew G. Knepley     for (i = 0; i < totDim; ++i) ul[cind*totDim+i] = x[i];
33472f856554SMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x);CHKERRQ(ierr);
33482f856554SMatthew G. Knepley     if (locX_t) {
33492f856554SMatthew G. Knepley       ierr = DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t);CHKERRQ(ierr);
33502f856554SMatthew G. Knepley       for (i = 0; i < totDim; ++i) ul_t[cind*totDim+i] = x_t[i];
33512f856554SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t);CHKERRQ(ierr);
33522f856554SMatthew G. Knepley     }
33532f856554SMatthew G. Knepley     if (locA) {
33542f856554SMatthew G. Knepley       PetscInt subcell;
3355a6e0b375SMatthew G. Knepley       ierr = DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell);CHKERRQ(ierr);
33562f856554SMatthew G. Knepley       ierr = DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x);CHKERRQ(ierr);
33572f856554SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) al[cind*totDimAux+i] = x[i];
33582f856554SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x);CHKERRQ(ierr);
33592f856554SMatthew G. Knepley     }
33602f856554SMatthew G. Knepley   }
33612f856554SMatthew G. Knepley   ierr = DMDestroy(&plex);CHKERRQ(ierr);
33622f856554SMatthew G. Knepley   if (locA) {ierr = DMDestroy(&plexA);CHKERRQ(ierr);}
33632f856554SMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
33642f856554SMatthew G. Knepley   PetscFunctionReturn(0);
33652f856554SMatthew G. Knepley }
33662f856554SMatthew G. Knepley 
33672f856554SMatthew G. Knepley /*@C
33682f856554SMatthew G. Knepley   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells
33692f856554SMatthew G. Knepley 
33702f856554SMatthew G. Knepley   Input Parameters:
33712f856554SMatthew G. Knepley + dm     - The DM
33722f856554SMatthew G. Knepley . cellIS - The cells to include
33732f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
33742f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
33752f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
33762f856554SMatthew G. Knepley 
33772f856554SMatthew G. Knepley   Output Parameters:
33782f856554SMatthew G. Knepley + u   - The field coefficients
33792f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
33802f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
33812f856554SMatthew G. Knepley 
33822f856554SMatthew G. Knepley   Level: developer
33832f856554SMatthew G. Knepley 
33842f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
33852f856554SMatthew G. Knepley @*/
33862f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
33872f856554SMatthew G. Knepley {
33882f856554SMatthew G. Knepley   PetscErrorCode ierr;
33892f856554SMatthew G. Knepley 
33902f856554SMatthew G. Knepley   PetscFunctionBegin;
33912f856554SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u);CHKERRQ(ierr);
33922f856554SMatthew G. Knepley   if (locX_t) {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t);CHKERRQ(ierr);}
33932f856554SMatthew G. Knepley   if (locA)   {ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a);CHKERRQ(ierr);}
33942f856554SMatthew G. Knepley   PetscFunctionReturn(0);
33952f856554SMatthew G. Knepley }
33962f856554SMatthew G. Knepley 
3397148442b3SMatthew G. Knepley static PetscErrorCode DMPlexGetHybridAuxFields(DM dm, DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[])
33986528b96dSMatthew G. Knepley {
339904c51a94SMatthew G. Knepley   DM              plexA[2];
3400148442b3SMatthew G. Knepley   DMEnclosureType encAux[2];
340104c51a94SMatthew G. Knepley   PetscSection    sectionAux[2];
34026528b96dSMatthew G. Knepley   const PetscInt *cells;
3403148442b3SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, s, totDimAux[2];
34046528b96dSMatthew G. Knepley   PetscErrorCode  ierr;
34056528b96dSMatthew G. Knepley 
34066528b96dSMatthew G. Knepley   PetscFunctionBegin;
3407148442b3SMatthew G. Knepley   PetscValidPointer(locA, 5);
340804c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
3409148442b3SMatthew G. Knepley   PetscValidPointer(dmAux, 2);
3410148442b3SMatthew G. Knepley   PetscValidPointer(dsAux, 3);
3411148442b3SMatthew G. Knepley   PetscValidPointer(a, 6);
34126528b96dSMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
34136528b96dSMatthew G. Knepley   numCells = cEnd - cStart;
3414148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
3415148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dmAux[s], DM_CLASSID, 2);
3416148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dsAux[s], PETSCDS_CLASSID, 3);
3417148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(locA[s], VEC_CLASSID, 5);
3418148442b3SMatthew G. Knepley     ierr = DMPlexConvertPlex(dmAux[s], &plexA[s], PETSC_FALSE);CHKERRQ(ierr);
3419148442b3SMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux[s], dm, &encAux[s]);CHKERRQ(ierr);
3420148442b3SMatthew G. Knepley     ierr = DMGetLocalSection(dmAux[s], &sectionAux[s]);CHKERRQ(ierr);
3421148442b3SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(dsAux[s], &totDimAux[s]);CHKERRQ(ierr);
3422148442b3SMatthew G. Knepley     ierr = DMGetWorkArray(dmAux[s], numCells*totDimAux[s], MPIU_SCALAR, &a[s]);CHKERRQ(ierr);
342304c51a94SMatthew G. Knepley   }
3424148442b3SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
34256528b96dSMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
34266528b96dSMatthew G. Knepley     const PetscInt  cind = c - cStart;
34276528b96dSMatthew G. Knepley     const PetscInt *cone, *ornt;
34286528b96dSMatthew G. Knepley 
3429148442b3SMatthew G. Knepley     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
3430148442b3SMatthew G. Knepley     ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
3431148442b3SMatthew G. Knepley     if (ornt[0]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %D in hybrid cell %D has orientation %D != 0", cone[0], cell, ornt[0]);
3432148442b3SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
3433148442b3SMatthew G. Knepley       PetscScalar   *x = NULL, *al = a[s];
3434148442b3SMatthew G. Knepley       const PetscInt tdA = totDimAux[s];
3435148442b3SMatthew G. Knepley       PetscInt       subface, Na, i;
34366528b96dSMatthew G. Knepley 
3437148442b3SMatthew G. Knepley       ierr = DMGetEnclosurePoint(plexA[s], dm, encAux[s], cone[0], &subface);CHKERRQ(ierr);
3438148442b3SMatthew G. Knepley       ierr = DMPlexVecGetClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x);CHKERRQ(ierr);
34396528b96dSMatthew G. Knepley       for (i = 0; i < Na; ++i) al[cind*tdA+i] = x[i];
3440148442b3SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x);CHKERRQ(ierr);
34416528b96dSMatthew G. Knepley     }
34426528b96dSMatthew G. Knepley   }
3443148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {ierr = DMDestroy(&plexA[s]);CHKERRQ(ierr);}
34446528b96dSMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
34456528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
34466528b96dSMatthew G. Knepley }
34476528b96dSMatthew G. Knepley 
344804c51a94SMatthew G. Knepley static PetscErrorCode DMPlexRestoreHybridAuxFields(DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[])
34496528b96dSMatthew G. Knepley {
34506528b96dSMatthew G. Knepley   PetscErrorCode ierr;
34516528b96dSMatthew G. Knepley 
34526528b96dSMatthew G. Knepley   PetscFunctionBegin;
345304c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
345404c51a94SMatthew G. Knepley   ierr = DMRestoreWorkArray(dmAux[0], 0, MPIU_SCALAR, &a[0]);CHKERRQ(ierr);
345504c51a94SMatthew G. Knepley   ierr = DMRestoreWorkArray(dmAux[1], 0, MPIU_SCALAR, &a[1]);CHKERRQ(ierr);
34566528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
34576528b96dSMatthew G. Knepley }
34586528b96dSMatthew G. Knepley 
34592f856554SMatthew G. Knepley /*@C
34602f856554SMatthew G. Knepley   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces
34612f856554SMatthew G. Knepley 
34622f856554SMatthew G. Knepley   Input Parameters:
34632f856554SMatthew G. Knepley + dm     - The DM
34642f856554SMatthew G. Knepley . fStart - The first face to include
34652f856554SMatthew G. Knepley . fEnd   - The first face to exclude
34662f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
34672f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
34682f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
34692f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
34702f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
34712f856554SMatthew G. Knepley 
34722f856554SMatthew G. Knepley   Output Parameters:
34732f856554SMatthew G. Knepley + Nface - The number of faces with field values
34742f856554SMatthew G. Knepley . uL - The field values at the left side of the face
34752f856554SMatthew G. Knepley - uR - The field values at the right side of the face
34762f856554SMatthew G. Knepley 
34772f856554SMatthew G. Knepley   Level: developer
34782f856554SMatthew G. Knepley 
34792f856554SMatthew G. Knepley .seealso: DMPlexGetCellFields()
34802f856554SMatthew G. Knepley @*/
34812f856554SMatthew G. Knepley PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
34822f856554SMatthew G. Knepley {
34832f856554SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad = NULL;
34842f856554SMatthew G. Knepley   PetscSection       section;
34852f856554SMatthew G. Knepley   PetscDS            prob;
34862f856554SMatthew G. Knepley   DMLabel            ghostLabel;
34872f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
34882f856554SMatthew G. Knepley   PetscBool         *isFE;
34892f856554SMatthew G. Knepley   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
34902f856554SMatthew G. Knepley   PetscErrorCode     ierr;
34912f856554SMatthew G. Knepley 
34922f856554SMatthew G. Knepley   PetscFunctionBegin;
34932f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
34942f856554SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 4);
34952f856554SMatthew G. Knepley   if (locX_t) {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5);}
34962f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6);
34972f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7);
34982f856554SMatthew G. Knepley   if (locGrad) {PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8);}
3499064a246eSJacob Faibussowitsch   PetscValidPointer(uL, 10);
3500064a246eSJacob Faibussowitsch   PetscValidPointer(uR, 11);
35012f856554SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
35022f856554SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
350392fd8e1eSJed Brown   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
35042f856554SMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
35052f856554SMatthew G. Knepley   ierr = PetscDSGetTotalComponents(prob, &Nc);CHKERRQ(ierr);
35062f856554SMatthew G. Knepley   ierr = PetscMalloc1(Nf, &isFE);CHKERRQ(ierr);
35072f856554SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
35082f856554SMatthew G. Knepley     PetscObject  obj;
35092f856554SMatthew G. Knepley     PetscClassId id;
35102f856554SMatthew G. Knepley 
35112f856554SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
35122f856554SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
35132f856554SMatthew G. Knepley     if (id == PETSCFE_CLASSID)      {isFE[f] = PETSC_TRUE;}
35142f856554SMatthew G. Knepley     else if (id == PETSCFV_CLASSID) {isFE[f] = PETSC_FALSE;}
35152f856554SMatthew G. Knepley     else                            {isFE[f] = PETSC_FALSE;}
35162f856554SMatthew G. Knepley   }
35172f856554SMatthew G. Knepley   ierr = DMGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
35182f856554SMatthew G. Knepley   ierr = VecGetArrayRead(locX, &x);CHKERRQ(ierr);
35192f856554SMatthew G. Knepley   ierr = VecGetDM(faceGeometry, &dmFace);CHKERRQ(ierr);
35202f856554SMatthew G. Knepley   ierr = VecGetArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
35212f856554SMatthew G. Knepley   ierr = VecGetDM(cellGeometry, &dmCell);CHKERRQ(ierr);
35222f856554SMatthew G. Knepley   ierr = VecGetArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
35232f856554SMatthew G. Knepley   if (locGrad) {
35242f856554SMatthew G. Knepley     ierr = VecGetDM(locGrad, &dmGrad);CHKERRQ(ierr);
35252f856554SMatthew G. Knepley     ierr = VecGetArrayRead(locGrad, &lgrad);CHKERRQ(ierr);
35262f856554SMatthew G. Knepley   }
35272f856554SMatthew G. Knepley   ierr = DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uL);CHKERRQ(ierr);
35282f856554SMatthew G. Knepley   ierr = DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uR);CHKERRQ(ierr);
35292f856554SMatthew G. Knepley   /* Right now just eat the extra work for FE (could make a cell loop) */
35302f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
35312f856554SMatthew G. Knepley     const PetscInt        *cells;
35322f856554SMatthew G. Knepley     PetscFVFaceGeom       *fg;
35332f856554SMatthew G. Knepley     PetscFVCellGeom       *cgL, *cgR;
35342f856554SMatthew G. Knepley     PetscScalar           *xL, *xR, *gL, *gR;
35352f856554SMatthew G. Knepley     PetscScalar           *uLl = *uL, *uRl = *uR;
35362f856554SMatthew G. Knepley     PetscInt               ghost, nsupp, nchild;
35372f856554SMatthew G. Knepley 
35382f856554SMatthew G. Knepley     ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
35392f856554SMatthew G. Knepley     ierr = DMPlexGetSupportSize(dm, face, &nsupp);CHKERRQ(ierr);
35402f856554SMatthew G. Knepley     ierr = DMPlexGetTreeChildren(dm, face, &nchild, NULL);CHKERRQ(ierr);
35412f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
35422f856554SMatthew G. Knepley     ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
35432f856554SMatthew G. Knepley     ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
35442f856554SMatthew G. Knepley     ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);CHKERRQ(ierr);
35452f856554SMatthew G. Knepley     ierr = DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);CHKERRQ(ierr);
35462f856554SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
35472f856554SMatthew G. Knepley       PetscInt off;
35482f856554SMatthew G. Knepley 
35492f856554SMatthew G. Knepley       ierr = PetscDSGetComponentOffset(prob, f, &off);CHKERRQ(ierr);
35502f856554SMatthew G. Knepley       if (isFE[f]) {
35512f856554SMatthew G. Knepley         const PetscInt *cone;
35522f856554SMatthew G. Knepley         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;
35532f856554SMatthew G. Knepley 
35542f856554SMatthew G. Knepley         xL = xR = NULL;
35552f856554SMatthew G. Knepley         ierr = PetscSectionGetFieldComponents(section, f, &comp);CHKERRQ(ierr);
35562f856554SMatthew G. Knepley         ierr = DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL);CHKERRQ(ierr);
35572f856554SMatthew G. Knepley         ierr = DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR);CHKERRQ(ierr);
35582f856554SMatthew G. Knepley         ierr = DMPlexGetCone(dm, cells[0], &cone);CHKERRQ(ierr);
35592f856554SMatthew G. Knepley         ierr = DMPlexGetConeSize(dm, cells[0], &coneSizeL);CHKERRQ(ierr);
35602f856554SMatthew G. Knepley         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL) if (cone[faceLocL] == face) break;
35612f856554SMatthew G. Knepley         ierr = DMPlexGetCone(dm, cells[1], &cone);CHKERRQ(ierr);
35622f856554SMatthew G. Knepley         ierr = DMPlexGetConeSize(dm, cells[1], &coneSizeR);CHKERRQ(ierr);
35632f856554SMatthew G. Knepley         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR) if (cone[faceLocR] == face) break;
3564ff1e0c32SBarry Smith         if (faceLocL == coneSizeL && faceLocR == coneSizeR) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %D in cone of cell %D or cell %D", face, cells[0], cells[1]);
35652f856554SMatthew G. Knepley         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
35662f856554SMatthew G. Knepley         /* TODO: this is a hack that might not be right for nonconforming */
35672f856554SMatthew G. Knepley         if (faceLocL < coneSizeL) {
3568a8f1f9e5SMatthew G. Knepley           ierr = PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface*Nc+off]);CHKERRQ(ierr);
3569a8f1f9e5SMatthew G. Knepley           if (rdof == ldof && faceLocR < coneSizeR) {ierr = PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface*Nc+off]);CHKERRQ(ierr);}
35702f856554SMatthew G. Knepley           else              {for (d = 0; d < comp; ++d) uRl[iface*Nc+off+d] = uLl[iface*Nc+off+d];}
35712f856554SMatthew G. Knepley         }
35722f856554SMatthew G. Knepley         else {
3573a8f1f9e5SMatthew G. Knepley           ierr = PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface*Nc+off]);CHKERRQ(ierr);
35742f856554SMatthew G. Knepley           ierr = PetscSectionGetFieldComponents(section, f, &comp);CHKERRQ(ierr);
35752f856554SMatthew G. Knepley           for (d = 0; d < comp; ++d) uLl[iface*Nc+off+d] = uRl[iface*Nc+off+d];
35762f856554SMatthew G. Knepley         }
35772f856554SMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL);CHKERRQ(ierr);
35782f856554SMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR);CHKERRQ(ierr);
35792f856554SMatthew G. Knepley       } else {
35802f856554SMatthew G. Knepley         PetscFV  fv;
35812f856554SMatthew G. Knepley         PetscInt numComp, c;
35822f856554SMatthew G. Knepley 
35832f856554SMatthew G. Knepley         ierr = PetscDSGetDiscretization(prob, f, (PetscObject *) &fv);CHKERRQ(ierr);
35842f856554SMatthew G. Knepley         ierr = PetscFVGetNumComponents(fv, &numComp);CHKERRQ(ierr);
35852f856554SMatthew G. Knepley         ierr = DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL);CHKERRQ(ierr);
35862f856554SMatthew G. Knepley         ierr = DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR);CHKERRQ(ierr);
35872f856554SMatthew G. Knepley         if (dmGrad) {
35882f856554SMatthew G. Knepley           PetscReal dxL[3], dxR[3];
35892f856554SMatthew G. Knepley 
35902f856554SMatthew G. Knepley           ierr = DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL);CHKERRQ(ierr);
35912f856554SMatthew G. Knepley           ierr = DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR);CHKERRQ(ierr);
35922f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
35932f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
35942f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
35952f856554SMatthew G. Knepley             uLl[iface*Nc+off+c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c*dim], dxL);
35962f856554SMatthew G. Knepley             uRl[iface*Nc+off+c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c*dim], dxR);
35972f856554SMatthew G. Knepley           }
35982f856554SMatthew G. Knepley         } else {
35992f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
36002f856554SMatthew G. Knepley             uLl[iface*Nc+off+c] = xL[c];
36012f856554SMatthew G. Knepley             uRl[iface*Nc+off+c] = xR[c];
36022f856554SMatthew G. Knepley           }
36032f856554SMatthew G. Knepley         }
36042f856554SMatthew G. Knepley       }
36052f856554SMatthew G. Knepley     }
36062f856554SMatthew G. Knepley     ++iface;
36072f856554SMatthew G. Knepley   }
36082f856554SMatthew G. Knepley   *Nface = iface;
36092f856554SMatthew G. Knepley   ierr = VecRestoreArrayRead(locX, &x);CHKERRQ(ierr);
36102f856554SMatthew G. Knepley   ierr = VecRestoreArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
36112f856554SMatthew G. Knepley   ierr = VecRestoreArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
36122f856554SMatthew G. Knepley   if (locGrad) {
36132f856554SMatthew G. Knepley     ierr = VecRestoreArrayRead(locGrad, &lgrad);CHKERRQ(ierr);
36142f856554SMatthew G. Knepley   }
36152f856554SMatthew G. Knepley   ierr = PetscFree(isFE);CHKERRQ(ierr);
36162f856554SMatthew G. Knepley   PetscFunctionReturn(0);
36172f856554SMatthew G. Knepley }
36182f856554SMatthew G. Knepley 
36192f856554SMatthew G. Knepley /*@C
36202f856554SMatthew G. Knepley   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces
36212f856554SMatthew G. Knepley 
36222f856554SMatthew G. Knepley   Input Parameters:
36232f856554SMatthew G. Knepley + dm     - The DM
36242f856554SMatthew G. Knepley . fStart - The first face to include
36252f856554SMatthew G. Knepley . fEnd   - The first face to exclude
36262f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
36272f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
36282f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
36292f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
36302f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
36312f856554SMatthew G. Knepley 
36322f856554SMatthew G. Knepley   Output Parameters:
36332f856554SMatthew G. Knepley + Nface - The number of faces with field values
36342f856554SMatthew G. Knepley . uL - The field values at the left side of the face
36352f856554SMatthew G. Knepley - uR - The field values at the right side of the face
36362f856554SMatthew G. Knepley 
36372f856554SMatthew G. Knepley   Level: developer
36382f856554SMatthew G. Knepley 
36392f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
36402f856554SMatthew G. Knepley @*/
36412f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
36422f856554SMatthew G. Knepley {
36432f856554SMatthew G. Knepley   PetscErrorCode ierr;
36442f856554SMatthew G. Knepley 
36452f856554SMatthew G. Knepley   PetscFunctionBegin;
36462f856554SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL);CHKERRQ(ierr);
36472f856554SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR);CHKERRQ(ierr);
36482f856554SMatthew G. Knepley   PetscFunctionReturn(0);
36492f856554SMatthew G. Knepley }
36502f856554SMatthew G. Knepley 
36512f856554SMatthew G. Knepley /*@C
36522f856554SMatthew G. Knepley   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces
36532f856554SMatthew G. Knepley 
36542f856554SMatthew G. Knepley   Input Parameters:
36552f856554SMatthew G. Knepley + dm     - The DM
36562f856554SMatthew G. Knepley . fStart - The first face to include
36572f856554SMatthew G. Knepley . fEnd   - The first face to exclude
36582f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
36592f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
36602f856554SMatthew G. Knepley 
36612f856554SMatthew G. Knepley   Output Parameters:
36622f856554SMatthew G. Knepley + Nface - The number of faces with field values
36632f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
36642f856554SMatthew G. Knepley - vol   - The cell volume
36652f856554SMatthew G. Knepley 
36662f856554SMatthew G. Knepley   Level: developer
36672f856554SMatthew G. Knepley 
36682f856554SMatthew G. Knepley .seealso: DMPlexGetCellFields()
36692f856554SMatthew G. Knepley @*/
36702f856554SMatthew G. Knepley PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
36712f856554SMatthew G. Knepley {
36722f856554SMatthew G. Knepley   DM                 dmFace, dmCell;
36732f856554SMatthew G. Knepley   DMLabel            ghostLabel;
36742f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom;
36752f856554SMatthew G. Knepley   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
36762f856554SMatthew G. Knepley   PetscErrorCode     ierr;
36772f856554SMatthew G. Knepley 
36782f856554SMatthew G. Knepley   PetscFunctionBegin;
36792f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
36802f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4);
36812f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5);
3682064a246eSJacob Faibussowitsch   PetscValidPointer(fgeom, 7);
3683064a246eSJacob Faibussowitsch   PetscValidPointer(vol, 8);
36842f856554SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
36852f856554SMatthew G. Knepley   ierr = DMGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
36862f856554SMatthew G. Knepley   ierr = VecGetDM(faceGeometry, &dmFace);CHKERRQ(ierr);
36872f856554SMatthew G. Knepley   ierr = VecGetArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
36882f856554SMatthew G. Knepley   ierr = VecGetDM(cellGeometry, &dmCell);CHKERRQ(ierr);
36892f856554SMatthew G. Knepley   ierr = VecGetArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
36902f856554SMatthew G. Knepley   ierr = PetscMalloc1(numFaces, fgeom);CHKERRQ(ierr);
36912f856554SMatthew G. Knepley   ierr = DMGetWorkArray(dm, numFaces*2, MPIU_SCALAR, vol);CHKERRQ(ierr);
36922f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
36932f856554SMatthew G. Knepley     const PetscInt        *cells;
36942f856554SMatthew G. Knepley     PetscFVFaceGeom       *fg;
36952f856554SMatthew G. Knepley     PetscFVCellGeom       *cgL, *cgR;
36962f856554SMatthew G. Knepley     PetscFVFaceGeom       *fgeoml = *fgeom;
36972f856554SMatthew G. Knepley     PetscReal             *voll   = *vol;
36982f856554SMatthew G. Knepley     PetscInt               ghost, d, nchild, nsupp;
36992f856554SMatthew G. Knepley 
37002f856554SMatthew G. Knepley     ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
37012f856554SMatthew G. Knepley     ierr = DMPlexGetSupportSize(dm, face, &nsupp);CHKERRQ(ierr);
37022f856554SMatthew G. Knepley     ierr = DMPlexGetTreeChildren(dm, face, &nchild, NULL);CHKERRQ(ierr);
37032f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
37042f856554SMatthew G. Knepley     ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
37052f856554SMatthew G. Knepley     ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
37062f856554SMatthew G. Knepley     ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);CHKERRQ(ierr);
37072f856554SMatthew G. Knepley     ierr = DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);CHKERRQ(ierr);
37082f856554SMatthew G. Knepley     for (d = 0; d < dim; ++d) {
37092f856554SMatthew G. Knepley       fgeoml[iface].centroid[d] = fg->centroid[d];
37102f856554SMatthew G. Knepley       fgeoml[iface].normal[d]   = fg->normal[d];
37112f856554SMatthew G. Knepley     }
37122f856554SMatthew G. Knepley     voll[iface*2+0] = cgL->volume;
37132f856554SMatthew G. Knepley     voll[iface*2+1] = cgR->volume;
37142f856554SMatthew G. Knepley     ++iface;
37152f856554SMatthew G. Knepley   }
37162f856554SMatthew G. Knepley   *Nface = iface;
37172f856554SMatthew G. Knepley   ierr = VecRestoreArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
37182f856554SMatthew G. Knepley   ierr = VecRestoreArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
37192f856554SMatthew G. Knepley   PetscFunctionReturn(0);
37202f856554SMatthew G. Knepley }
37212f856554SMatthew G. Knepley 
37222f856554SMatthew G. Knepley /*@C
37232f856554SMatthew G. Knepley   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces
37242f856554SMatthew G. Knepley 
37252f856554SMatthew G. Knepley   Input Parameters:
37262f856554SMatthew G. Knepley + dm     - The DM
37272f856554SMatthew G. Knepley . fStart - The first face to include
37282f856554SMatthew G. Knepley . fEnd   - The first face to exclude
37292f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
37302f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
37312f856554SMatthew G. Knepley 
37322f856554SMatthew G. Knepley   Output Parameters:
37332f856554SMatthew G. Knepley + Nface - The number of faces with field values
37342f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
37352f856554SMatthew G. Knepley - vol   - The cell volume
37362f856554SMatthew G. Knepley 
37372f856554SMatthew G. Knepley   Level: developer
37382f856554SMatthew G. Knepley 
37392f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
37402f856554SMatthew G. Knepley @*/
37412f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
37422f856554SMatthew G. Knepley {
37432f856554SMatthew G. Knepley   PetscErrorCode ierr;
37442f856554SMatthew G. Knepley 
37452f856554SMatthew G. Knepley   PetscFunctionBegin;
37462f856554SMatthew G. Knepley   ierr = PetscFree(*fgeom);CHKERRQ(ierr);
37472f856554SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, 0, MPIU_REAL, vol);CHKERRQ(ierr);
37482f856554SMatthew G. Knepley   PetscFunctionReturn(0);
37492f856554SMatthew G. Knepley }
37502f856554SMatthew G. Knepley 
3751a1cf66bbSMatthew G. Knepley PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
3752a1cf66bbSMatthew G. Knepley {
3753a1cf66bbSMatthew G. Knepley   char            composeStr[33] = {0};
3754a1cf66bbSMatthew G. Knepley   PetscObjectId   id;
3755a1cf66bbSMatthew G. Knepley   PetscContainer  container;
3756a1cf66bbSMatthew G. Knepley   PetscErrorCode  ierr;
3757a1cf66bbSMatthew G. Knepley 
3758a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
3759a1cf66bbSMatthew G. Knepley   ierr = PetscObjectGetId((PetscObject)quad,&id);CHKERRQ(ierr);
3760a1cf66bbSMatthew G. Knepley   ierr = PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%x\n", id);CHKERRQ(ierr);
3761a1cf66bbSMatthew G. Knepley   ierr = PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container);CHKERRQ(ierr);
3762a1cf66bbSMatthew G. Knepley   if (container) {
3763a1cf66bbSMatthew G. Knepley     ierr = PetscContainerGetPointer(container, (void **) geom);CHKERRQ(ierr);
3764a1cf66bbSMatthew G. Knepley   } else {
3765a1cf66bbSMatthew G. Knepley     ierr = DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom);CHKERRQ(ierr);
3766a1cf66bbSMatthew G. Knepley     ierr = PetscContainerCreate(PETSC_COMM_SELF,&container);CHKERRQ(ierr);
3767a1cf66bbSMatthew G. Knepley     ierr = PetscContainerSetPointer(container, (void *) *geom);CHKERRQ(ierr);
3768a1cf66bbSMatthew G. Knepley     ierr = PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom);CHKERRQ(ierr);
3769a1cf66bbSMatthew G. Knepley     ierr = PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container);CHKERRQ(ierr);
3770a1cf66bbSMatthew G. Knepley     ierr = PetscContainerDestroy(&container);CHKERRQ(ierr);
3771a1cf66bbSMatthew G. Knepley   }
3772a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3773a1cf66bbSMatthew G. Knepley }
3774a1cf66bbSMatthew G. Knepley 
3775a1cf66bbSMatthew G. Knepley PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
3776a1cf66bbSMatthew G. Knepley {
3777a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
3778a1cf66bbSMatthew G. Knepley   *geom = NULL;
3779a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3780a1cf66bbSMatthew G. Knepley }
3781a1cf66bbSMatthew G. Knepley 
378292d50984SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user)
378392d50984SMatthew G. Knepley {
378492d50984SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
378592d50984SMatthew G. Knepley   const char      *name       = "Residual";
378692d50984SMatthew G. Knepley   DM               dmAux      = NULL;
378792d50984SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
378892d50984SMatthew G. Knepley   PetscDS          prob       = NULL;
378992d50984SMatthew G. Knepley   PetscDS          probAux    = NULL;
379092d50984SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
379192d50984SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
379292d50984SMatthew G. Knepley   DMField          coordField = NULL;
3793c0006e53SPatrick Farrell   Vec              locA;
3794c0006e53SPatrick Farrell   PetscScalar     *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
379592d50984SMatthew G. Knepley   IS               chunkIS;
379692d50984SMatthew G. Knepley   const PetscInt  *cells;
379792d50984SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
3798364207b6SKarl Rupp   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
379992d50984SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
380006ad1575SMatthew G. Knepley   PetscFormKey key;
380192d50984SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
380292d50984SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
380392d50984SMatthew G. Knepley   PetscErrorCode   ierr;
380492d50984SMatthew G. Knepley 
380592d50984SMatthew G. Knepley   PetscFunctionBegin;
380692d50984SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0);CHKERRQ(ierr);
380792d50984SMatthew G. Knepley   /* FEM+FVM */
380892d50984SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
380992d50984SMatthew G. Knepley   ierr = DMGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
381092d50984SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
381192d50984SMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
381292d50984SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
38139a2a23afSMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, NULL, 0, &locA);CHKERRQ(ierr);
381492d50984SMatthew G. Knepley   if (locA) {
381592d50984SMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
381692d50984SMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
381792d50984SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
381892d50984SMatthew G. Knepley   }
381992d50984SMatthew G. Knepley   /* 2: Get geometric data */
382092d50984SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
382192d50984SMatthew G. Knepley     PetscObject  obj;
382292d50984SMatthew G. Knepley     PetscClassId id;
382392d50984SMatthew G. Knepley     PetscBool    fimp;
382492d50984SMatthew G. Knepley 
382592d50984SMatthew G. Knepley     ierr = PetscDSGetImplicit(prob, f, &fimp);CHKERRQ(ierr);
382692d50984SMatthew G. Knepley     if (isImplicit != fimp) continue;
382792d50984SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
382892d50984SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
382992d50984SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
3830364207b6SKarl Rupp     if (id == PETSCFV_CLASSID) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Use of FVM with PCPATCH not yet implemented");
383192d50984SMatthew G. Knepley   }
383292d50984SMatthew G. Knepley   if (useFEM) {
383392d50984SMatthew G. Knepley     ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
383492d50984SMatthew G. Knepley     ierr = DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);CHKERRQ(ierr);
383592d50984SMatthew G. Knepley     if (maxDegree <= 1) {
383692d50984SMatthew G. Knepley       ierr = DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad);CHKERRQ(ierr);
383792d50984SMatthew G. Knepley       if (affineQuad) {
383892d50984SMatthew G. Knepley         ierr = DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);CHKERRQ(ierr);
383992d50984SMatthew G. Knepley       }
384092d50984SMatthew G. Knepley     } else {
384192d50984SMatthew G. Knepley       ierr = PetscCalloc2(Nf,&quads,Nf,&geoms);CHKERRQ(ierr);
384292d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
384392d50984SMatthew G. Knepley         PetscObject  obj;
384492d50984SMatthew G. Knepley         PetscClassId id;
384592d50984SMatthew G. Knepley         PetscBool    fimp;
384692d50984SMatthew G. Knepley 
384792d50984SMatthew G. Knepley         ierr = PetscDSGetImplicit(prob, f, &fimp);CHKERRQ(ierr);
384892d50984SMatthew G. Knepley         if (isImplicit != fimp) continue;
384992d50984SMatthew G. Knepley         ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
385092d50984SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
385192d50984SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
385292d50984SMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
385392d50984SMatthew G. Knepley 
385492d50984SMatthew G. Knepley           ierr = PetscFEGetQuadrature(fe, &quads[f]);CHKERRQ(ierr);
385592d50984SMatthew G. Knepley           ierr = PetscObjectReference((PetscObject)quads[f]);CHKERRQ(ierr);
385692d50984SMatthew G. Knepley           ierr = DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);CHKERRQ(ierr);
385792d50984SMatthew G. Knepley         }
385892d50984SMatthew G. Knepley       }
385992d50984SMatthew G. Knepley     }
386092d50984SMatthew G. Knepley   }
386192d50984SMatthew G. Knepley   /* Loop over chunks */
386292d50984SMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
386392d50984SMatthew G. Knepley   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
386492d50984SMatthew G. Knepley   if (useFEM) {ierr = ISCreate(PETSC_COMM_SELF, &chunkIS);CHKERRQ(ierr);}
386592d50984SMatthew G. Knepley   numCells      = cEnd - cStart;
386692d50984SMatthew G. Knepley   numChunks     = 1;
386792d50984SMatthew G. Knepley   cellChunkSize = numCells/numChunks;
386892d50984SMatthew G. Knepley   numChunks     = PetscMin(1,numCells);
38696528b96dSMatthew G. Knepley   key.label     = NULL;
38706528b96dSMatthew G. Knepley   key.value     = 0;
387106ad1575SMatthew G. Knepley   key.part      = 0;
387292d50984SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
3873c0006e53SPatrick Farrell     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
3874c0006e53SPatrick Farrell     PetscReal       *vol = NULL;
3875c0006e53SPatrick Farrell     PetscFVFaceGeom *fgeom = NULL;
387692d50984SMatthew G. Knepley     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
3877c0006e53SPatrick Farrell     PetscInt         numFaces = 0;
387892d50984SMatthew G. Knepley 
387992d50984SMatthew G. Knepley     /* Extract field coefficients */
388092d50984SMatthew G. Knepley     if (useFEM) {
388192d50984SMatthew G. Knepley       ierr = ISGetPointSubrange(chunkIS, cS, cE, cells);CHKERRQ(ierr);
388292d50984SMatthew G. Knepley       ierr = DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);CHKERRQ(ierr);
388392d50984SMatthew G. Knepley       ierr = DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);CHKERRQ(ierr);
3884580bdb30SBarry Smith       ierr = PetscArrayzero(elemVec, numCells*totDim);CHKERRQ(ierr);
388592d50984SMatthew G. Knepley     }
388692d50984SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
388792d50984SMatthew G. Knepley     /* Loop over fields */
388892d50984SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
388992d50984SMatthew G. Knepley       PetscObject  obj;
389092d50984SMatthew G. Knepley       PetscClassId id;
389192d50984SMatthew G. Knepley       PetscBool    fimp;
389292d50984SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
389392d50984SMatthew G. Knepley 
38946528b96dSMatthew G. Knepley       key.field = f;
389592d50984SMatthew G. Knepley       ierr = PetscDSGetImplicit(prob, f, &fimp);CHKERRQ(ierr);
389692d50984SMatthew G. Knepley       if (isImplicit != fimp) continue;
389792d50984SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
389892d50984SMatthew G. Knepley       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
389992d50984SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
390092d50984SMatthew G. Knepley         PetscFE         fe = (PetscFE) obj;
390192d50984SMatthew G. Knepley         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
390292d50984SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
390392d50984SMatthew G. Knepley         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
390492d50984SMatthew G. Knepley         PetscInt        Nq, Nb;
390592d50984SMatthew G. Knepley 
390692d50984SMatthew G. Knepley         ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
390792d50984SMatthew G. Knepley         ierr = PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
390892d50984SMatthew G. Knepley         ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
390992d50984SMatthew G. Knepley         blockSize = Nb;
391092d50984SMatthew G. Knepley         batchSize = numBlocks * blockSize;
391192d50984SMatthew G. Knepley         ierr      = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
391292d50984SMatthew G. Knepley         numChunks = numCells / (numBatches*batchSize);
391392d50984SMatthew G. Knepley         Ne        = numChunks*numBatches*batchSize;
391492d50984SMatthew G. Knepley         Nr        = numCells % (numBatches*batchSize);
391592d50984SMatthew G. Knepley         offset    = numCells - Nr;
391692d50984SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
391792d50984SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
391892d50984SMatthew G. Knepley         ierr = PetscFEGeomGetChunk(geom,0,offset,&chunkGeom);CHKERRQ(ierr);
39196528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec);CHKERRQ(ierr);
392092d50984SMatthew G. Knepley         ierr = PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom);CHKERRQ(ierr);
39216528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]);CHKERRQ(ierr);
392292d50984SMatthew G. Knepley         ierr = PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom);CHKERRQ(ierr);
392392d50984SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
392492d50984SMatthew G. Knepley         PetscFV fv = (PetscFV) obj;
392592d50984SMatthew G. Knepley 
392692d50984SMatthew G. Knepley         Ne = numFaces;
392792d50984SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
392892d50984SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
392992d50984SMatthew G. Knepley         ierr = PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR);CHKERRQ(ierr);
3930ff1e0c32SBarry Smith       } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", f);
393192d50984SMatthew G. Knepley     }
393292d50984SMatthew G. Knepley     /* Loop over domain */
393392d50984SMatthew G. Knepley     if (useFEM) {
393492d50984SMatthew G. Knepley       /* Add elemVec to locX */
393592d50984SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
393692d50984SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
393792d50984SMatthew G. Knepley         const PetscInt cind = c - cStart;
393892d50984SMatthew G. Knepley 
393992d50984SMatthew G. Knepley         if (mesh->printFEM > 1) {ierr = DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]);CHKERRQ(ierr);}
394092d50984SMatthew G. Knepley         if (ghostLabel) {
394192d50984SMatthew G. Knepley           PetscInt ghostVal;
394292d50984SMatthew G. Knepley 
394392d50984SMatthew G. Knepley           ierr = DMLabelGetValue(ghostLabel,cell,&ghostVal);CHKERRQ(ierr);
394492d50984SMatthew G. Knepley           if (ghostVal > 0) continue;
394592d50984SMatthew G. Knepley         }
394692d50984SMatthew G. Knepley         ierr = DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES);CHKERRQ(ierr);
394792d50984SMatthew G. Knepley       }
394892d50984SMatthew G. Knepley     }
394992d50984SMatthew G. Knepley     /* Handle time derivative */
395092d50984SMatthew G. Knepley     if (locX_t) {
395192d50984SMatthew G. Knepley       PetscScalar *x_t, *fa;
395292d50984SMatthew G. Knepley 
395392d50984SMatthew G. Knepley       ierr = VecGetArray(locF, &fa);CHKERRQ(ierr);
395492d50984SMatthew G. Knepley       ierr = VecGetArray(locX_t, &x_t);CHKERRQ(ierr);
395592d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
395692d50984SMatthew G. Knepley         PetscFV      fv;
395792d50984SMatthew G. Knepley         PetscObject  obj;
395892d50984SMatthew G. Knepley         PetscClassId id;
395992d50984SMatthew G. Knepley         PetscInt     pdim, d;
396092d50984SMatthew G. Knepley 
396192d50984SMatthew G. Knepley         ierr = PetscDSGetDiscretization(prob, f, &obj);CHKERRQ(ierr);
396292d50984SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
396392d50984SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
396492d50984SMatthew G. Knepley         fv   = (PetscFV) obj;
396592d50984SMatthew G. Knepley         ierr = PetscFVGetNumComponents(fv, &pdim);CHKERRQ(ierr);
396692d50984SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
396792d50984SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
396892d50984SMatthew G. Knepley           PetscScalar   *u_t, *r;
396992d50984SMatthew G. Knepley 
397092d50984SMatthew G. Knepley           if (ghostLabel) {
397192d50984SMatthew G. Knepley             PetscInt ghostVal;
397292d50984SMatthew G. Knepley 
397392d50984SMatthew G. Knepley             ierr = DMLabelGetValue(ghostLabel, cell, &ghostVal);CHKERRQ(ierr);
397492d50984SMatthew G. Knepley             if (ghostVal > 0) continue;
397592d50984SMatthew G. Knepley           }
397692d50984SMatthew G. Knepley           ierr = DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t);CHKERRQ(ierr);
397792d50984SMatthew G. Knepley           ierr = DMPlexPointLocalFieldRef(dm, cell, f, fa, &r);CHKERRQ(ierr);
397892d50984SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
397992d50984SMatthew G. Knepley         }
398092d50984SMatthew G. Knepley       }
398192d50984SMatthew G. Knepley       ierr = VecRestoreArray(locX_t, &x_t);CHKERRQ(ierr);
398292d50984SMatthew G. Knepley       ierr = VecRestoreArray(locF, &fa);CHKERRQ(ierr);
398392d50984SMatthew G. Knepley     }
398492d50984SMatthew G. Knepley     if (useFEM) {
398592d50984SMatthew G. Knepley       ierr = DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);CHKERRQ(ierr);
398692d50984SMatthew G. Knepley       ierr = DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);CHKERRQ(ierr);
398792d50984SMatthew G. Knepley     }
398892d50984SMatthew G. Knepley   }
398992d50984SMatthew G. Knepley   if (useFEM) {ierr = ISDestroy(&chunkIS);CHKERRQ(ierr);}
399092d50984SMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
399192d50984SMatthew G. Knepley   /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */
399292d50984SMatthew G. Knepley   if (useFEM) {
399392d50984SMatthew G. Knepley     if (maxDegree <= 1) {
399492d50984SMatthew G. Knepley       ierr = DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);CHKERRQ(ierr);
399592d50984SMatthew G. Knepley       ierr = PetscQuadratureDestroy(&affineQuad);CHKERRQ(ierr);
399692d50984SMatthew G. Knepley     } else {
399792d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
399892d50984SMatthew G. Knepley         ierr = DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);CHKERRQ(ierr);
399992d50984SMatthew G. Knepley         ierr = PetscQuadratureDestroy(&quads[f]);CHKERRQ(ierr);
400092d50984SMatthew G. Knepley       }
400192d50984SMatthew G. Knepley       ierr = PetscFree2(quads,geoms);CHKERRQ(ierr);
400292d50984SMatthew G. Knepley     }
400392d50984SMatthew G. Knepley   }
400492d50984SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0);CHKERRQ(ierr);
400592d50984SMatthew G. Knepley   PetscFunctionReturn(0);
400692d50984SMatthew G. Knepley }
400792d50984SMatthew G. Knepley 
4008a1cf66bbSMatthew G. Knepley /*
4009a1cf66bbSMatthew G. Knepley   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac
4010a1cf66bbSMatthew G. Knepley 
4011a1cf66bbSMatthew G. Knepley   X   - The local solution vector
4012a1cf66bbSMatthew G. Knepley   X_t - The local solution time derviative vector, or NULL
4013a1cf66bbSMatthew G. Knepley */
4014a1cf66bbSMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS,
4015a1cf66bbSMatthew G. Knepley                                                     PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx)
4016a1cf66bbSMatthew G. Knepley {
4017a1cf66bbSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *) dm->data;
4018a1cf66bbSMatthew G. Knepley   const char      *name = "Jacobian", *nameP = "JacobianPre";
4019a1cf66bbSMatthew G. Knepley   DM               dmAux = NULL;
4020a1cf66bbSMatthew G. Knepley   PetscDS          prob,   probAux = NULL;
4021a1cf66bbSMatthew G. Knepley   PetscSection     sectionAux = NULL;
4022a1cf66bbSMatthew G. Knepley   Vec              A;
4023a1cf66bbSMatthew G. Knepley   DMField          coordField;
4024a1cf66bbSMatthew G. Knepley   PetscFEGeom     *cgeomFEM;
4025a1cf66bbSMatthew G. Knepley   PetscQuadrature  qGeom = NULL;
4026a1cf66bbSMatthew G. Knepley   Mat              J = Jac, JP = JacP;
4027a1cf66bbSMatthew G. Knepley   PetscScalar     *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
40289b2fc754SMatthew G. Knepley   PetscBool        hasJac, hasPrec, hasDyn, assembleJac, isMatIS, isMatISP, *isFE, hasFV = PETSC_FALSE;
4029a1cf66bbSMatthew G. Knepley   const PetscInt  *cells;
403006ad1575SMatthew G. Knepley   PetscFormKey key;
40319b2fc754SMatthew G. Knepley   PetscInt         Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
4032a1cf66bbSMatthew G. Knepley   PetscErrorCode   ierr;
4033a1cf66bbSMatthew G. Knepley 
4034a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
4035a1cf66bbSMatthew G. Knepley   ierr = ISGetLocalSize(cellIS, &numCells);CHKERRQ(ierr);
4036a1cf66bbSMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
4037a1cf66bbSMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
4038a1cf66bbSMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
40399a2a23afSMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, NULL, 0, &A);CHKERRQ(ierr);
40409a2a23afSMatthew G. Knepley   if (A) {
40419a2a23afSMatthew G. Knepley     ierr = VecGetDM(A, &dmAux);CHKERRQ(ierr);
404292fd8e1eSJed Brown     ierr = DMGetLocalSection(dmAux, &sectionAux);CHKERRQ(ierr);
4043a1cf66bbSMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
4044a1cf66bbSMatthew G. Knepley   }
4045a1cf66bbSMatthew G. Knepley   /* Get flags */
4046a1cf66bbSMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
4047a1cf66bbSMatthew G. Knepley   ierr = DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE);CHKERRQ(ierr);
4048a1cf66bbSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
4049a1cf66bbSMatthew G. Knepley     PetscObject  disc;
4050a1cf66bbSMatthew G. Knepley     PetscClassId id;
4051a1cf66bbSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, fieldI, &disc);CHKERRQ(ierr);
4052a1cf66bbSMatthew G. Knepley     ierr = PetscObjectGetClassId(disc, &id);CHKERRQ(ierr);
4053a1cf66bbSMatthew G. Knepley     if (id == PETSCFE_CLASSID)      {isFE[fieldI] = PETSC_TRUE;}
4054a1cf66bbSMatthew G. Knepley     else if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; isFE[fieldI] = PETSC_FALSE;}
4055a1cf66bbSMatthew G. Knepley   }
4056a1cf66bbSMatthew G. Knepley   ierr = PetscDSHasJacobian(prob, &hasJac);CHKERRQ(ierr);
4057a1cf66bbSMatthew G. Knepley   ierr = PetscDSHasJacobianPreconditioner(prob, &hasPrec);CHKERRQ(ierr);
4058a1cf66bbSMatthew G. Knepley   ierr = PetscDSHasDynamicJacobian(prob, &hasDyn);CHKERRQ(ierr);
4059a1cf66bbSMatthew G. Knepley   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
4060a1cf66bbSMatthew G. Knepley   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
4061a1cf66bbSMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) Jac,  MATIS, &isMatIS);CHKERRQ(ierr);
4062a1cf66bbSMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);CHKERRQ(ierr);
4063a1cf66bbSMatthew G. Knepley   /* Setup input data and temp arrays (should be DMGetWorkArray) */
4064a1cf66bbSMatthew G. Knepley   if (isMatISP || isMatISP) {ierr = DMPlexGetSubdomainSection(dm, &globalSection);CHKERRQ(ierr);}
4065a1cf66bbSMatthew G. Knepley   if (isMatIS)  {ierr = MatISGetLocalMat(Jac,  &J);CHKERRQ(ierr);}
4066a1cf66bbSMatthew G. Knepley   if (isMatISP) {ierr = MatISGetLocalMat(JacP, &JP);CHKERRQ(ierr);}
4067a1cf66bbSMatthew G. Knepley   if (hasFV)    {ierr = MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE);CHKERRQ(ierr);} /* No allocated space for FV stuff, so ignore the zero entries */
4068a1cf66bbSMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
4069a1cf66bbSMatthew G. Knepley   if (probAux) {ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);}
4070a1cf66bbSMatthew G. Knepley   /* Compute batch sizes */
4071a1cf66bbSMatthew G. Knepley   if (isFE[0]) {
4072a1cf66bbSMatthew G. Knepley     PetscFE         fe;
4073a1cf66bbSMatthew G. Knepley     PetscQuadrature q;
4074a1cf66bbSMatthew G. Knepley     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;
4075a1cf66bbSMatthew G. Knepley 
4076a1cf66bbSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe);CHKERRQ(ierr);
4077a1cf66bbSMatthew G. Knepley     ierr = PetscFEGetQuadrature(fe, &q);CHKERRQ(ierr);
4078a1cf66bbSMatthew G. Knepley     ierr = PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL);CHKERRQ(ierr);
4079a1cf66bbSMatthew G. Knepley     ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
4080a1cf66bbSMatthew G. Knepley     ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
4081a1cf66bbSMatthew G. Knepley     blockSize = Nb*numQuadPoints;
4082a1cf66bbSMatthew G. Knepley     batchSize = numBlocks  * blockSize;
4083a1cf66bbSMatthew G. Knepley     chunkSize = numBatches * batchSize;
4084a1cf66bbSMatthew G. Knepley     numChunks = numCells / chunkSize + numCells % chunkSize;
4085a1cf66bbSMatthew G. Knepley     ierr = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
4086a1cf66bbSMatthew G. Knepley   } else {
4087a1cf66bbSMatthew G. Knepley     chunkSize = numCells;
4088a1cf66bbSMatthew G. Knepley     numChunks = 1;
4089a1cf66bbSMatthew G. Knepley   }
4090a1cf66bbSMatthew G. Knepley   /* Get work space */
4091a1cf66bbSMatthew G. Knepley   wsz  = (((X?1:0) + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize;
4092a1cf66bbSMatthew G. Knepley   ierr = DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work);CHKERRQ(ierr);
4093580bdb30SBarry Smith   ierr = PetscArrayzero(work, wsz);CHKERRQ(ierr);
4094a1cf66bbSMatthew G. Knepley   off      = 0;
4095a1cf66bbSMatthew G. Knepley   u        = X       ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
4096a1cf66bbSMatthew G. Knepley   u_t      = X_t     ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
4097a1cf66bbSMatthew G. Knepley   a        = dmAux   ? (sz = chunkSize*totDimAux,     off += sz, work+off-sz) : NULL;
4098a1cf66bbSMatthew G. Knepley   elemMat  = hasJac  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4099a1cf66bbSMatthew G. Knepley   elemMatP = hasPrec ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4100a1cf66bbSMatthew G. Knepley   elemMatD = hasDyn  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4101a1cf66bbSMatthew G. Knepley   if (off != wsz) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %D should be %D", off, wsz);
4102a1cf66bbSMatthew G. Knepley   /* Setup geometry */
4103a1cf66bbSMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
41049b2fc754SMatthew G. Knepley   ierr = DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree);CHKERRQ(ierr);
41059b2fc754SMatthew G. Knepley   if (maxDegree <= 1) {ierr = DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom);CHKERRQ(ierr);}
4106a1cf66bbSMatthew G. Knepley   if (!qGeom) {
4107a1cf66bbSMatthew G. Knepley     PetscFE fe;
4108a1cf66bbSMatthew G. Knepley 
4109a1cf66bbSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe);CHKERRQ(ierr);
4110a1cf66bbSMatthew G. Knepley     ierr = PetscFEGetQuadrature(fe, &qGeom);CHKERRQ(ierr);
4111a1cf66bbSMatthew G. Knepley     ierr = PetscObjectReference((PetscObject) qGeom);CHKERRQ(ierr);
4112a1cf66bbSMatthew G. Knepley   }
4113a1cf66bbSMatthew G. Knepley   ierr = DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM);CHKERRQ(ierr);
4114a1cf66bbSMatthew G. Knepley   /* Compute volume integrals */
4115a1cf66bbSMatthew G. Knepley   if (assembleJac) {ierr = MatZeroEntries(J);CHKERRQ(ierr);}
4116a1cf66bbSMatthew G. Knepley   ierr = MatZeroEntries(JP);CHKERRQ(ierr);
41176528b96dSMatthew G. Knepley   key.label = NULL;
41186528b96dSMatthew G. Knepley   key.value = 0;
411906ad1575SMatthew G. Knepley   key.part  = 0;
4120a1cf66bbSMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
4121a1cf66bbSMatthew G. Knepley     const PetscInt   Ncell = PetscMin(chunkSize, numCells - offCell);
4122a1cf66bbSMatthew G. Knepley     PetscInt         c;
4123a1cf66bbSMatthew G. Knepley 
4124a1cf66bbSMatthew G. Knepley     /* Extract values */
4125a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4126a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
4127a1cf66bbSMatthew G. Knepley       PetscScalar   *x = NULL,  *x_t = NULL;
4128a1cf66bbSMatthew G. Knepley       PetscInt       i;
4129a1cf66bbSMatthew G. Knepley 
4130a1cf66bbSMatthew G. Knepley       if (X) {
4131a1cf66bbSMatthew G. Knepley         ierr = DMPlexVecGetClosure(dm, section, X, cell, NULL, &x);CHKERRQ(ierr);
4132a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
4133a1cf66bbSMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x);CHKERRQ(ierr);
4134a1cf66bbSMatthew G. Knepley       }
4135a1cf66bbSMatthew G. Knepley       if (X_t) {
4136a1cf66bbSMatthew G. Knepley         ierr = DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t);CHKERRQ(ierr);
4137a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[c*totDim+i] = x_t[i];
4138a1cf66bbSMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t);CHKERRQ(ierr);
4139a1cf66bbSMatthew G. Knepley       }
4140a1cf66bbSMatthew G. Knepley       if (dmAux) {
4141a1cf66bbSMatthew G. Knepley         ierr = DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x);CHKERRQ(ierr);
4142a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
4143a1cf66bbSMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x);CHKERRQ(ierr);
4144a1cf66bbSMatthew G. Knepley       }
4145a1cf66bbSMatthew G. Knepley     }
4146a1cf66bbSMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
4147a1cf66bbSMatthew G. Knepley       PetscFE fe;
4148a1cf66bbSMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);CHKERRQ(ierr);
4149a1cf66bbSMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
41506528b96dSMatthew G. Knepley         key.field = fieldI*Nf + fieldJ;
41516528b96dSMatthew G. Knepley         if (hasJac)  {ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN,     key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat);CHKERRQ(ierr);}
41526528b96dSMatthew G. Knepley         if (hasPrec) {ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP);CHKERRQ(ierr);}
41536528b96dSMatthew G. Knepley         if (hasDyn)  {ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD);CHKERRQ(ierr);}
4154a1cf66bbSMatthew G. Knepley       }
4155a1cf66bbSMatthew G. Knepley       /* For finite volume, add the identity */
4156a1cf66bbSMatthew G. Knepley       if (!isFE[fieldI]) {
4157a1cf66bbSMatthew G. Knepley         PetscFV  fv;
4158a1cf66bbSMatthew G. Knepley         PetscInt eOffset = 0, Nc, fc, foff;
4159a1cf66bbSMatthew G. Knepley 
4160a1cf66bbSMatthew G. Knepley         ierr = PetscDSGetFieldOffset(prob, fieldI, &foff);CHKERRQ(ierr);
4161a1cf66bbSMatthew G. Knepley         ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv);CHKERRQ(ierr);
4162a1cf66bbSMatthew G. Knepley         ierr = PetscFVGetNumComponents(fv, &Nc);CHKERRQ(ierr);
4163a1cf66bbSMatthew G. Knepley         for (c = 0; c < chunkSize; ++c, eOffset += totDim*totDim) {
4164a1cf66bbSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
4165a1cf66bbSMatthew G. Knepley             const PetscInt i = foff + fc;
4166a1cf66bbSMatthew G. Knepley             if (hasJac)  {elemMat [eOffset+i*totDim+i] = 1.0;}
4167a1cf66bbSMatthew G. Knepley             if (hasPrec) {elemMatP[eOffset+i*totDim+i] = 1.0;}
4168a1cf66bbSMatthew G. Knepley           }
4169a1cf66bbSMatthew G. Knepley         }
4170a1cf66bbSMatthew G. Knepley       }
4171a1cf66bbSMatthew G. Knepley     }
4172a1cf66bbSMatthew G. Knepley     /*   Add contribution from X_t */
4173a1cf66bbSMatthew G. Knepley     if (hasDyn) {for (c = 0; c < chunkSize*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
4174a1cf66bbSMatthew G. Knepley     /* Insert values into matrix */
4175a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4176a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
4177a1cf66bbSMatthew G. Knepley       if (mesh->printFEM > 1) {
4178a1cf66bbSMatthew G. Knepley         if (hasJac)  {ierr = DMPrintCellMatrix(cell, name,  totDim, totDim, &elemMat[(c-cStart)*totDim*totDim]);CHKERRQ(ierr);}
4179a1cf66bbSMatthew G. Knepley         if (hasPrec) {ierr = DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c-cStart)*totDim*totDim]);CHKERRQ(ierr);}
4180a1cf66bbSMatthew G. Knepley       }
4181a1cf66bbSMatthew G. Knepley       if (assembleJac) {ierr = DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);}
4182a1cf66bbSMatthew G. Knepley       ierr = DMPlexMatSetClosure(dm, section, globalSection, JP, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
4183a1cf66bbSMatthew G. Knepley     }
4184a1cf66bbSMatthew G. Knepley   }
4185a1cf66bbSMatthew G. Knepley   /* Cleanup */
4186a1cf66bbSMatthew G. Knepley   ierr = DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM);CHKERRQ(ierr);
4187a1cf66bbSMatthew G. Knepley   ierr = PetscQuadratureDestroy(&qGeom);CHKERRQ(ierr);
4188a1cf66bbSMatthew G. Knepley   if (hasFV) {ierr = MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE);CHKERRQ(ierr);}
4189a1cf66bbSMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE);CHKERRQ(ierr);
4190a1cf66bbSMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, ((1 + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize, MPIU_SCALAR, &work);CHKERRQ(ierr);
4191a1cf66bbSMatthew G. Knepley   /* Compute boundary integrals */
4192a1cf66bbSMatthew G. Knepley   /* ierr = DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx);CHKERRQ(ierr); */
4193a1cf66bbSMatthew G. Knepley   /* Assemble matrix */
4194a1cf66bbSMatthew G. Knepley   if (assembleJac) {ierr = MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);ierr = MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);}
4195a1cf66bbSMatthew G. Knepley   ierr = MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);ierr = MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
4196a1cf66bbSMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
4197a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
4198a1cf66bbSMatthew G. Knepley }
41993e9753d6SMatthew G. Knepley 
42003e9753d6SMatthew G. Knepley /******** FEM Assembly Function ********/
42013e9753d6SMatthew G. Knepley 
42023e9753d6SMatthew G. Knepley static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy)
42033e9753d6SMatthew G. Knepley {
42043e9753d6SMatthew G. Knepley   PetscBool      isPlex;
42053e9753d6SMatthew G. Knepley   PetscErrorCode ierr;
42063e9753d6SMatthew G. Knepley 
42073e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42083e9753d6SMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex);CHKERRQ(ierr);
42093e9753d6SMatthew G. Knepley   if (isPlex) {
42103e9753d6SMatthew G. Knepley     *plex = dm;
42113e9753d6SMatthew G. Knepley     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr);
42123e9753d6SMatthew G. Knepley   } else {
42133e9753d6SMatthew G. Knepley     ierr = PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex);CHKERRQ(ierr);
42143e9753d6SMatthew G. Knepley     if (!*plex) {
42153e9753d6SMatthew G. Knepley       ierr = DMConvert(dm,DMPLEX,plex);CHKERRQ(ierr);
42163e9753d6SMatthew G. Knepley       ierr = PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex);CHKERRQ(ierr);
42173e9753d6SMatthew G. Knepley       if (copy) {
42189a2a23afSMatthew G. Knepley         ierr = DMCopyAuxiliaryVec(dm, *plex);CHKERRQ(ierr);
42193e9753d6SMatthew G. Knepley       }
42203e9753d6SMatthew G. Knepley     } else {
42213e9753d6SMatthew G. Knepley       ierr = PetscObjectReference((PetscObject) *plex);CHKERRQ(ierr);
42223e9753d6SMatthew G. Knepley     }
42233e9753d6SMatthew G. Knepley   }
42243e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
42253e9753d6SMatthew G. Knepley }
42263e9753d6SMatthew G. Knepley 
42273e9753d6SMatthew G. Knepley /*@
42283e9753d6SMatthew G. Knepley   DMPlexGetGeometryFVM - Return precomputed geometric data
42293e9753d6SMatthew G. Knepley 
42303e9753d6SMatthew G. Knepley   Collective on DM
42313e9753d6SMatthew G. Knepley 
42323e9753d6SMatthew G. Knepley   Input Parameter:
42333e9753d6SMatthew G. Knepley . dm - The DM
42343e9753d6SMatthew G. Knepley 
42353e9753d6SMatthew G. Knepley   Output Parameters:
42363e9753d6SMatthew G. Knepley + facegeom - The values precomputed from face geometry
42373e9753d6SMatthew G. Knepley . cellgeom - The values precomputed from cell geometry
42383e9753d6SMatthew G. Knepley - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell
42393e9753d6SMatthew G. Knepley 
42403e9753d6SMatthew G. Knepley   Level: developer
42413e9753d6SMatthew G. Knepley 
4242446c23c1SPierre Jolivet .seealso: DMTSSetRHSFunctionLocal()
42433e9753d6SMatthew G. Knepley @*/
42443e9753d6SMatthew G. Knepley PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius)
42453e9753d6SMatthew G. Knepley {
42463e9753d6SMatthew G. Knepley   DM             plex;
42473e9753d6SMatthew G. Knepley   PetscErrorCode ierr;
42483e9753d6SMatthew G. Knepley 
42493e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42503e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
42513e9753d6SMatthew G. Knepley   ierr = DMConvertPlex_Internal(dm,&plex,PETSC_TRUE);CHKERRQ(ierr);
42523e9753d6SMatthew G. Knepley   ierr = DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL);CHKERRQ(ierr);
42533e9753d6SMatthew G. Knepley   if (minRadius) {ierr = DMPlexGetMinRadius(plex, minRadius);CHKERRQ(ierr);}
42543e9753d6SMatthew G. Knepley   ierr = DMDestroy(&plex);CHKERRQ(ierr);
42553e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
42563e9753d6SMatthew G. Knepley }
42573e9753d6SMatthew G. Knepley 
42583e9753d6SMatthew G. Knepley /*@
42593e9753d6SMatthew G. Knepley   DMPlexGetGradientDM - Return gradient data layout
42603e9753d6SMatthew G. Knepley 
42613e9753d6SMatthew G. Knepley   Collective on DM
42623e9753d6SMatthew G. Knepley 
42633e9753d6SMatthew G. Knepley   Input Parameters:
42643e9753d6SMatthew G. Knepley + dm - The DM
42653e9753d6SMatthew G. Knepley - fv - The PetscFV
42663e9753d6SMatthew G. Knepley 
42673e9753d6SMatthew G. Knepley   Output Parameter:
42683e9753d6SMatthew G. Knepley . dmGrad - The layout for gradient values
42693e9753d6SMatthew G. Knepley 
42703e9753d6SMatthew G. Knepley   Level: developer
42713e9753d6SMatthew G. Knepley 
4272446c23c1SPierre Jolivet .seealso: DMPlexGetGeometryFVM()
42733e9753d6SMatthew G. Knepley @*/
42743e9753d6SMatthew G. Knepley PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad)
42753e9753d6SMatthew G. Knepley {
42763e9753d6SMatthew G. Knepley   DM             plex;
42773e9753d6SMatthew G. Knepley   PetscBool      computeGradients;
42783e9753d6SMatthew G. Knepley   PetscErrorCode ierr;
42793e9753d6SMatthew G. Knepley 
42803e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42813e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
42823e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(fv,PETSCFV_CLASSID,2);
42833e9753d6SMatthew G. Knepley   PetscValidPointer(dmGrad,3);
42843e9753d6SMatthew G. Knepley   ierr = PetscFVGetComputeGradients(fv, &computeGradients);CHKERRQ(ierr);
42853e9753d6SMatthew G. Knepley   if (!computeGradients) {*dmGrad = NULL; PetscFunctionReturn(0);}
42863e9753d6SMatthew G. Knepley   ierr = DMConvertPlex_Internal(dm,&plex,PETSC_TRUE);CHKERRQ(ierr);
42873e9753d6SMatthew G. Knepley   ierr = DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad);CHKERRQ(ierr);
42883e9753d6SMatthew G. Knepley   ierr = DMDestroy(&plex);CHKERRQ(ierr);
42893e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
42903e9753d6SMatthew G. Knepley }
42913e9753d6SMatthew G. Knepley 
429245480ffeSMatthew G. Knepley static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS)
42933e9753d6SMatthew G. Knepley {
42943e9753d6SMatthew G. Knepley   DM_Plex         *mesh = (DM_Plex *) dm->data;
42953e9753d6SMatthew G. Knepley   DM               plex = NULL, plexA = NULL;
42963e9753d6SMatthew G. Knepley   DMEnclosureType  encAux;
42973e9753d6SMatthew G. Knepley   PetscDS          prob, probAux = NULL;
42983e9753d6SMatthew G. Knepley   PetscSection     section, sectionAux = NULL;
42993e9753d6SMatthew G. Knepley   Vec              locA = NULL;
43003e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
43013e9753d6SMatthew G. Knepley   PetscInt         v;
43023e9753d6SMatthew G. Knepley   PetscInt         totDim, totDimAux = 0;
43033e9753d6SMatthew G. Knepley   PetscErrorCode   ierr;
43043e9753d6SMatthew G. Knepley 
43053e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43063e9753d6SMatthew G. Knepley   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
43073e9753d6SMatthew G. Knepley   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
43083e9753d6SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
43093e9753d6SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
431004c51a94SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, label, values[0], &locA);CHKERRQ(ierr);
43113e9753d6SMatthew G. Knepley   if (locA) {
43123e9753d6SMatthew G. Knepley     DM dmAux;
43133e9753d6SMatthew G. Knepley 
43143e9753d6SMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
43153e9753d6SMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux, dm, &encAux);CHKERRQ(ierr);
43163e9753d6SMatthew G. Knepley     ierr = DMConvert(dmAux, DMPLEX, &plexA);CHKERRQ(ierr);
43173e9753d6SMatthew G. Knepley     ierr = DMGetDS(plexA, &probAux);CHKERRQ(ierr);
43183e9753d6SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
43193e9753d6SMatthew G. Knepley     ierr = DMGetLocalSection(plexA, &sectionAux);CHKERRQ(ierr);
43203e9753d6SMatthew G. Knepley   }
43213e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
43223e9753d6SMatthew G. Knepley     PetscFEGeom     *fgeom;
43233e9753d6SMatthew G. Knepley     PetscInt         maxDegree;
43243e9753d6SMatthew G. Knepley     PetscQuadrature  qGeom = NULL;
43253e9753d6SMatthew G. Knepley     IS               pointIS;
43263e9753d6SMatthew G. Knepley     const PetscInt  *points;
432706ad1575SMatthew G. Knepley     PetscFormKey key;
43283e9753d6SMatthew G. Knepley     PetscInt         numFaces, face, Nq;
43293e9753d6SMatthew G. Knepley 
433006d8a0d3SMatthew G. Knepley     key.label = label;
433106d8a0d3SMatthew G. Knepley     key.value = values[v];
433206d8a0d3SMatthew G. Knepley     key.field = field;
433306ad1575SMatthew G. Knepley     key.part  = 0;
43343e9753d6SMatthew G. Knepley     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
43353e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
43363e9753d6SMatthew G. Knepley     {
43373e9753d6SMatthew G. Knepley       IS isectIS;
43383e9753d6SMatthew G. Knepley 
43393e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
43403e9753d6SMatthew G. Knepley       ierr = ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS);CHKERRQ(ierr);
43413e9753d6SMatthew G. Knepley       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
43423e9753d6SMatthew G. Knepley       pointIS = isectIS;
43433e9753d6SMatthew G. Knepley     }
43443e9753d6SMatthew G. Knepley     ierr = ISGetLocalSize(pointIS,&numFaces);CHKERRQ(ierr);
43453e9753d6SMatthew G. Knepley     ierr = ISGetIndices(pointIS,&points);CHKERRQ(ierr);
43463e9753d6SMatthew G. Knepley     ierr = PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim, &elemVec, locA ? numFaces*totDimAux : 0, &a);CHKERRQ(ierr);
43473e9753d6SMatthew G. Knepley     ierr = DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree);CHKERRQ(ierr);
43483e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
43493e9753d6SMatthew G. Knepley       ierr = DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom);CHKERRQ(ierr);
43503e9753d6SMatthew G. Knepley     }
43513e9753d6SMatthew G. Knepley     if (!qGeom) {
43523e9753d6SMatthew G. Knepley       PetscFE fe;
43533e9753d6SMatthew G. Knepley 
43543e9753d6SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, field, (PetscObject *) &fe);CHKERRQ(ierr);
43553e9753d6SMatthew G. Knepley       ierr = PetscFEGetFaceQuadrature(fe, &qGeom);CHKERRQ(ierr);
43563e9753d6SMatthew G. Knepley       ierr = PetscObjectReference((PetscObject)qGeom);CHKERRQ(ierr);
43573e9753d6SMatthew G. Knepley     }
43583e9753d6SMatthew G. Knepley     ierr = PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
43593e9753d6SMatthew G. Knepley     ierr = DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);CHKERRQ(ierr);
43603e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
4361f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
43623e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
4363f15274beSMatthew Knepley       PetscInt       i;
43643e9753d6SMatthew G. Knepley 
43653e9753d6SMatthew G. Knepley       ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
43663e9753d6SMatthew G. Knepley       ierr = DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x);CHKERRQ(ierr);
43673e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
43683e9753d6SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x);CHKERRQ(ierr);
43693e9753d6SMatthew G. Knepley       if (locX_t) {
43703e9753d6SMatthew G. Knepley         ierr = DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x);CHKERRQ(ierr);
43713e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
43723e9753d6SMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x);CHKERRQ(ierr);
43733e9753d6SMatthew G. Knepley       }
43743e9753d6SMatthew G. Knepley       if (locA) {
43753e9753d6SMatthew G. Knepley         PetscInt subp;
43763e9753d6SMatthew G. Knepley 
43773e9753d6SMatthew G. Knepley         ierr = DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp);CHKERRQ(ierr);
43783e9753d6SMatthew G. Knepley         ierr = DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x);CHKERRQ(ierr);
43793e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
43803e9753d6SMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x);CHKERRQ(ierr);
43813e9753d6SMatthew G. Knepley       }
43823e9753d6SMatthew G. Knepley     }
43833e9753d6SMatthew G. Knepley     ierr = PetscArrayzero(elemVec, numFaces*totDim);CHKERRQ(ierr);
43843e9753d6SMatthew G. Knepley     {
43853e9753d6SMatthew G. Knepley       PetscFE         fe;
43863e9753d6SMatthew G. Knepley       PetscInt        Nb;
43873e9753d6SMatthew G. Knepley       PetscFEGeom     *chunkGeom = NULL;
43883e9753d6SMatthew G. Knepley       /* Conforming batches */
43893e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
43903e9753d6SMatthew G. Knepley       /* Remainder */
43913e9753d6SMatthew G. Knepley       PetscInt        Nr, offset;
43923e9753d6SMatthew G. Knepley 
43933e9753d6SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, field, (PetscObject *) &fe);CHKERRQ(ierr);
43943e9753d6SMatthew G. Knepley       ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
43953e9753d6SMatthew G. Knepley       ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
43963e9753d6SMatthew G. Knepley       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
43973e9753d6SMatthew G. Knepley       blockSize = Nb;
43983e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
43993e9753d6SMatthew G. Knepley       ierr =  PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
44003e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches*batchSize);
44013e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
44023e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches*batchSize);
44033e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
44043e9753d6SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom);CHKERRQ(ierr);
440545480ffeSMatthew G. Knepley       ierr = PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec);CHKERRQ(ierr);
44063e9753d6SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom);CHKERRQ(ierr);
44073e9753d6SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom);CHKERRQ(ierr);
440845480ffeSMatthew G. Knepley       ierr = PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, &elemVec[offset*totDim]);CHKERRQ(ierr);
44093e9753d6SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom);CHKERRQ(ierr);
44103e9753d6SMatthew G. Knepley     }
44113e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
44123e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
44133e9753d6SMatthew G. Knepley 
44143e9753d6SMatthew G. Knepley       if (mesh->printFEM > 1) {ierr = DMPrintCellVector(point, "BdResidual", totDim, &elemVec[face*totDim]);CHKERRQ(ierr);}
44153e9753d6SMatthew G. Knepley       ierr = DMPlexGetSupport(plex, point, &support);CHKERRQ(ierr);
44163e9753d6SMatthew G. Knepley       ierr = DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face*totDim], ADD_ALL_VALUES);CHKERRQ(ierr);
44173e9753d6SMatthew G. Knepley     }
44183e9753d6SMatthew G. Knepley     ierr = DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);CHKERRQ(ierr);
44193e9753d6SMatthew G. Knepley     ierr = PetscQuadratureDestroy(&qGeom);CHKERRQ(ierr);
44203e9753d6SMatthew G. Knepley     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
44213e9753d6SMatthew G. Knepley     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
44223e9753d6SMatthew G. Knepley     ierr = PetscFree4(u, u_t, elemVec, a);CHKERRQ(ierr);
44233e9753d6SMatthew G. Knepley   }
44243e9753d6SMatthew G. Knepley   ierr = DMDestroy(&plex);CHKERRQ(ierr);
44253e9753d6SMatthew G. Knepley   ierr = DMDestroy(&plexA);CHKERRQ(ierr);
44263e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
44273e9753d6SMatthew G. Knepley }
44283e9753d6SMatthew G. Knepley 
442945480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, Vec locF)
44303e9753d6SMatthew G. Knepley {
44313e9753d6SMatthew G. Knepley   DMField        coordField;
44323e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
44333e9753d6SMatthew G. Knepley   IS             facetIS;
44343e9753d6SMatthew G. Knepley   PetscInt       dim;
44353e9753d6SMatthew G. Knepley   PetscErrorCode ierr;
44363e9753d6SMatthew G. Knepley 
44373e9753d6SMatthew G. Knepley   PetscFunctionBegin;
44383e9753d6SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44393e9753d6SMatthew G. Knepley   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
44403e9753d6SMatthew G. Knepley   ierr = DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);CHKERRQ(ierr);
44413e9753d6SMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
444245480ffeSMatthew G. Knepley   ierr = DMPlexComputeBdResidual_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, locF, coordField, facetIS);CHKERRQ(ierr);
44433e9753d6SMatthew G. Knepley   ierr = ISDestroy(&facetIS);CHKERRQ(ierr);
44443e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
44453e9753d6SMatthew G. Knepley }
44463e9753d6SMatthew G. Knepley 
44473e9753d6SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
44483e9753d6SMatthew G. Knepley {
44493e9753d6SMatthew G. Knepley   PetscDS        prob;
44503e9753d6SMatthew G. Knepley   PetscInt       numBd, bd;
44513e9753d6SMatthew G. Knepley   DMField        coordField = NULL;
44523e9753d6SMatthew G. Knepley   IS             facetIS    = NULL;
44533e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
44543e9753d6SMatthew G. Knepley   PetscInt       dim;
44553e9753d6SMatthew G. Knepley   PetscErrorCode ierr;
44563e9753d6SMatthew G. Knepley 
44573e9753d6SMatthew G. Knepley   PetscFunctionBegin;
44583e9753d6SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
44593e9753d6SMatthew G. Knepley   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
44603e9753d6SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44613e9753d6SMatthew G. Knepley   ierr = DMLabelGetStratumIS(depthLabel,dim - 1,&facetIS);CHKERRQ(ierr);
44623e9753d6SMatthew G. Knepley   ierr = PetscDSGetNumBoundary(prob, &numBd);CHKERRQ(ierr);
44633e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
446445480ffeSMatthew G. Knepley     PetscWeakForm           wf;
44653e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
44663e9753d6SMatthew G. Knepley     DMLabel                 label;
44673e9753d6SMatthew G. Knepley     const PetscInt         *values;
44683e9753d6SMatthew G. Knepley     PetscInt                field, numValues;
44693e9753d6SMatthew G. Knepley     PetscObject             obj;
44703e9753d6SMatthew G. Knepley     PetscClassId            id;
44713e9753d6SMatthew G. Knepley 
447245480ffeSMatthew G. Knepley     ierr = PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
44733e9753d6SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, field, &obj);CHKERRQ(ierr);
44743e9753d6SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
44753e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
44763e9753d6SMatthew G. Knepley     if (!facetIS) {
44773e9753d6SMatthew G. Knepley       DMLabel  depthLabel;
44783e9753d6SMatthew G. Knepley       PetscInt dim;
44793e9753d6SMatthew G. Knepley 
44803e9753d6SMatthew G. Knepley       ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
44813e9753d6SMatthew G. Knepley       ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44823e9753d6SMatthew G. Knepley       ierr = DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS);CHKERRQ(ierr);
44833e9753d6SMatthew G. Knepley     }
44843e9753d6SMatthew G. Knepley     ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
448545480ffeSMatthew G. Knepley     ierr = DMPlexComputeBdResidual_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, locF, coordField, facetIS);CHKERRQ(ierr);
44863e9753d6SMatthew G. Knepley   }
44873e9753d6SMatthew G. Knepley   ierr = ISDestroy(&facetIS);CHKERRQ(ierr);
44883e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
44893e9753d6SMatthew G. Knepley }
44903e9753d6SMatthew G. Knepley 
449106ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
44923e9753d6SMatthew G. Knepley {
44933e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
44943e9753d6SMatthew G. Knepley   const char      *name       = "Residual";
44953e9753d6SMatthew G. Knepley   DM               dmAux      = NULL;
44963e9753d6SMatthew G. Knepley   DM               dmGrad     = NULL;
44973e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
44986528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
44996528b96dSMatthew G. Knepley   PetscDS          dsAux      = NULL;
45003e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
45013e9753d6SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
45023e9753d6SMatthew G. Knepley   PetscBool        useFVM     = PETSC_FALSE;
45033e9753d6SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
45043e9753d6SMatthew G. Knepley   PetscFV          fvm        = NULL;
45053e9753d6SMatthew G. Knepley   PetscFVCellGeom *cgeomFVM   = NULL;
45063e9753d6SMatthew G. Knepley   PetscFVFaceGeom *fgeomFVM   = NULL;
45073e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
45083e9753d6SMatthew G. Knepley   Vec              locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, grad, locGrad = NULL;
45093e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a, *uL, *uR;
45103e9753d6SMatthew G. Knepley   IS               chunkIS;
45113e9753d6SMatthew G. Knepley   const PetscInt  *cells;
45123e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
45133e9753d6SMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
45143e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
45153e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
45163e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
45173e9753d6SMatthew G. Knepley   PetscErrorCode   ierr;
45183e9753d6SMatthew G. Knepley 
45193e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45203e9753d6SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0);CHKERRQ(ierr);
45213e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
45223e9753d6SMatthew G. Knepley   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
45233e9753d6SMatthew G. Knepley   /* FEM+FVM */
45243e9753d6SMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
45253e9753d6SMatthew G. Knepley   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
45263e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
45273e9753d6SMatthew G. Knepley   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
45283e9753d6SMatthew G. Knepley   ierr = DMGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
45296528b96dSMatthew G. Knepley   ierr = DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds);CHKERRQ(ierr);
45306528b96dSMatthew G. Knepley   ierr = PetscDSGetNumFields(ds, &Nf);CHKERRQ(ierr);
45316528b96dSMatthew G. Knepley   ierr = PetscDSGetTotalDimension(ds, &totDim);CHKERRQ(ierr);
453204c51a94SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, key.label, key.value, &locA);CHKERRQ(ierr);
45333e9753d6SMatthew G. Knepley   if (locA) {
45343e9753d6SMatthew G. Knepley     PetscInt subcell;
45353e9753d6SMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
45363e9753d6SMatthew G. Knepley     ierr = DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cStart, &subcell);CHKERRQ(ierr);
45376528b96dSMatthew G. Knepley     ierr = DMGetCellDS(dmAux, subcell, &dsAux);CHKERRQ(ierr);
45386528b96dSMatthew G. Knepley     ierr = PetscDSGetTotalDimension(dsAux, &totDimAux);CHKERRQ(ierr);
45393e9753d6SMatthew G. Knepley   }
45403e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
45413e9753d6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
45423e9753d6SMatthew G. Knepley     PetscObject  obj;
45433e9753d6SMatthew G. Knepley     PetscClassId id;
45443e9753d6SMatthew G. Knepley     PetscBool    fimp;
45453e9753d6SMatthew G. Knepley 
45466528b96dSMatthew G. Knepley     ierr = PetscDSGetImplicit(ds, f, &fimp);CHKERRQ(ierr);
45473e9753d6SMatthew G. Knepley     if (isImplicit != fimp) continue;
45486528b96dSMatthew G. Knepley     ierr = PetscDSGetDiscretization(ds, f, &obj);CHKERRQ(ierr);
45493e9753d6SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
45503e9753d6SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
45513e9753d6SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {useFVM = PETSC_TRUE; fvm = (PetscFV) obj;}
45523e9753d6SMatthew G. Knepley   }
45533e9753d6SMatthew G. Knepley   if (useFEM) {
45543e9753d6SMatthew G. Knepley     ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
45553e9753d6SMatthew G. Knepley     ierr = DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);CHKERRQ(ierr);
45563e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
45573e9753d6SMatthew G. Knepley       ierr = DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad);CHKERRQ(ierr);
45583e9753d6SMatthew G. Knepley       if (affineQuad) {
45593e9753d6SMatthew G. Knepley         ierr = DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);CHKERRQ(ierr);
45603e9753d6SMatthew G. Knepley       }
45613e9753d6SMatthew G. Knepley     } else {
45623e9753d6SMatthew G. Knepley       ierr = PetscCalloc2(Nf,&quads,Nf,&geoms);CHKERRQ(ierr);
45633e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
45643e9753d6SMatthew G. Knepley         PetscObject  obj;
45653e9753d6SMatthew G. Knepley         PetscClassId id;
45663e9753d6SMatthew G. Knepley         PetscBool    fimp;
45673e9753d6SMatthew G. Knepley 
45686528b96dSMatthew G. Knepley         ierr = PetscDSGetImplicit(ds, f, &fimp);CHKERRQ(ierr);
45693e9753d6SMatthew G. Knepley         if (isImplicit != fimp) continue;
45706528b96dSMatthew G. Knepley         ierr = PetscDSGetDiscretization(ds, f, &obj);CHKERRQ(ierr);
45713e9753d6SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
45723e9753d6SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
45733e9753d6SMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
45743e9753d6SMatthew G. Knepley 
45753e9753d6SMatthew G. Knepley           ierr = PetscFEGetQuadrature(fe, &quads[f]);CHKERRQ(ierr);
45763e9753d6SMatthew G. Knepley           ierr = PetscObjectReference((PetscObject)quads[f]);CHKERRQ(ierr);
45773e9753d6SMatthew G. Knepley           ierr = DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);CHKERRQ(ierr);
45783e9753d6SMatthew G. Knepley         }
45793e9753d6SMatthew G. Knepley       }
45803e9753d6SMatthew G. Knepley     }
45813e9753d6SMatthew G. Knepley   }
45823e9753d6SMatthew G. Knepley   if (useFVM) {
45833e9753d6SMatthew G. Knepley     ierr = DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL);CHKERRQ(ierr);
45843e9753d6SMatthew G. Knepley     ierr = VecGetArrayRead(faceGeometryFVM, (const PetscScalar **) &fgeomFVM);CHKERRQ(ierr);
45853e9753d6SMatthew G. Knepley     ierr = VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM);CHKERRQ(ierr);
45863e9753d6SMatthew G. Knepley     /* Reconstruct and limit cell gradients */
45873e9753d6SMatthew G. Knepley     ierr = DMPlexGetGradientDM(dm, fvm, &dmGrad);CHKERRQ(ierr);
45883e9753d6SMatthew G. Knepley     if (dmGrad) {
45893e9753d6SMatthew G. Knepley       ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
45903e9753d6SMatthew G. Knepley       ierr = DMGetGlobalVector(dmGrad, &grad);CHKERRQ(ierr);
45913e9753d6SMatthew G. Knepley       ierr = DMPlexReconstructGradients_Internal(dm, fvm, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad);CHKERRQ(ierr);
45923e9753d6SMatthew G. Knepley       /* Communicate gradient values */
45933e9753d6SMatthew G. Knepley       ierr = DMGetLocalVector(dmGrad, &locGrad);CHKERRQ(ierr);
45943e9753d6SMatthew G. Knepley       ierr = DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
45953e9753d6SMatthew G. Knepley       ierr = DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
45963e9753d6SMatthew G. Knepley       ierr = DMRestoreGlobalVector(dmGrad, &grad);CHKERRQ(ierr);
45973e9753d6SMatthew G. Knepley     }
45983e9753d6SMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
45993e9753d6SMatthew G. Knepley     ierr = DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad);CHKERRQ(ierr);
46003e9753d6SMatthew G. Knepley   }
46013e9753d6SMatthew G. Knepley   /* Loop over chunks */
46023e9753d6SMatthew G. Knepley   if (useFEM) {ierr = ISCreate(PETSC_COMM_SELF, &chunkIS);CHKERRQ(ierr);}
46033e9753d6SMatthew G. Knepley   numCells      = cEnd - cStart;
46043e9753d6SMatthew G. Knepley   numChunks     = 1;
46053e9753d6SMatthew G. Knepley   cellChunkSize = numCells/numChunks;
46063e9753d6SMatthew G. Knepley   faceChunkSize = (fEnd - fStart)/numChunks;
46073e9753d6SMatthew G. Knepley   numChunks     = PetscMin(1,numCells);
46083e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
46093e9753d6SMatthew G. Knepley     PetscScalar     *elemVec, *fluxL, *fluxR;
46103e9753d6SMatthew G. Knepley     PetscReal       *vol;
46113e9753d6SMatthew G. Knepley     PetscFVFaceGeom *fgeom;
46123e9753d6SMatthew G. Knepley     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
46133e9753d6SMatthew G. Knepley     PetscInt         fS = fStart+chunk*faceChunkSize, fE = PetscMin(fS+faceChunkSize, fEnd), numFaces = 0, face;
46143e9753d6SMatthew G. Knepley 
46153e9753d6SMatthew G. Knepley     /* Extract field coefficients */
46163e9753d6SMatthew G. Knepley     if (useFEM) {
46173e9753d6SMatthew G. Knepley       ierr = ISGetPointSubrange(chunkIS, cS, cE, cells);CHKERRQ(ierr);
46183e9753d6SMatthew G. Knepley       ierr = DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);CHKERRQ(ierr);
46193e9753d6SMatthew G. Knepley       ierr = DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);CHKERRQ(ierr);
46203e9753d6SMatthew G. Knepley       ierr = PetscArrayzero(elemVec, numCells*totDim);CHKERRQ(ierr);
46213e9753d6SMatthew G. Knepley     }
46223e9753d6SMatthew G. Knepley     if (useFVM) {
46233e9753d6SMatthew G. Knepley       ierr = DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR);CHKERRQ(ierr);
46243e9753d6SMatthew G. Knepley       ierr = DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol);CHKERRQ(ierr);
46253e9753d6SMatthew G. Knepley       ierr = DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL);CHKERRQ(ierr);
46263e9753d6SMatthew G. Knepley       ierr = DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR);CHKERRQ(ierr);
46273e9753d6SMatthew G. Knepley       ierr = PetscArrayzero(fluxL, numFaces*totDim);CHKERRQ(ierr);
46283e9753d6SMatthew G. Knepley       ierr = PetscArrayzero(fluxR, numFaces*totDim);CHKERRQ(ierr);
46293e9753d6SMatthew G. Knepley     }
46303e9753d6SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
46313e9753d6SMatthew G. Knepley     /* Loop over fields */
46323e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
46333e9753d6SMatthew G. Knepley       PetscObject  obj;
46343e9753d6SMatthew G. Knepley       PetscClassId id;
46353e9753d6SMatthew G. Knepley       PetscBool    fimp;
46363e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
46373e9753d6SMatthew G. Knepley 
46386528b96dSMatthew G. Knepley       key.field = f;
46396528b96dSMatthew G. Knepley       ierr = PetscDSGetImplicit(ds, f, &fimp);CHKERRQ(ierr);
46403e9753d6SMatthew G. Knepley       if (isImplicit != fimp) continue;
46416528b96dSMatthew G. Knepley       ierr = PetscDSGetDiscretization(ds, f, &obj);CHKERRQ(ierr);
46423e9753d6SMatthew G. Knepley       ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
46433e9753d6SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
46443e9753d6SMatthew G. Knepley         PetscFE         fe = (PetscFE) obj;
46453e9753d6SMatthew G. Knepley         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
46463e9753d6SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
46473e9753d6SMatthew G. Knepley         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
46483e9753d6SMatthew G. Knepley         PetscInt        Nq, Nb;
46493e9753d6SMatthew G. Knepley 
46503e9753d6SMatthew G. Knepley         ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
46513e9753d6SMatthew G. Knepley         ierr = PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
46523e9753d6SMatthew G. Knepley         ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
46533e9753d6SMatthew G. Knepley         blockSize = Nb;
46543e9753d6SMatthew G. Knepley         batchSize = numBlocks * blockSize;
46553e9753d6SMatthew G. Knepley         ierr      = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
46563e9753d6SMatthew G. Knepley         numChunks = numCells / (numBatches*batchSize);
46573e9753d6SMatthew G. Knepley         Ne        = numChunks*numBatches*batchSize;
46583e9753d6SMatthew G. Knepley         Nr        = numCells % (numBatches*batchSize);
46593e9753d6SMatthew G. Knepley         offset    = numCells - Nr;
46603e9753d6SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
46613e9753d6SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
46623e9753d6SMatthew G. Knepley         ierr = PetscFEGeomGetChunk(geom,0,offset,&chunkGeom);CHKERRQ(ierr);
46636528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec);CHKERRQ(ierr);
46643e9753d6SMatthew G. Knepley         ierr = PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom);CHKERRQ(ierr);
46656528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]);CHKERRQ(ierr);
46663e9753d6SMatthew G. Knepley         ierr = PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom);CHKERRQ(ierr);
46673e9753d6SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
46683e9753d6SMatthew G. Knepley         PetscFV fv = (PetscFV) obj;
46693e9753d6SMatthew G. Knepley 
46703e9753d6SMatthew G. Knepley         Ne = numFaces;
46713e9753d6SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
46723e9753d6SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
46736528b96dSMatthew G. Knepley         ierr = PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR);CHKERRQ(ierr);
46743e9753d6SMatthew G. Knepley       } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %D", f);
46753e9753d6SMatthew G. Knepley     }
46763e9753d6SMatthew G. Knepley     /* Loop over domain */
46773e9753d6SMatthew G. Knepley     if (useFEM) {
46783e9753d6SMatthew G. Knepley       /* Add elemVec to locX */
46793e9753d6SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
46803e9753d6SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
46813e9753d6SMatthew G. Knepley         const PetscInt cind = c - cStart;
46823e9753d6SMatthew G. Knepley 
46833e9753d6SMatthew G. Knepley         if (mesh->printFEM > 1) {ierr = DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]);CHKERRQ(ierr);}
46843e9753d6SMatthew G. Knepley         if (ghostLabel) {
46853e9753d6SMatthew G. Knepley           PetscInt ghostVal;
46863e9753d6SMatthew G. Knepley 
46873e9753d6SMatthew G. Knepley           ierr = DMLabelGetValue(ghostLabel,cell,&ghostVal);CHKERRQ(ierr);
46883e9753d6SMatthew G. Knepley           if (ghostVal > 0) continue;
46893e9753d6SMatthew G. Knepley         }
46903e9753d6SMatthew G. Knepley         ierr = DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES);CHKERRQ(ierr);
46913e9753d6SMatthew G. Knepley       }
46923e9753d6SMatthew G. Knepley     }
46933e9753d6SMatthew G. Knepley     if (useFVM) {
46943e9753d6SMatthew G. Knepley       PetscScalar *fa;
46953e9753d6SMatthew G. Knepley       PetscInt     iface;
46963e9753d6SMatthew G. Knepley 
46973e9753d6SMatthew G. Knepley       ierr = VecGetArray(locF, &fa);CHKERRQ(ierr);
46983e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
46993e9753d6SMatthew G. Knepley         PetscFV      fv;
47003e9753d6SMatthew G. Knepley         PetscObject  obj;
47013e9753d6SMatthew G. Knepley         PetscClassId id;
47023e9753d6SMatthew G. Knepley         PetscInt     foff, pdim;
47033e9753d6SMatthew G. Knepley 
47046528b96dSMatthew G. Knepley         ierr = PetscDSGetDiscretization(ds, f, &obj);CHKERRQ(ierr);
47056528b96dSMatthew G. Knepley         ierr = PetscDSGetFieldOffset(ds, f, &foff);CHKERRQ(ierr);
47063e9753d6SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
47073e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
47083e9753d6SMatthew G. Knepley         fv   = (PetscFV) obj;
47093e9753d6SMatthew G. Knepley         ierr = PetscFVGetNumComponents(fv, &pdim);CHKERRQ(ierr);
47103e9753d6SMatthew G. Knepley         /* Accumulate fluxes to cells */
47113e9753d6SMatthew G. Knepley         for (face = fS, iface = 0; face < fE; ++face) {
47123e9753d6SMatthew G. Knepley           const PetscInt *scells;
47133e9753d6SMatthew G. Knepley           PetscScalar    *fL = NULL, *fR = NULL;
47143e9753d6SMatthew G. Knepley           PetscInt        ghost, d, nsupp, nchild;
47153e9753d6SMatthew G. Knepley 
47163e9753d6SMatthew G. Knepley           ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
47173e9753d6SMatthew G. Knepley           ierr = DMPlexGetSupportSize(dm, face, &nsupp);CHKERRQ(ierr);
47183e9753d6SMatthew G. Knepley           ierr = DMPlexGetTreeChildren(dm, face, &nchild, NULL);CHKERRQ(ierr);
47193e9753d6SMatthew G. Knepley           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
47203e9753d6SMatthew G. Knepley           ierr = DMPlexGetSupport(dm, face, &scells);CHKERRQ(ierr);
47213e9753d6SMatthew G. Knepley           ierr = DMLabelGetValue(ghostLabel,scells[0],&ghost);CHKERRQ(ierr);
47223e9753d6SMatthew G. Knepley           if (ghost <= 0) {ierr = DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL);CHKERRQ(ierr);}
47233e9753d6SMatthew G. Knepley           ierr = DMLabelGetValue(ghostLabel,scells[1],&ghost);CHKERRQ(ierr);
47243e9753d6SMatthew G. Knepley           if (ghost <= 0) {ierr = DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR);CHKERRQ(ierr);}
47253e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) {
47263e9753d6SMatthew G. Knepley             if (fL) fL[d] -= fluxL[iface*totDim+foff+d];
47273e9753d6SMatthew G. Knepley             if (fR) fR[d] += fluxR[iface*totDim+foff+d];
47283e9753d6SMatthew G. Knepley           }
47293e9753d6SMatthew G. Knepley           ++iface;
47303e9753d6SMatthew G. Knepley         }
47313e9753d6SMatthew G. Knepley       }
47323e9753d6SMatthew G. Knepley       ierr = VecRestoreArray(locF, &fa);CHKERRQ(ierr);
47333e9753d6SMatthew G. Knepley     }
47343e9753d6SMatthew G. Knepley     /* Handle time derivative */
47353e9753d6SMatthew G. Knepley     if (locX_t) {
47363e9753d6SMatthew G. Knepley       PetscScalar *x_t, *fa;
47373e9753d6SMatthew G. Knepley 
47383e9753d6SMatthew G. Knepley       ierr = VecGetArray(locF, &fa);CHKERRQ(ierr);
47393e9753d6SMatthew G. Knepley       ierr = VecGetArray(locX_t, &x_t);CHKERRQ(ierr);
47403e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
47413e9753d6SMatthew G. Knepley         PetscFV      fv;
47423e9753d6SMatthew G. Knepley         PetscObject  obj;
47433e9753d6SMatthew G. Knepley         PetscClassId id;
47443e9753d6SMatthew G. Knepley         PetscInt     pdim, d;
47453e9753d6SMatthew G. Knepley 
47466528b96dSMatthew G. Knepley         ierr = PetscDSGetDiscretization(ds, f, &obj);CHKERRQ(ierr);
47473e9753d6SMatthew G. Knepley         ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
47483e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
47493e9753d6SMatthew G. Knepley         fv   = (PetscFV) obj;
47503e9753d6SMatthew G. Knepley         ierr = PetscFVGetNumComponents(fv, &pdim);CHKERRQ(ierr);
47513e9753d6SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
47523e9753d6SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
47533e9753d6SMatthew G. Knepley           PetscScalar   *u_t, *r;
47543e9753d6SMatthew G. Knepley 
47553e9753d6SMatthew G. Knepley           if (ghostLabel) {
47563e9753d6SMatthew G. Knepley             PetscInt ghostVal;
47573e9753d6SMatthew G. Knepley 
47583e9753d6SMatthew G. Knepley             ierr = DMLabelGetValue(ghostLabel, cell, &ghostVal);CHKERRQ(ierr);
47593e9753d6SMatthew G. Knepley             if (ghostVal > 0) continue;
47603e9753d6SMatthew G. Knepley           }
47613e9753d6SMatthew G. Knepley           ierr = DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t);CHKERRQ(ierr);
47623e9753d6SMatthew G. Knepley           ierr = DMPlexPointLocalFieldRef(dm, cell, f, fa, &r);CHKERRQ(ierr);
47633e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
47643e9753d6SMatthew G. Knepley         }
47653e9753d6SMatthew G. Knepley       }
47663e9753d6SMatthew G. Knepley       ierr = VecRestoreArray(locX_t, &x_t);CHKERRQ(ierr);
47673e9753d6SMatthew G. Knepley       ierr = VecRestoreArray(locF, &fa);CHKERRQ(ierr);
47683e9753d6SMatthew G. Knepley     }
47693e9753d6SMatthew G. Knepley     if (useFEM) {
47703e9753d6SMatthew G. Knepley       ierr = DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);CHKERRQ(ierr);
47713e9753d6SMatthew G. Knepley       ierr = DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);CHKERRQ(ierr);
47723e9753d6SMatthew G. Knepley     }
47733e9753d6SMatthew G. Knepley     if (useFVM) {
47743e9753d6SMatthew G. Knepley       ierr = DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR);CHKERRQ(ierr);
47753e9753d6SMatthew G. Knepley       ierr = DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol);CHKERRQ(ierr);
47763e9753d6SMatthew G. Knepley       ierr = DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL);CHKERRQ(ierr);
47773e9753d6SMatthew G. Knepley       ierr = DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR);CHKERRQ(ierr);
47783e9753d6SMatthew G. Knepley       if (dmGrad) {ierr = DMRestoreLocalVector(dmGrad, &locGrad);CHKERRQ(ierr);}
47793e9753d6SMatthew G. Knepley     }
47803e9753d6SMatthew G. Knepley   }
47813e9753d6SMatthew G. Knepley   if (useFEM) {ierr = ISDestroy(&chunkIS);CHKERRQ(ierr);}
47823e9753d6SMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
47833e9753d6SMatthew G. Knepley 
47843e9753d6SMatthew G. Knepley   if (useFEM) {
47853e9753d6SMatthew G. Knepley     ierr = DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user);CHKERRQ(ierr);
47863e9753d6SMatthew G. Knepley 
47873e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
47883e9753d6SMatthew G. Knepley       ierr = DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);CHKERRQ(ierr);
47893e9753d6SMatthew G. Knepley       ierr = PetscQuadratureDestroy(&affineQuad);CHKERRQ(ierr);
47903e9753d6SMatthew G. Knepley     } else {
47913e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
47923e9753d6SMatthew G. Knepley         ierr = DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);CHKERRQ(ierr);
47933e9753d6SMatthew G. Knepley         ierr = PetscQuadratureDestroy(&quads[f]);CHKERRQ(ierr);
47943e9753d6SMatthew G. Knepley       }
47953e9753d6SMatthew G. Knepley       ierr = PetscFree2(quads,geoms);CHKERRQ(ierr);
47963e9753d6SMatthew G. Knepley     }
47973e9753d6SMatthew G. Knepley   }
47983e9753d6SMatthew G. Knepley 
47993e9753d6SMatthew G. Knepley   /* FEM */
48003e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
48013e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
48023e9753d6SMatthew G. Knepley   /* 3: Handle boundary values */
48033e9753d6SMatthew G. Knepley   /* 4: Loop over domain */
48043e9753d6SMatthew G. Knepley   /*   Extract coefficients */
48053e9753d6SMatthew G. Knepley   /* Loop over fields */
48063e9753d6SMatthew G. Knepley   /*   Set tiling for FE*/
48073e9753d6SMatthew G. Knepley   /*   Integrate FE residual to get elemVec */
48083e9753d6SMatthew G. Knepley   /*     Loop over subdomain */
48093e9753d6SMatthew G. Knepley   /*       Loop over quad points */
48103e9753d6SMatthew G. Knepley   /*         Transform coords to real space */
48113e9753d6SMatthew G. Knepley   /*         Evaluate field and aux fields at point */
48123e9753d6SMatthew G. Knepley   /*         Evaluate residual at point */
48133e9753d6SMatthew G. Knepley   /*         Transform residual to real space */
48143e9753d6SMatthew G. Knepley   /*       Add residual to elemVec */
48153e9753d6SMatthew G. Knepley   /* Loop over domain */
48163e9753d6SMatthew G. Knepley   /*   Add elemVec to locX */
48173e9753d6SMatthew G. Knepley 
48183e9753d6SMatthew G. Knepley   /* FVM */
48193e9753d6SMatthew G. Knepley   /* Get geometric data */
48203e9753d6SMatthew G. Knepley   /* If using gradients */
48213e9753d6SMatthew G. Knepley   /*   Compute gradient data */
48223e9753d6SMatthew G. Knepley   /*   Loop over domain faces */
48233e9753d6SMatthew G. Knepley   /*     Count computational faces */
48243e9753d6SMatthew G. Knepley   /*     Reconstruct cell gradient */
48253e9753d6SMatthew G. Knepley   /*   Loop over domain cells */
48263e9753d6SMatthew G. Knepley   /*     Limit cell gradients */
48273e9753d6SMatthew G. Knepley   /* Handle boundary values */
48283e9753d6SMatthew G. Knepley   /* Loop over domain faces */
48293e9753d6SMatthew G. Knepley   /*   Read out field, centroid, normal, volume for each side of face */
48303e9753d6SMatthew G. Knepley   /* Riemann solve over faces */
48313e9753d6SMatthew G. Knepley   /* Loop over domain faces */
48323e9753d6SMatthew G. Knepley   /*   Accumulate fluxes to cells */
48333e9753d6SMatthew G. Knepley   /* TODO Change printFEM to printDisc here */
48343e9753d6SMatthew G. Knepley   if (mesh->printFEM) {
48353e9753d6SMatthew G. Knepley     Vec         locFbc;
48363e9753d6SMatthew G. Knepley     PetscInt    pStart, pEnd, p, maxDof;
48373e9753d6SMatthew G. Knepley     PetscScalar *zeroes;
48383e9753d6SMatthew G. Knepley 
48393e9753d6SMatthew G. Knepley     ierr = VecDuplicate(locF,&locFbc);CHKERRQ(ierr);
48403e9753d6SMatthew G. Knepley     ierr = VecCopy(locF,locFbc);CHKERRQ(ierr);
48413e9753d6SMatthew G. Knepley     ierr = PetscSectionGetChart(section,&pStart,&pEnd);CHKERRQ(ierr);
48423e9753d6SMatthew G. Knepley     ierr = PetscSectionGetMaxDof(section,&maxDof);CHKERRQ(ierr);
48433e9753d6SMatthew G. Knepley     ierr = PetscCalloc1(maxDof,&zeroes);CHKERRQ(ierr);
48443e9753d6SMatthew G. Knepley     for (p = pStart; p < pEnd; p++) {
48453e9753d6SMatthew G. Knepley       ierr = VecSetValuesSection(locFbc,section,p,zeroes,INSERT_BC_VALUES);CHKERRQ(ierr);
48463e9753d6SMatthew G. Knepley     }
48473e9753d6SMatthew G. Knepley     ierr = PetscFree(zeroes);CHKERRQ(ierr);
48483e9753d6SMatthew G. Knepley     ierr = DMPrintLocalVec(dm, name, mesh->printTol, locFbc);CHKERRQ(ierr);
48493e9753d6SMatthew G. Knepley     ierr = VecDestroy(&locFbc);CHKERRQ(ierr);
48503e9753d6SMatthew G. Knepley   }
48513e9753d6SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0);CHKERRQ(ierr);
48523e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
48533e9753d6SMatthew G. Knepley }
48543e9753d6SMatthew G. Knepley 
48556528b96dSMatthew G. Knepley /*
48566528b96dSMatthew G. Knepley   1) Allow multiple kernels for BdResidual for hybrid DS
48576528b96dSMatthew G. Knepley 
48586528b96dSMatthew G. Knepley   DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux
48596528b96dSMatthew G. Knepley 
48606528b96dSMatthew G. Knepley   DONE 3) Change DMGetCellFields() to get different aux data a[] for each side
48616528b96dSMatthew G. Knepley      - I think I just need to replace a[] with the closure from each face
48626528b96dSMatthew G. Knepley 
48636528b96dSMatthew G. Knepley   4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before
48646528b96dSMatthew G. Knepley */
486506ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
48663e9753d6SMatthew G. Knepley {
48673e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
48683e9753d6SMatthew G. Knepley   const char      *name       = "Hybrid Residual";
486904c51a94SMatthew G. Knepley   DM               dmAux[3]   = {NULL, NULL, NULL};
48703e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
48716528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
48726528b96dSMatthew G. Knepley   PetscDS          dsAux[3]   = {NULL, NULL, NULL};
487304c51a94SMatthew G. Knepley   Vec              locA[3]    = {NULL, NULL, NULL};
48743e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
48753e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
48766528b96dSMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a[3];
48773e9753d6SMatthew G. Knepley   PetscScalar     *elemVec;
48783e9753d6SMatthew G. Knepley   IS               chunkIS;
48793e9753d6SMatthew G. Knepley   const PetscInt  *cells;
48803e9753d6SMatthew G. Knepley   PetscInt        *faces;
48813e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
48826528b96dSMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
48833e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
48843e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
48853e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
48863e9753d6SMatthew G. Knepley   PetscErrorCode   ierr;
48873e9753d6SMatthew G. Knepley 
48883e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48893e9753d6SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0);CHKERRQ(ierr);
48903e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
48913e9753d6SMatthew G. Knepley   /* FEM */
4892148442b3SMatthew G. Knepley   ierr = ISGetLocalSize(cellIS, &numCells);CHKERRQ(ierr);
48933e9753d6SMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
48943e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
48953e9753d6SMatthew G. Knepley   ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
48963e9753d6SMatthew G. Knepley   ierr = DMGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
48976528b96dSMatthew G. Knepley   ierr = DMGetCellDS(dm, cStart, &ds);CHKERRQ(ierr);
48986528b96dSMatthew G. Knepley   ierr = PetscDSGetNumFields(ds, &Nf);CHKERRQ(ierr);
48996528b96dSMatthew G. Knepley   ierr = PetscDSGetTotalDimension(ds, &totDim);CHKERRQ(ierr);
490004c51a94SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, key[2].label, key[2].value, &locA[2]);CHKERRQ(ierr);
490104c51a94SMatthew G. Knepley   if (locA[2]) {
490204c51a94SMatthew G. Knepley     ierr = VecGetDM(locA[2], &dmAux[2]);CHKERRQ(ierr);
490304c51a94SMatthew G. Knepley     ierr = DMGetCellDS(dmAux[2], cStart, &dsAux[2]);CHKERRQ(ierr);
49046528b96dSMatthew G. Knepley     ierr = PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]);CHKERRQ(ierr);
49056528b96dSMatthew G. Knepley     {
49066528b96dSMatthew G. Knepley       const PetscInt *cone;
49076528b96dSMatthew G. Knepley       PetscInt        c;
49086528b96dSMatthew G. Knepley 
49096528b96dSMatthew G. Knepley       ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
49106528b96dSMatthew G. Knepley       for (c = 0; c < 2; ++c) {
49116528b96dSMatthew G. Knepley         const PetscInt *support;
49126528b96dSMatthew G. Knepley         PetscInt ssize, s;
49136528b96dSMatthew G. Knepley 
49146528b96dSMatthew G. Knepley         ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
49156528b96dSMatthew G. Knepley         ierr = DMPlexGetSupportSize(dm, cone[c], &ssize);CHKERRQ(ierr);
49166528b96dSMatthew G. Knepley         if (ssize != 2) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D from cell %D has support size %D != 2", cone[c], cStart, ssize);
49176528b96dSMatthew G. Knepley         if      (support[0] == cStart) s = 1;
49186528b96dSMatthew G. Knepley         else if (support[1] == cStart) s = 0;
49196528b96dSMatthew G. Knepley         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D does not have cell %D in its support", cone[c], cStart);
492004c51a94SMatthew G. Knepley         ierr = DMGetAuxiliaryVec(dm, key[c].label, key[c].value, &locA[c]);CHKERRQ(ierr);
492104c51a94SMatthew G. Knepley         if (locA[c]) {ierr = VecGetDM(locA[c], &dmAux[c]);CHKERRQ(ierr);}
492204c51a94SMatthew G. Knepley         else         {dmAux[c] = dmAux[2];}
492304c51a94SMatthew G. Knepley         ierr = DMGetCellDS(dmAux[c], support[s], &dsAux[c]);CHKERRQ(ierr);
49246528b96dSMatthew G. Knepley         ierr = PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]);CHKERRQ(ierr);
49256528b96dSMatthew G. Knepley       }
49266528b96dSMatthew G. Knepley     }
49273e9753d6SMatthew G. Knepley   }
49283e9753d6SMatthew G. Knepley   /* 2: Setup geometric data */
49293e9753d6SMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
49303e9753d6SMatthew G. Knepley   ierr = DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree);CHKERRQ(ierr);
49313e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
49323e9753d6SMatthew G. Knepley     ierr = PetscCalloc2(Nf, &quads, Nf, &geoms);CHKERRQ(ierr);
49333e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
49343e9753d6SMatthew G. Knepley       PetscFE fe;
49353e9753d6SMatthew G. Knepley 
49366528b96dSMatthew G. Knepley       ierr = PetscDSGetDiscretization(ds, f, (PetscObject *) &fe);CHKERRQ(ierr);
49373e9753d6SMatthew G. Knepley       if (fe) {
49383e9753d6SMatthew G. Knepley         ierr = PetscFEGetQuadrature(fe, &quads[f]);CHKERRQ(ierr);
49393e9753d6SMatthew G. Knepley         ierr = PetscObjectReference((PetscObject) quads[f]);CHKERRQ(ierr);
49403e9753d6SMatthew G. Knepley       }
49413e9753d6SMatthew G. Knepley     }
49423e9753d6SMatthew G. Knepley   }
49433e9753d6SMatthew G. Knepley   /* Loop over chunks */
49443e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
49453e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal) numCells)/cellChunkSize);
49463e9753d6SMatthew G. Knepley   ierr = PetscCalloc1(2*cellChunkSize, &faces);CHKERRQ(ierr);
49473e9753d6SMatthew G. Knepley   ierr = ISCreateGeneral(PETSC_COMM_SELF, cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS);CHKERRQ(ierr);
49483e9753d6SMatthew G. Knepley   /* Extract field coefficients */
49493e9753d6SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
495004c51a94SMatthew G. Knepley   ierr = DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]);CHKERRQ(ierr);
4951148442b3SMatthew G. Knepley   ierr = DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a);CHKERRQ(ierr);
49523e9753d6SMatthew G. Knepley   ierr = DMGetWorkArray(dm, cellChunkSize*totDim, MPIU_SCALAR, &elemVec);CHKERRQ(ierr);
49533e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
49543e9753d6SMatthew G. Knepley     PetscInt cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
49553e9753d6SMatthew G. Knepley 
49563e9753d6SMatthew G. Knepley     ierr = PetscMemzero(elemVec, cellChunkSize*totDim * sizeof(PetscScalar));CHKERRQ(ierr);
49573e9753d6SMatthew G. Knepley     /* Get faces */
49583e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
49593e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
49603e9753d6SMatthew G. Knepley       const PetscInt *cone;
49613e9753d6SMatthew G. Knepley       ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
49623e9753d6SMatthew G. Knepley       faces[(c-cS)*2+0] = cone[0];
49633e9753d6SMatthew G. Knepley       faces[(c-cS)*2+1] = cone[1];
49643e9753d6SMatthew G. Knepley     }
49653e9753d6SMatthew G. Knepley     ierr = ISGeneralSetIndices(chunkIS, cellChunkSize, faces, PETSC_USE_POINTER);CHKERRQ(ierr);
49663e9753d6SMatthew G. Knepley     /* Get geometric data */
49673e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
49683e9753d6SMatthew G. Knepley       if (!affineQuad) {ierr = DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad);CHKERRQ(ierr);}
49693e9753d6SMatthew G. Knepley       if (affineQuad)  {ierr = DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom);CHKERRQ(ierr);}
49703e9753d6SMatthew G. Knepley     } else {
49713e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
49723e9753d6SMatthew G. Knepley         if (quads[f]) {ierr = DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]);CHKERRQ(ierr);}
49733e9753d6SMatthew G. Knepley       }
49743e9753d6SMatthew G. Knepley     }
49753e9753d6SMatthew G. Knepley     /* Loop over fields */
49763e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
49773e9753d6SMatthew G. Knepley       PetscFE         fe;
49783e9753d6SMatthew G. Knepley       PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
4979148442b3SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
49803e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
49813e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
49823e9753d6SMatthew G. Knepley 
49836528b96dSMatthew G. Knepley       ierr = PetscDSGetDiscretization(ds, f, (PetscObject *) &fe);CHKERRQ(ierr);
49843e9753d6SMatthew G. Knepley       if (!fe) continue;
49853e9753d6SMatthew G. Knepley       ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
49863e9753d6SMatthew G. Knepley       ierr = PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
49873e9753d6SMatthew G. Knepley       ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
49883e9753d6SMatthew G. Knepley       blockSize = Nb;
49893e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
49903e9753d6SMatthew G. Knepley       ierr      = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
49913e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
49923e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
49933e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
49943e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
4995148442b3SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(geom,0,offset,&chunkGeom);CHKERRQ(ierr);
4996148442b3SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(geom,offset,numCells,&remGeom);CHKERRQ(ierr);
499779b86a80SMatthew G. Knepley       chunkGeom->isHybrid = remGeom->isHybrid = PETSC_TRUE;
49986528b96dSMatthew G. Knepley       if (f == Nf-1) {
49996528b96dSMatthew G. Knepley         key[2].field = f;
50006528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateHybridResidual(ds, key[2], Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, elemVec);CHKERRQ(ierr);
5001148442b3SMatthew G. Knepley         ierr = PetscFEIntegrateHybridResidual(ds, key[2], Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, &elemVec[offset*totDim]);CHKERRQ(ierr);
50026528b96dSMatthew G. Knepley       } else {
50036528b96dSMatthew G. Knepley         key[0].field = f;
50046528b96dSMatthew G. Knepley         key[1].field = f;
50056528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateHybridResidual(ds, key[0], Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, elemVec);CHKERRQ(ierr);
50066528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateHybridResidual(ds, key[1], Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, elemVec);CHKERRQ(ierr);
5007148442b3SMatthew G. Knepley         ierr = PetscFEIntegrateHybridResidual(ds, key[0], Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, &elemVec[offset*totDim]);CHKERRQ(ierr);
5008148442b3SMatthew G. Knepley         ierr = PetscFEIntegrateHybridResidual(ds, key[1], Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, &elemVec[offset*totDim]);CHKERRQ(ierr);
50096528b96dSMatthew G. Knepley       }
5010148442b3SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(geom,offset,numCells,&remGeom);CHKERRQ(ierr);
5011148442b3SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(geom,0,offset,&chunkGeom);CHKERRQ(ierr);
50123e9753d6SMatthew G. Knepley     }
50133e9753d6SMatthew G. Knepley     /* Add elemVec to locX */
50143e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
50153e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
50163e9753d6SMatthew G. Knepley       const PetscInt cind = c - cStart;
50173e9753d6SMatthew G. Knepley 
50183e9753d6SMatthew G. Knepley       if (mesh->printFEM > 1) {ierr = DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]);CHKERRQ(ierr);}
50193e9753d6SMatthew G. Knepley       if (ghostLabel) {
50203e9753d6SMatthew G. Knepley         PetscInt ghostVal;
50213e9753d6SMatthew G. Knepley 
50223e9753d6SMatthew G. Knepley         ierr = DMLabelGetValue(ghostLabel,cell,&ghostVal);CHKERRQ(ierr);
50233e9753d6SMatthew G. Knepley         if (ghostVal > 0) continue;
50243e9753d6SMatthew G. Knepley       }
50253e9753d6SMatthew G. Knepley       ierr = DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES);CHKERRQ(ierr);
50263e9753d6SMatthew G. Knepley     }
50273e9753d6SMatthew G. Knepley   }
502804c51a94SMatthew G. Knepley   ierr = DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]);CHKERRQ(ierr);
502904c51a94SMatthew G. Knepley   ierr = DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a);CHKERRQ(ierr);
50303e9753d6SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);CHKERRQ(ierr);
50313e9753d6SMatthew G. Knepley   ierr = PetscFree(faces);CHKERRQ(ierr);
50323e9753d6SMatthew G. Knepley   ierr = ISDestroy(&chunkIS);CHKERRQ(ierr);
50333e9753d6SMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
50343e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
50353e9753d6SMatthew G. Knepley     ierr = DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);CHKERRQ(ierr);
50363e9753d6SMatthew G. Knepley     ierr = PetscQuadratureDestroy(&affineQuad);CHKERRQ(ierr);
50373e9753d6SMatthew G. Knepley   } else {
50383e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
50393e9753d6SMatthew G. Knepley       if (geoms) {ierr = DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);CHKERRQ(ierr);}
50403e9753d6SMatthew G. Knepley       if (quads) {ierr = PetscQuadratureDestroy(&quads[f]);CHKERRQ(ierr);}
50413e9753d6SMatthew G. Knepley     }
50423e9753d6SMatthew G. Knepley     ierr = PetscFree2(quads,geoms);CHKERRQ(ierr);
50433e9753d6SMatthew G. Knepley   }
50443e9753d6SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0);CHKERRQ(ierr);
50453e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
50463e9753d6SMatthew G. Knepley }
50473e9753d6SMatthew G. Knepley 
504845480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS)
50493e9753d6SMatthew G. Knepley {
50503e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *) dm->data;
50513e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL, tdm;
50523e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
50533e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
50543e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
50553e9753d6SMatthew G. Knepley   PetscSection    globalSection, subSection = NULL;
50563e9753d6SMatthew G. Knepley   Vec             locA = NULL, tv;
50573e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL;
50583e9753d6SMatthew G. Knepley   PetscInt        v;
50593e9753d6SMatthew G. Knepley   PetscInt        Nf, totDim, totDimAux = 0;
50603e9753d6SMatthew G. Knepley   PetscBool       isMatISP, transform;
50613e9753d6SMatthew G. Knepley   PetscErrorCode  ierr;
50623e9753d6SMatthew G. Knepley 
50633e9753d6SMatthew G. Knepley   PetscFunctionBegin;
50643e9753d6SMatthew G. Knepley   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
50653e9753d6SMatthew G. Knepley   ierr = DMHasBasisTransform(dm, &transform);CHKERRQ(ierr);
50663e9753d6SMatthew G. Knepley   ierr = DMGetBasisTransformDM_Internal(dm, &tdm);CHKERRQ(ierr);
50673e9753d6SMatthew G. Knepley   ierr = DMGetBasisTransformVec_Internal(dm, &tv);CHKERRQ(ierr);
50683e9753d6SMatthew G. Knepley   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
50693e9753d6SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
50703e9753d6SMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
50713e9753d6SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
507204c51a94SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, label, values[0], &locA);CHKERRQ(ierr);
50733e9753d6SMatthew G. Knepley   if (locA) {
50743e9753d6SMatthew G. Knepley     DM dmAux;
50753e9753d6SMatthew G. Knepley 
50763e9753d6SMatthew G. Knepley     ierr = VecGetDM(locA, &dmAux);CHKERRQ(ierr);
50773e9753d6SMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux, dm, &encAux);CHKERRQ(ierr);
50783e9753d6SMatthew G. Knepley     ierr = DMConvert(dmAux, DMPLEX, &plexA);CHKERRQ(ierr);
50793e9753d6SMatthew G. Knepley     ierr = DMGetDS(plexA, &probAux);CHKERRQ(ierr);
50803e9753d6SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
50813e9753d6SMatthew G. Knepley     ierr = DMGetLocalSection(plexA, &sectionAux);CHKERRQ(ierr);
50823e9753d6SMatthew G. Knepley   }
50833e9753d6SMatthew G. Knepley 
50843e9753d6SMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);CHKERRQ(ierr);
50853e9753d6SMatthew G. Knepley   ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);
50863e9753d6SMatthew G. Knepley   if (isMatISP) {ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);}
50873e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
50883e9753d6SMatthew G. Knepley     PetscFEGeom     *fgeom;
50893e9753d6SMatthew G. Knepley     PetscInt         maxDegree;
50903e9753d6SMatthew G. Knepley     PetscQuadrature  qGeom = NULL;
50913e9753d6SMatthew G. Knepley     IS               pointIS;
50923e9753d6SMatthew G. Knepley     const PetscInt  *points;
509306ad1575SMatthew G. Knepley     PetscFormKey key;
50943e9753d6SMatthew G. Knepley     PetscInt         numFaces, face, Nq;
50953e9753d6SMatthew G. Knepley 
509645480ffeSMatthew G. Knepley     key.label = label;
509745480ffeSMatthew G. Knepley     key.value = values[v];
509806ad1575SMatthew G. Knepley     key.part  = 0;
50993e9753d6SMatthew G. Knepley     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
51003e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
51013e9753d6SMatthew G. Knepley     {
51023e9753d6SMatthew G. Knepley       IS isectIS;
51033e9753d6SMatthew G. Knepley 
51043e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
51053e9753d6SMatthew G. Knepley       ierr = ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS);CHKERRQ(ierr);
51063e9753d6SMatthew G. Knepley       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
51073e9753d6SMatthew G. Knepley       pointIS = isectIS;
51083e9753d6SMatthew G. Knepley     }
51093e9753d6SMatthew G. Knepley     ierr = ISGetLocalSize(pointIS, &numFaces);CHKERRQ(ierr);
51103e9753d6SMatthew G. Knepley     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
51113e9753d6SMatthew G. Knepley     ierr = PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim*totDim, &elemMat, locA ? numFaces*totDimAux : 0, &a);CHKERRQ(ierr);
51123e9753d6SMatthew G. Knepley     ierr = DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree);CHKERRQ(ierr);
51133e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
51143e9753d6SMatthew G. Knepley       ierr = DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom);CHKERRQ(ierr);
51153e9753d6SMatthew G. Knepley     }
51163e9753d6SMatthew G. Knepley     if (!qGeom) {
51173e9753d6SMatthew G. Knepley       PetscFE fe;
51183e9753d6SMatthew G. Knepley 
51193e9753d6SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);CHKERRQ(ierr);
51203e9753d6SMatthew G. Knepley       ierr = PetscFEGetFaceQuadrature(fe, &qGeom);CHKERRQ(ierr);
51213e9753d6SMatthew G. Knepley       ierr = PetscObjectReference((PetscObject)qGeom);CHKERRQ(ierr);
51223e9753d6SMatthew G. Knepley     }
51233e9753d6SMatthew G. Knepley     ierr = PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
51243e9753d6SMatthew G. Knepley     ierr = DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);CHKERRQ(ierr);
51253e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
5126f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
51273e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
5128f15274beSMatthew Knepley       PetscInt       i;
51293e9753d6SMatthew G. Knepley 
51303e9753d6SMatthew G. Knepley       ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
51313e9753d6SMatthew G. Knepley       ierr = DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x);CHKERRQ(ierr);
51323e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
51333e9753d6SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x);CHKERRQ(ierr);
51343e9753d6SMatthew G. Knepley       if (locX_t) {
51353e9753d6SMatthew G. Knepley         ierr = DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x);CHKERRQ(ierr);
51363e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
51373e9753d6SMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x);CHKERRQ(ierr);
51383e9753d6SMatthew G. Knepley       }
51393e9753d6SMatthew G. Knepley       if (locA) {
51403e9753d6SMatthew G. Knepley         PetscInt subp;
51413e9753d6SMatthew G. Knepley         ierr = DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp);CHKERRQ(ierr);
51423e9753d6SMatthew G. Knepley         ierr = DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x);CHKERRQ(ierr);
51433e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
51443e9753d6SMatthew G. Knepley         ierr = DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x);CHKERRQ(ierr);
51453e9753d6SMatthew G. Knepley       }
51463e9753d6SMatthew G. Knepley     }
51473e9753d6SMatthew G. Knepley     ierr = PetscArrayzero(elemMat, numFaces*totDim*totDim);CHKERRQ(ierr);
51483e9753d6SMatthew G. Knepley     {
51493e9753d6SMatthew G. Knepley       PetscFE         fe;
51503e9753d6SMatthew G. Knepley       PetscInt        Nb;
51513e9753d6SMatthew G. Knepley       /* Conforming batches */
51523e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
51533e9753d6SMatthew G. Knepley       /* Remainder */
51543e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL;
51553e9753d6SMatthew G. Knepley       PetscInt        fieldJ, Nr, offset;
51563e9753d6SMatthew G. Knepley 
51573e9753d6SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);CHKERRQ(ierr);
51583e9753d6SMatthew G. Knepley       ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
51593e9753d6SMatthew G. Knepley       ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
51603e9753d6SMatthew G. Knepley       blockSize = Nb;
51613e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
51623e9753d6SMatthew G. Knepley       ierr = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
51633e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches*batchSize);
51643e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
51653e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches*batchSize);
51663e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
51673e9753d6SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom);CHKERRQ(ierr);
51683e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
516945480ffeSMatthew G. Knepley         key.field = fieldI*Nf+fieldJ;
517045480ffeSMatthew G. Knepley         ierr = PetscFEIntegrateBdJacobian(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat);CHKERRQ(ierr);
51713e9753d6SMatthew G. Knepley       }
51723e9753d6SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom);CHKERRQ(ierr);
51733e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
517445480ffeSMatthew G. Knepley         key.field = fieldI*Nf+fieldJ;
517545480ffeSMatthew G. Knepley         ierr = PetscFEIntegrateBdJacobian(prob, wf, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, X_tShift, &elemMat[offset*totDim*totDim]);CHKERRQ(ierr);
51763e9753d6SMatthew G. Knepley       }
51773e9753d6SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom);CHKERRQ(ierr);
51783e9753d6SMatthew G. Knepley     }
51793e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
51803e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
51813e9753d6SMatthew G. Knepley 
51823e9753d6SMatthew G. Knepley       /* Transform to global basis before insertion in Jacobian */
51833e9753d6SMatthew G. Knepley       ierr = DMPlexGetSupport(plex, point, &support);CHKERRQ(ierr);
51843e9753d6SMatthew G. Knepley       if (transform) {ierr = DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face*totDim*totDim]);CHKERRQ(ierr);}
51853e9753d6SMatthew G. Knepley       if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face*totDim*totDim]);CHKERRQ(ierr);}
51863e9753d6SMatthew G. Knepley       if (!isMatISP) {
51873e9753d6SMatthew G. Knepley         ierr = DMPlexMatSetClosure(plex, section, globalSection, JacP, support[0], &elemMat[face*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
51883e9753d6SMatthew G. Knepley       } else {
51893e9753d6SMatthew G. Knepley         Mat lJ;
51903e9753d6SMatthew G. Knepley 
51913e9753d6SMatthew G. Knepley         ierr = MatISGetLocalMat(JacP, &lJ);CHKERRQ(ierr);
51923e9753d6SMatthew G. Knepley         ierr = DMPlexMatSetClosure(plex, section, subSection, lJ, support[0], &elemMat[face*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
51933e9753d6SMatthew G. Knepley       }
51943e9753d6SMatthew G. Knepley     }
51953e9753d6SMatthew G. Knepley     ierr = DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);CHKERRQ(ierr);
51963e9753d6SMatthew G. Knepley     ierr = PetscQuadratureDestroy(&qGeom);CHKERRQ(ierr);
51973e9753d6SMatthew G. Knepley     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
51983e9753d6SMatthew G. Knepley     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
51993e9753d6SMatthew G. Knepley     ierr = PetscFree4(u, u_t, elemMat, a);CHKERRQ(ierr);
52003e9753d6SMatthew G. Knepley   }
52013e9753d6SMatthew G. Knepley   if (plex)  {ierr = DMDestroy(&plex);CHKERRQ(ierr);}
52023e9753d6SMatthew G. Knepley   if (plexA) {ierr = DMDestroy(&plexA);CHKERRQ(ierr);}
52033e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52043e9753d6SMatthew G. Knepley }
52053e9753d6SMatthew G. Knepley 
520645480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP)
52073e9753d6SMatthew G. Knepley {
52083e9753d6SMatthew G. Knepley   DMField        coordField;
52093e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
52103e9753d6SMatthew G. Knepley   IS             facetIS;
52113e9753d6SMatthew G. Knepley   PetscInt       dim;
52123e9753d6SMatthew G. Knepley   PetscErrorCode ierr;
52133e9753d6SMatthew G. Knepley 
52143e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52153e9753d6SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
52163e9753d6SMatthew G. Knepley   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
52173e9753d6SMatthew G. Knepley   ierr = DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);CHKERRQ(ierr);
52183e9753d6SMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
521945480ffeSMatthew G. Knepley   ierr = DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS);CHKERRQ(ierr);
52203e9753d6SMatthew G. Knepley   ierr = ISDestroy(&facetIS);CHKERRQ(ierr);
52213e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52223e9753d6SMatthew G. Knepley }
52233e9753d6SMatthew G. Knepley 
52243e9753d6SMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user)
52253e9753d6SMatthew G. Knepley {
52263e9753d6SMatthew G. Knepley   PetscDS          prob;
52273e9753d6SMatthew G. Knepley   PetscInt         dim, numBd, bd;
52283e9753d6SMatthew G. Knepley   DMLabel          depthLabel;
52293e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
52303e9753d6SMatthew G. Knepley   IS               facetIS;
52313e9753d6SMatthew G. Knepley   PetscErrorCode   ierr;
52323e9753d6SMatthew G. Knepley 
52333e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52343e9753d6SMatthew G. Knepley   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
52353e9753d6SMatthew G. Knepley   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
52363e9753d6SMatthew G. Knepley   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
52373e9753d6SMatthew G. Knepley   ierr = DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);CHKERRQ(ierr);
52383e9753d6SMatthew G. Knepley   ierr = PetscDSGetNumBoundary(prob, &numBd);CHKERRQ(ierr);
52393e9753d6SMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
52403e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
524145480ffeSMatthew G. Knepley     PetscWeakForm           wf;
52423e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
52433e9753d6SMatthew G. Knepley     DMLabel                 label;
52443e9753d6SMatthew G. Knepley     const PetscInt         *values;
52453e9753d6SMatthew G. Knepley     PetscInt                fieldI, numValues;
52463e9753d6SMatthew G. Knepley     PetscObject             obj;
52473e9753d6SMatthew G. Knepley     PetscClassId            id;
52483e9753d6SMatthew G. Knepley 
524945480ffeSMatthew G. Knepley     ierr = PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL);CHKERRQ(ierr);
52503e9753d6SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, fieldI, &obj);CHKERRQ(ierr);
52513e9753d6SMatthew G. Knepley     ierr = PetscObjectGetClassId(obj, &id);CHKERRQ(ierr);
52523e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
525345480ffeSMatthew G. Knepley     ierr = DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS);CHKERRQ(ierr);
52543e9753d6SMatthew G. Knepley   }
52553e9753d6SMatthew G. Knepley   ierr = ISDestroy(&facetIS);CHKERRQ(ierr);
52563e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52573e9753d6SMatthew G. Knepley }
52583e9753d6SMatthew G. Knepley 
525906ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP,void *user)
52603e9753d6SMatthew G. Knepley {
52613e9753d6SMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *) dm->data;
52623e9753d6SMatthew G. Knepley   const char     *name  = "Jacobian";
52639a2a23afSMatthew G. Knepley   DM              dmAux = NULL, plex, tdm;
52643e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
52653e9753d6SMatthew G. Knepley   Vec             A, tv;
52663e9753d6SMatthew G. Knepley   DMField         coordField;
52673e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
52683e9753d6SMatthew G. Knepley   PetscSection    section, globalSection, subSection, sectionAux;
52693e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
52703e9753d6SMatthew G. Knepley   const PetscInt *cells;
52713e9753d6SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
52723e9753d6SMatthew G. Knepley   PetscInt        totDim, totDimAux, cStart, cEnd, numCells, c;
52733e9753d6SMatthew G. Knepley   PetscBool       isMatIS, isMatISP, hasJac, hasPrec, hasDyn, hasFV = PETSC_FALSE, transform;
52743e9753d6SMatthew G. Knepley   PetscErrorCode  ierr;
52753e9753d6SMatthew G. Knepley 
52763e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52773e9753d6SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
52783e9753d6SMatthew G. Knepley   ierr = ISGetLocalSize(cellIS, &numCells);CHKERRQ(ierr);
52793e9753d6SMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
52803e9753d6SMatthew G. Knepley   ierr = DMHasBasisTransform(dm, &transform);CHKERRQ(ierr);
52813e9753d6SMatthew G. Knepley   ierr = DMGetBasisTransformDM_Internal(dm, &tdm);CHKERRQ(ierr);
52823e9753d6SMatthew G. Knepley   ierr = DMGetBasisTransformVec_Internal(dm, &tv);CHKERRQ(ierr);
52833e9753d6SMatthew G. Knepley   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
52843e9753d6SMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);CHKERRQ(ierr);
52853e9753d6SMatthew G. Knepley   ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);
52863e9753d6SMatthew G. Knepley   if (isMatISP) {ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);}
52873e9753d6SMatthew G. Knepley   ierr = DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob);CHKERRQ(ierr);
52883e9753d6SMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
52893e9753d6SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
52903e9753d6SMatthew G. Knepley   ierr = PetscDSHasJacobian(prob, &hasJac);CHKERRQ(ierr);
52913e9753d6SMatthew G. Knepley   ierr = PetscDSHasJacobianPreconditioner(prob, &hasPrec);CHKERRQ(ierr);
52923e9753d6SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
52933e9753d6SMatthew G. Knepley      only assemble the Jacobian */
52943e9753d6SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
52953e9753d6SMatthew G. Knepley   ierr = PetscDSHasDynamicJacobian(prob, &hasDyn);CHKERRQ(ierr);
52963e9753d6SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
529704c51a94SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, key.label, key.value, &A);CHKERRQ(ierr);
52989a2a23afSMatthew G. Knepley   if (A) {
52999a2a23afSMatthew G. Knepley     ierr = VecGetDM(A, &dmAux);CHKERRQ(ierr);
53003e9753d6SMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux, dm, &encAux);CHKERRQ(ierr);
53013e9753d6SMatthew G. Knepley     ierr = DMConvert(dmAux, DMPLEX, &plex);CHKERRQ(ierr);
53023e9753d6SMatthew G. Knepley     ierr = DMGetLocalSection(plex, &sectionAux);CHKERRQ(ierr);
53033e9753d6SMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
53043e9753d6SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
53053e9753d6SMatthew G. Knepley   }
53063e9753d6SMatthew G. Knepley   ierr = PetscMalloc5(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,hasJac ? numCells*totDim*totDim : 0,&elemMat,hasPrec ? numCells*totDim*totDim : 0, &elemMatP,hasDyn ? numCells*totDim*totDim : 0, &elemMatD);CHKERRQ(ierr);
53073e9753d6SMatthew G. Knepley   if (dmAux) {ierr = PetscMalloc1(numCells*totDimAux, &a);CHKERRQ(ierr);}
53083e9753d6SMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
53093e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
53103e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
53113e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
53123e9753d6SMatthew G. Knepley     PetscScalar   *x = NULL,  *x_t = NULL;
53133e9753d6SMatthew G. Knepley     PetscInt       i;
53143e9753d6SMatthew G. Knepley 
53153e9753d6SMatthew G. Knepley     ierr = DMPlexVecGetClosure(dm, section, X, cell, NULL, &x);CHKERRQ(ierr);
53163e9753d6SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
53173e9753d6SMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x);CHKERRQ(ierr);
53183e9753d6SMatthew G. Knepley     if (X_t) {
53193e9753d6SMatthew G. Knepley       ierr = DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t);CHKERRQ(ierr);
53203e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
53213e9753d6SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t);CHKERRQ(ierr);
53223e9753d6SMatthew G. Knepley     }
53233e9753d6SMatthew G. Knepley     if (dmAux) {
53243e9753d6SMatthew G. Knepley       PetscInt subcell;
53253e9753d6SMatthew G. Knepley       ierr = DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell);CHKERRQ(ierr);
53263e9753d6SMatthew G. Knepley       ierr = DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x);CHKERRQ(ierr);
53273e9753d6SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
53283e9753d6SMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x);CHKERRQ(ierr);
53293e9753d6SMatthew G. Knepley     }
53303e9753d6SMatthew G. Knepley   }
53313e9753d6SMatthew G. Knepley   if (hasJac)  {ierr = PetscArrayzero(elemMat,  numCells*totDim*totDim);CHKERRQ(ierr);}
53323e9753d6SMatthew G. Knepley   if (hasPrec) {ierr = PetscArrayzero(elemMatP, numCells*totDim*totDim);CHKERRQ(ierr);}
53333e9753d6SMatthew G. Knepley   if (hasDyn)  {ierr = PetscArrayzero(elemMatD, numCells*totDim*totDim);CHKERRQ(ierr);}
53343e9753d6SMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
53353e9753d6SMatthew G. Knepley     PetscClassId    id;
53363e9753d6SMatthew G. Knepley     PetscFE         fe;
53373e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
53383e9753d6SMatthew G. Knepley     PetscInt        Nb;
53393e9753d6SMatthew G. Knepley     /* Conforming batches */
53403e9753d6SMatthew G. Knepley     PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
53413e9753d6SMatthew G. Knepley     /* Remainder */
53423e9753d6SMatthew G. Knepley     PetscInt        Nr, offset, Nq;
53433e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
53443e9753d6SMatthew G. Knepley     PetscFEGeom     *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
53453e9753d6SMatthew G. Knepley 
53463e9753d6SMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);CHKERRQ(ierr);
53473e9753d6SMatthew G. Knepley     ierr = PetscObjectGetClassId((PetscObject) fe, &id);CHKERRQ(ierr);
53483e9753d6SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; continue;}
53493e9753d6SMatthew G. Knepley     ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
53503e9753d6SMatthew G. Knepley     ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
53513e9753d6SMatthew G. Knepley     ierr = DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);CHKERRQ(ierr);
53523e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
53533e9753d6SMatthew G. Knepley       ierr = DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom);CHKERRQ(ierr);
53543e9753d6SMatthew G. Knepley     }
53553e9753d6SMatthew G. Knepley     if (!qGeom) {
53563e9753d6SMatthew G. Knepley       ierr = PetscFEGetQuadrature(fe,&qGeom);CHKERRQ(ierr);
53573e9753d6SMatthew G. Knepley       ierr = PetscObjectReference((PetscObject)qGeom);CHKERRQ(ierr);
53583e9753d6SMatthew G. Knepley     }
53593e9753d6SMatthew G. Knepley     ierr = PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
53603e9753d6SMatthew G. Knepley     ierr = DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);CHKERRQ(ierr);
53613e9753d6SMatthew G. Knepley     blockSize = Nb;
53623e9753d6SMatthew G. Knepley     batchSize = numBlocks * blockSize;
53633e9753d6SMatthew G. Knepley     ierr = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
53643e9753d6SMatthew G. Knepley     numChunks = numCells / (numBatches*batchSize);
53653e9753d6SMatthew G. Knepley     Ne        = numChunks*numBatches*batchSize;
53663e9753d6SMatthew G. Knepley     Nr        = numCells % (numBatches*batchSize);
53673e9753d6SMatthew G. Knepley     offset    = numCells - Nr;
53683e9753d6SMatthew G. Knepley     ierr = PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);CHKERRQ(ierr);
53693e9753d6SMatthew G. Knepley     ierr = PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom);CHKERRQ(ierr);
53703e9753d6SMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
53716528b96dSMatthew G. Knepley       key.field = fieldI*Nf+fieldJ;
53723e9753d6SMatthew G. Knepley       if (hasJac) {
53736528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat);CHKERRQ(ierr);
53746528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]);CHKERRQ(ierr);
53753e9753d6SMatthew G. Knepley       }
53763e9753d6SMatthew G. Knepley       if (hasPrec) {
53776528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP);CHKERRQ(ierr);
53786528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatP[offset*totDim*totDim]);CHKERRQ(ierr);
53793e9753d6SMatthew G. Knepley       }
53803e9753d6SMatthew G. Knepley       if (hasDyn) {
53816528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD);CHKERRQ(ierr);
53826528b96dSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]);CHKERRQ(ierr);
53833e9753d6SMatthew G. Knepley       }
53843e9753d6SMatthew G. Knepley     }
53853e9753d6SMatthew G. Knepley     ierr = PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom);CHKERRQ(ierr);
53863e9753d6SMatthew G. Knepley     ierr = PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom);CHKERRQ(ierr);
53873e9753d6SMatthew G. Knepley     ierr = DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);CHKERRQ(ierr);
53883e9753d6SMatthew G. Knepley     ierr = PetscQuadratureDestroy(&qGeom);CHKERRQ(ierr);
53893e9753d6SMatthew G. Knepley   }
53903e9753d6SMatthew G. Knepley   /*   Add contribution from X_t */
53913e9753d6SMatthew G. Knepley   if (hasDyn) {for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
53923e9753d6SMatthew G. Knepley   if (hasFV) {
53933e9753d6SMatthew G. Knepley     PetscClassId id;
53943e9753d6SMatthew G. Knepley     PetscFV      fv;
53953e9753d6SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1, fc, f;
53963e9753d6SMatthew G. Knepley 
53973e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
53983e9753d6SMatthew G. Knepley       ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv);CHKERRQ(ierr);
53993e9753d6SMatthew G. Knepley       ierr = PetscDSGetFieldOffset(prob, fieldI, &offsetI);CHKERRQ(ierr);
54003e9753d6SMatthew G. Knepley       ierr = PetscObjectGetClassId((PetscObject) fv, &id);CHKERRQ(ierr);
54013e9753d6SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
54023e9753d6SMatthew G. Knepley       /* Put in the identity */
54033e9753d6SMatthew G. Knepley       ierr = PetscFVGetNumComponents(fv, &NcI);CHKERRQ(ierr);
54043e9753d6SMatthew G. Knepley       for (c = cStart; c < cEnd; ++c) {
54053e9753d6SMatthew G. Knepley         const PetscInt cind    = c - cStart;
54063e9753d6SMatthew G. Knepley         const PetscInt eOffset = cind*totDim*totDim;
54073e9753d6SMatthew G. Knepley         for (fc = 0; fc < NcI; ++fc) {
54083e9753d6SMatthew G. Knepley           for (f = 0; f < NbI; ++f) {
54093e9753d6SMatthew G. Knepley             const PetscInt i = offsetI + f*NcI+fc;
54103e9753d6SMatthew G. Knepley             if (hasPrec) {
54113e9753d6SMatthew G. Knepley               if (hasJac) {elemMat[eOffset+i*totDim+i] = 1.0;}
54123e9753d6SMatthew G. Knepley               elemMatP[eOffset+i*totDim+i] = 1.0;
54133e9753d6SMatthew G. Knepley             } else {elemMat[eOffset+i*totDim+i] = 1.0;}
54143e9753d6SMatthew G. Knepley           }
54153e9753d6SMatthew G. Knepley         }
54163e9753d6SMatthew G. Knepley       }
54173e9753d6SMatthew G. Knepley     }
54183e9753d6SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
54193e9753d6SMatthew G. Knepley     ierr = MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE);CHKERRQ(ierr);
54203e9753d6SMatthew G. Knepley   }
54213e9753d6SMatthew G. Knepley   /* Insert values into matrix */
54223e9753d6SMatthew G. Knepley   isMatIS = PETSC_FALSE;
54233e9753d6SMatthew G. Knepley   if (hasPrec && hasJac) {
54243e9753d6SMatthew G. Knepley     ierr = PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatIS);CHKERRQ(ierr);
54253e9753d6SMatthew G. Knepley   }
54263e9753d6SMatthew G. Knepley   if (isMatIS && !subSection) {
54273e9753d6SMatthew G. Knepley     ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
54283e9753d6SMatthew G. Knepley   }
54293e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
54303e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
54313e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
54323e9753d6SMatthew G. Knepley 
54333e9753d6SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
54343e9753d6SMatthew G. Knepley     if (transform) {ierr = DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind*totDim*totDim]);CHKERRQ(ierr);}
54353e9753d6SMatthew G. Knepley     if (hasPrec) {
54363e9753d6SMatthew G. Knepley       if (hasJac) {
54373e9753d6SMatthew G. Knepley         if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]);CHKERRQ(ierr);}
54383e9753d6SMatthew G. Knepley         if (!isMatIS) {
54393e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
54403e9753d6SMatthew G. Knepley         } else {
54413e9753d6SMatthew G. Knepley           Mat lJ;
54423e9753d6SMatthew G. Knepley 
54433e9753d6SMatthew G. Knepley           ierr = MatISGetLocalMat(Jac,&lJ);CHKERRQ(ierr);
54443e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(dm, section, subSection, lJ, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
54453e9753d6SMatthew G. Knepley         }
54463e9753d6SMatthew G. Knepley       }
54473e9753d6SMatthew G. Knepley       if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]);CHKERRQ(ierr);}
54483e9753d6SMatthew G. Knepley       if (!isMatISP) {
54493e9753d6SMatthew G. Knepley         ierr = DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
54503e9753d6SMatthew G. Knepley       } else {
54513e9753d6SMatthew G. Knepley         Mat lJ;
54523e9753d6SMatthew G. Knepley 
54533e9753d6SMatthew G. Knepley         ierr = MatISGetLocalMat(JacP,&lJ);CHKERRQ(ierr);
54543e9753d6SMatthew G. Knepley         ierr = DMPlexMatSetClosure(dm, section, subSection, lJ, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
54553e9753d6SMatthew G. Knepley       }
54563e9753d6SMatthew G. Knepley     } else {
54573e9753d6SMatthew G. Knepley       if (hasJac) {
54583e9753d6SMatthew G. Knepley         if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]);CHKERRQ(ierr);}
54593e9753d6SMatthew G. Knepley         if (!isMatISP) {
54603e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
54613e9753d6SMatthew G. Knepley         } else {
54623e9753d6SMatthew G. Knepley           Mat lJ;
54633e9753d6SMatthew G. Knepley 
54643e9753d6SMatthew G. Knepley           ierr = MatISGetLocalMat(JacP,&lJ);CHKERRQ(ierr);
54653e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(dm, section, subSection, lJ, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
54663e9753d6SMatthew G. Knepley         }
54673e9753d6SMatthew G. Knepley       }
54683e9753d6SMatthew G. Knepley     }
54693e9753d6SMatthew G. Knepley   }
54703e9753d6SMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
54713e9753d6SMatthew G. Knepley   if (hasFV) {ierr = MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE);CHKERRQ(ierr);}
54723e9753d6SMatthew G. Knepley   ierr = PetscFree5(u,u_t,elemMat,elemMatP,elemMatD);CHKERRQ(ierr);
54733e9753d6SMatthew G. Knepley   if (dmAux) {
54743e9753d6SMatthew G. Knepley     ierr = PetscFree(a);CHKERRQ(ierr);
54753e9753d6SMatthew G. Knepley     ierr = DMDestroy(&plex);CHKERRQ(ierr);
54763e9753d6SMatthew G. Knepley   }
54773e9753d6SMatthew G. Knepley   /* Compute boundary integrals */
54783e9753d6SMatthew G. Knepley   ierr = DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user);CHKERRQ(ierr);
54793e9753d6SMatthew G. Knepley   /* Assemble matrix */
54803e9753d6SMatthew G. Knepley   if (hasJac && hasPrec) {
54813e9753d6SMatthew G. Knepley     ierr = MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
54823e9753d6SMatthew G. Knepley     ierr = MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
54833e9753d6SMatthew G. Knepley   }
54843e9753d6SMatthew G. Knepley   ierr = MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
54853e9753d6SMatthew G. Knepley   ierr = MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
54863e9753d6SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
54873e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
54883e9753d6SMatthew G. Knepley }
54893e9753d6SMatthew G. Knepley 
549006ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
54913e9753d6SMatthew G. Knepley {
54923e9753d6SMatthew G. Knepley   DM_Plex         *mesh          = (DM_Plex *) dm->data;
54933e9753d6SMatthew G. Knepley   const char      *name          = "Hybrid Jacobian";
5494148442b3SMatthew G. Knepley   DM               dmAux[3]      = {NULL, NULL, NULL};
5495148442b3SMatthew G. Knepley   DMLabel          ghostLabel    = NULL;
54963e9753d6SMatthew G. Knepley   DM               plex          = NULL;
54973e9753d6SMatthew G. Knepley   DM               plexA         = NULL;
5498148442b3SMatthew G. Knepley   PetscDS          ds            = NULL;
5499148442b3SMatthew G. Knepley   PetscDS          dsAux[3]      = {NULL, NULL, NULL};
5500148442b3SMatthew G. Knepley   Vec              locA[3]       = {NULL, NULL, NULL};
55013e9753d6SMatthew G. Knepley   PetscSection     section       = NULL;
5502148442b3SMatthew G. Knepley   PetscSection     sectionAux[3] = {NULL, NULL, NULL};
55033e9753d6SMatthew G. Knepley   DMField          coordField    = NULL;
5504148442b3SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a[3];
55053e9753d6SMatthew G. Knepley   PetscScalar     *elemMat, *elemMatP;
5506148442b3SMatthew G. Knepley   PetscSection     globalSection, subSection;
55073e9753d6SMatthew G. Knepley   IS               chunkIS;
55083e9753d6SMatthew G. Knepley   const PetscInt  *cells;
55093e9753d6SMatthew G. Knepley   PetscInt        *faces;
55103e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
5511148442b3SMatthew G. Knepley   PetscInt         Nf, fieldI, fieldJ, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
55123e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
55133e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
55143e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
5515148442b3SMatthew G. Knepley   PetscBool        repeatKey = PETSC_FALSE, isMatIS = PETSC_FALSE, isMatISP = PETSC_FALSE, hasBdJac, hasBdPrec;
55163e9753d6SMatthew G. Knepley   PetscErrorCode   ierr;
55173e9753d6SMatthew G. Knepley 
55183e9753d6SMatthew G. Knepley   PetscFunctionBegin;
5519148442b3SMatthew G. Knepley   /* If keys are the same, both kernel will be run using the first key */
5520148442b3SMatthew G. Knepley   repeatKey = ((key[0].label == key[1].label) && (key[0].value == key[1].value)) ? PETSC_TRUE : PETSC_FALSE;
55213e9753d6SMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
55223e9753d6SMatthew G. Knepley   ierr = ISGetLocalSize(cellIS, &numCells);CHKERRQ(ierr);
55233e9753d6SMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
55243e9753d6SMatthew G. Knepley   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
55253e9753d6SMatthew G. Knepley   ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
55263e9753d6SMatthew G. Knepley   ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);
55273e9753d6SMatthew G. Knepley   ierr = DMGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
5528148442b3SMatthew G. Knepley   ierr = DMGetCellDS(dm, cStart, &ds);CHKERRQ(ierr);
5529148442b3SMatthew G. Knepley   ierr = PetscDSGetNumFields(ds, &Nf);CHKERRQ(ierr);
5530148442b3SMatthew G. Knepley   ierr = PetscDSGetTotalDimension(ds, &totDim);CHKERRQ(ierr);
5531148442b3SMatthew G. Knepley   ierr = PetscDSHasBdJacobian(ds, &hasBdJac);CHKERRQ(ierr);
5532148442b3SMatthew G. Knepley   ierr = PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec);CHKERRQ(ierr);
55333e9753d6SMatthew G. Knepley   ierr = PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);CHKERRQ(ierr);
55343e9753d6SMatthew G. Knepley   if (isMatISP)               {ierr = DMPlexGetSubdomainSection(plex, &subSection);CHKERRQ(ierr);}
55353e9753d6SMatthew G. Knepley   if (hasBdPrec && hasBdJac)  {ierr = PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatIS);CHKERRQ(ierr);}
55363e9753d6SMatthew G. Knepley   if (isMatIS && !subSection) {ierr = DMPlexGetSubdomainSection(plex, &subSection);CHKERRQ(ierr);}
5537148442b3SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, key[2].label, key[2].value, &locA[2]);CHKERRQ(ierr);
5538148442b3SMatthew G. Knepley   if (locA[2]) {
5539148442b3SMatthew G. Knepley     ierr = VecGetDM(locA[2], &dmAux[2]);CHKERRQ(ierr);
5540148442b3SMatthew G. Knepley     ierr = DMConvert(dmAux[2], DMPLEX, &plexA);CHKERRQ(ierr);
5541148442b3SMatthew G. Knepley     ierr = DMGetSection(dmAux[2], &sectionAux[2]);CHKERRQ(ierr);
5542148442b3SMatthew G. Knepley     ierr = DMGetCellDS(dmAux[2], cStart, &dsAux[2]);CHKERRQ(ierr);
5543148442b3SMatthew G. Knepley     ierr = PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]);CHKERRQ(ierr);
5544148442b3SMatthew G. Knepley     {
5545148442b3SMatthew G. Knepley       const PetscInt *cone;
5546148442b3SMatthew G. Knepley       PetscInt        c;
5547148442b3SMatthew G. Knepley 
5548148442b3SMatthew G. Knepley       ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
5549148442b3SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
5550148442b3SMatthew G. Knepley         const PetscInt *support;
5551148442b3SMatthew G. Knepley         PetscInt ssize, s;
5552148442b3SMatthew G. Knepley 
5553148442b3SMatthew G. Knepley         ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
5554148442b3SMatthew G. Knepley         ierr = DMPlexGetSupportSize(dm, cone[c], &ssize);CHKERRQ(ierr);
5555148442b3SMatthew G. Knepley         if (ssize != 2) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D from cell %D has support size %D != 2", cone[c], cStart, ssize);
5556148442b3SMatthew G. Knepley         if      (support[0] == cStart) s = 1;
5557148442b3SMatthew G. Knepley         else if (support[1] == cStart) s = 0;
5558148442b3SMatthew G. Knepley         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D does not have cell %D in its support", cone[c], cStart);
5559148442b3SMatthew G. Knepley         ierr = DMGetAuxiliaryVec(dm, key[c].label, key[c].value, &locA[c]);CHKERRQ(ierr);
5560148442b3SMatthew G. Knepley         if (locA[c]) {ierr = VecGetDM(locA[c], &dmAux[c]);CHKERRQ(ierr);}
5561148442b3SMatthew G. Knepley         else         {dmAux[c] = dmAux[2];}
5562148442b3SMatthew G. Knepley         ierr = DMGetCellDS(dmAux[c], support[s], &dsAux[c]);CHKERRQ(ierr);
5563148442b3SMatthew G. Knepley         ierr = PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]);CHKERRQ(ierr);
5564148442b3SMatthew G. Knepley       }
5565148442b3SMatthew G. Knepley     }
55663e9753d6SMatthew G. Knepley   }
55673e9753d6SMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
55683e9753d6SMatthew G. Knepley   ierr = DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree);CHKERRQ(ierr);
55693e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
55703e9753d6SMatthew G. Knepley     PetscInt f;
55713e9753d6SMatthew G. Knepley     ierr = PetscCalloc2(Nf, &quads, Nf, &geoms);CHKERRQ(ierr);
55723e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
55733e9753d6SMatthew G. Knepley       PetscFE fe;
55743e9753d6SMatthew G. Knepley 
5575148442b3SMatthew G. Knepley       ierr = PetscDSGetDiscretization(ds, f, (PetscObject *) &fe);CHKERRQ(ierr);
55763e9753d6SMatthew G. Knepley       if (fe) {
55773e9753d6SMatthew G. Knepley         ierr = PetscFEGetQuadrature(fe, &quads[f]);CHKERRQ(ierr);
55783e9753d6SMatthew G. Knepley         ierr = PetscObjectReference((PetscObject) quads[f]);CHKERRQ(ierr);
55793e9753d6SMatthew G. Knepley       }
55803e9753d6SMatthew G. Knepley     }
55813e9753d6SMatthew G. Knepley   }
55823e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
55833e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal) numCells)/cellChunkSize);
55843e9753d6SMatthew G. Knepley   ierr = PetscCalloc1(2*cellChunkSize, &faces);CHKERRQ(ierr);
55853e9753d6SMatthew G. Knepley   ierr = ISCreateGeneral(PETSC_COMM_SELF, cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS);CHKERRQ(ierr);
5586148442b3SMatthew G. Knepley   ierr = DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]);CHKERRQ(ierr);
5587148442b3SMatthew G. Knepley   ierr = DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a);CHKERRQ(ierr);
55883e9753d6SMatthew G. Knepley   ierr = DMGetWorkArray(dm, hasBdJac  ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMat);CHKERRQ(ierr);
55893e9753d6SMatthew G. Knepley   ierr = DMGetWorkArray(dm, hasBdPrec ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMatP);CHKERRQ(ierr);
55903e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
55913e9753d6SMatthew G. Knepley     PetscInt cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
55923e9753d6SMatthew G. Knepley 
55933e9753d6SMatthew G. Knepley     if (hasBdJac)  {ierr = PetscMemzero(elemMat,  numCells*totDim*totDim * sizeof(PetscScalar));CHKERRQ(ierr);}
55943e9753d6SMatthew G. Knepley     if (hasBdPrec) {ierr = PetscMemzero(elemMatP, numCells*totDim*totDim * sizeof(PetscScalar));CHKERRQ(ierr);}
55953e9753d6SMatthew G. Knepley     /* Get faces */
55963e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
55973e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
55983e9753d6SMatthew G. Knepley       const PetscInt *cone;
55993e9753d6SMatthew G. Knepley       ierr = DMPlexGetCone(plex, cell, &cone);CHKERRQ(ierr);
56003e9753d6SMatthew G. Knepley       faces[(c-cS)*2+0] = cone[0];
56013e9753d6SMatthew G. Knepley       faces[(c-cS)*2+1] = cone[1];
56023e9753d6SMatthew G. Knepley     }
56033e9753d6SMatthew G. Knepley     ierr = ISGeneralSetIndices(chunkIS, cellChunkSize, faces, PETSC_USE_POINTER);CHKERRQ(ierr);
56043e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
56053e9753d6SMatthew G. Knepley       if (!affineQuad) {ierr = DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad);CHKERRQ(ierr);}
56063e9753d6SMatthew G. Knepley       if (affineQuad)  {ierr = DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom);CHKERRQ(ierr);}
56073e9753d6SMatthew G. Knepley     } else {
56083e9753d6SMatthew G. Knepley       PetscInt f;
56093e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
56103e9753d6SMatthew G. Knepley         if (quads[f]) {ierr = DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]);CHKERRQ(ierr);}
56113e9753d6SMatthew G. Knepley       }
56123e9753d6SMatthew G. Knepley     }
56133e9753d6SMatthew G. Knepley 
56143e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
56153e9753d6SMatthew G. Knepley       PetscFE         feI;
56163e9753d6SMatthew G. Knepley       PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[fieldI];
56173e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
56183e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[fieldI];
56193e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
56203e9753d6SMatthew G. Knepley 
5621148442b3SMatthew G. Knepley       ierr = PetscDSGetDiscretization(ds, fieldI, (PetscObject *) &feI);CHKERRQ(ierr);
56223e9753d6SMatthew G. Knepley       if (!feI) continue;
56233e9753d6SMatthew G. Knepley       ierr = PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
56243e9753d6SMatthew G. Knepley       ierr = PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
56253e9753d6SMatthew G. Knepley       ierr = PetscFEGetDimension(feI, &Nb);CHKERRQ(ierr);
56263e9753d6SMatthew G. Knepley       blockSize = Nb;
56273e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
56283e9753d6SMatthew G. Knepley       ierr      = PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
56293e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
56303e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
56313e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
56323e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
56333e9753d6SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(geom,0,offset,&chunkGeom);CHKERRQ(ierr);
56343e9753d6SMatthew G. Knepley       ierr = PetscFEGeomGetChunk(geom,offset,numCells,&remGeom);CHKERRQ(ierr);
56353e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
56363e9753d6SMatthew G. Knepley         PetscFE feJ;
56373e9753d6SMatthew G. Knepley 
5638148442b3SMatthew G. Knepley         ierr = PetscDSGetDiscretization(ds, fieldJ, (PetscObject *) &feJ);CHKERRQ(ierr);
56393e9753d6SMatthew G. Knepley         if (!feJ) continue;
5640148442b3SMatthew G. Knepley         if (fieldI == Nf-1) {
5641148442b3SMatthew G. Knepley           key[2].field = fieldI*Nf+fieldJ;
56423e9753d6SMatthew G. Knepley           if (hasBdJac) {
5643148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMat);CHKERRQ(ierr);
5644148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, X_tShift, &elemMat[offset*totDim*totDim]);CHKERRQ(ierr);
56453e9753d6SMatthew G. Knepley           }
56463e9753d6SMatthew G. Knepley           if (hasBdPrec) {
5647148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatP);CHKERRQ(ierr);
5648148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, X_tShift, &elemMatP[offset*totDim*totDim]);CHKERRQ(ierr);
5649148442b3SMatthew G. Knepley           }
5650148442b3SMatthew G. Knepley         } else {
5651148442b3SMatthew G. Knepley           key[0].field = fieldI*Nf+fieldJ;
5652148442b3SMatthew G. Knepley           key[1].field = fieldI*Nf+fieldJ;
5653148442b3SMatthew G. Knepley           if (hasBdJac) {
5654148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMat);CHKERRQ(ierr);
5655148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, X_tShift, &elemMat[offset*totDim*totDim]);CHKERRQ(ierr);
5656148442b3SMatthew G. Knepley             if (!repeatKey) {
5657148442b3SMatthew G. Knepley               ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMat);CHKERRQ(ierr);
5658148442b3SMatthew G. Knepley               ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, X_tShift, &elemMat[offset*totDim*totDim]);CHKERRQ(ierr);
5659148442b3SMatthew G. Knepley             }
5660148442b3SMatthew G. Knepley           }
5661148442b3SMatthew G. Knepley           if (hasBdPrec) {
5662148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatP);CHKERRQ(ierr);
5663148442b3SMatthew G. Knepley             ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, X_tShift, &elemMatP[offset*totDim*totDim]);CHKERRQ(ierr);
5664148442b3SMatthew G. Knepley             if (!repeatKey) {
5665148442b3SMatthew G. Knepley               ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatP);CHKERRQ(ierr);
5666148442b3SMatthew G. Knepley               ierr = PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, X_tShift, &elemMatP[offset*totDim*totDim]);CHKERRQ(ierr);
5667148442b3SMatthew G. Knepley             }
5668148442b3SMatthew G. Knepley           }
56693e9753d6SMatthew G. Knepley         }
56703e9753d6SMatthew G. Knepley       }
56713e9753d6SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(geom,offset,numCells,&remGeom);CHKERRQ(ierr);
56723e9753d6SMatthew G. Knepley       ierr = PetscFEGeomRestoreChunk(geom,0,offset,&chunkGeom);CHKERRQ(ierr);
56733e9753d6SMatthew G. Knepley     }
56743e9753d6SMatthew G. Knepley     /* Insert values into matrix */
56753e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
56763e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
56773e9753d6SMatthew G. Knepley       const PetscInt cind = c - cS;
56783e9753d6SMatthew G. Knepley 
56793e9753d6SMatthew G. Knepley       if (hasBdPrec) {
56803e9753d6SMatthew G. Knepley         if (hasBdJac) {
56813e9753d6SMatthew G. Knepley           if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]);CHKERRQ(ierr);}
56823e9753d6SMatthew G. Knepley           if (!isMatIS) {
56833e9753d6SMatthew G. Knepley             ierr = DMPlexMatSetClosure(plex, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
56843e9753d6SMatthew G. Knepley           } else {
56853e9753d6SMatthew G. Knepley             Mat lJ;
56863e9753d6SMatthew G. Knepley 
56873e9753d6SMatthew G. Knepley             ierr = MatISGetLocalMat(Jac,&lJ);CHKERRQ(ierr);
56883e9753d6SMatthew G. Knepley             ierr = DMPlexMatSetClosure(plex, section, subSection, lJ, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
56893e9753d6SMatthew G. Knepley           }
56903e9753d6SMatthew G. Knepley         }
56913e9753d6SMatthew G. Knepley         if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]);CHKERRQ(ierr);}
56923e9753d6SMatthew G. Knepley         if (!isMatISP) {
56933e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
56943e9753d6SMatthew G. Knepley         } else {
56953e9753d6SMatthew G. Knepley           Mat lJ;
56963e9753d6SMatthew G. Knepley 
56973e9753d6SMatthew G. Knepley           ierr = MatISGetLocalMat(JacP,&lJ);CHKERRQ(ierr);
56983e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(plex, section, subSection, lJ, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
56993e9753d6SMatthew G. Knepley         }
57003e9753d6SMatthew G. Knepley       } else if (hasBdJac) {
57013e9753d6SMatthew G. Knepley         if (mesh->printFEM > 1) {ierr = DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]);CHKERRQ(ierr);}
57023e9753d6SMatthew G. Knepley         if (!isMatISP) {
57033e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
57043e9753d6SMatthew G. Knepley         } else {
57053e9753d6SMatthew G. Knepley           Mat lJ;
57063e9753d6SMatthew G. Knepley 
57073e9753d6SMatthew G. Knepley           ierr = MatISGetLocalMat(JacP,&lJ);CHKERRQ(ierr);
57083e9753d6SMatthew G. Knepley           ierr = DMPlexMatSetClosure(plex, section, subSection, lJ, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);CHKERRQ(ierr);
57093e9753d6SMatthew G. Knepley         }
57103e9753d6SMatthew G. Knepley       }
57113e9753d6SMatthew G. Knepley     }
57123e9753d6SMatthew G. Knepley   }
5713148442b3SMatthew G. Knepley   ierr = DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]);CHKERRQ(ierr);
5714148442b3SMatthew G. Knepley   ierr = DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a);CHKERRQ(ierr);
57153e9753d6SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, hasBdJac  ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMat);CHKERRQ(ierr);
57163e9753d6SMatthew G. Knepley   ierr = DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMatP);CHKERRQ(ierr);
57173e9753d6SMatthew G. Knepley   ierr = PetscFree(faces);CHKERRQ(ierr);
57183e9753d6SMatthew G. Knepley   ierr = ISDestroy(&chunkIS);CHKERRQ(ierr);
57193e9753d6SMatthew G. Knepley   ierr = ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
57203e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
57213e9753d6SMatthew G. Knepley     ierr = DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);CHKERRQ(ierr);
57223e9753d6SMatthew G. Knepley     ierr = PetscQuadratureDestroy(&affineQuad);CHKERRQ(ierr);
57233e9753d6SMatthew G. Knepley   } else {
57243e9753d6SMatthew G. Knepley     PetscInt f;
57253e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
57263e9753d6SMatthew G. Knepley       if (geoms) {ierr = DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE, &geoms[f]);CHKERRQ(ierr);}
57273e9753d6SMatthew G. Knepley       if (quads) {ierr = PetscQuadratureDestroy(&quads[f]);CHKERRQ(ierr);}
57283e9753d6SMatthew G. Knepley     }
57293e9753d6SMatthew G. Knepley     ierr = PetscFree2(quads,geoms);CHKERRQ(ierr);
57303e9753d6SMatthew G. Knepley   }
5731148442b3SMatthew G. Knepley   if (dmAux[2]) {ierr = DMDestroy(&plexA);CHKERRQ(ierr);}
57323e9753d6SMatthew G. Knepley   ierr = DMDestroy(&plex);CHKERRQ(ierr);
57333e9753d6SMatthew G. Knepley   /* Assemble matrix */
57343e9753d6SMatthew G. Knepley   if (hasBdJac && hasBdPrec) {
57353e9753d6SMatthew G. Knepley     ierr = MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
57363e9753d6SMatthew G. Knepley     ierr = MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
57373e9753d6SMatthew G. Knepley   }
57383e9753d6SMatthew G. Knepley   ierr = MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
57393e9753d6SMatthew G. Knepley   ierr = MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
57403e9753d6SMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
57413e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
57423e9753d6SMatthew G. Knepley }
57438e3a2eefSMatthew G. Knepley 
57448e3a2eefSMatthew G. Knepley /*
57458e3a2eefSMatthew G. Knepley   DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user.
57468e3a2eefSMatthew G. Knepley 
57478e3a2eefSMatthew G. Knepley   Input Parameters:
57488e3a2eefSMatthew G. Knepley + dm     - The mesh
57498e3a2eefSMatthew G. Knepley . key    - The PetscWeakFormKey indcating where integration should happen
575006ad1575SMatthew G. Knepley . cellIS - The cells to integrate over
57518e3a2eefSMatthew G. Knepley . t      - The time
57528e3a2eefSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with repsect to X_t
57538e3a2eefSMatthew G. Knepley . X      - Local solution vector
57548e3a2eefSMatthew G. Knepley . X_t    - Time-derivative of the local solution vector
57558e3a2eefSMatthew G. Knepley . Y      - Local input vector
575606ad1575SMatthew G. Knepley - user   - the user context
57578e3a2eefSMatthew G. Knepley 
57588e3a2eefSMatthew G. Knepley   Output Parameter:
57598e3a2eefSMatthew G. Knepley . Z - Local output vector
57608e3a2eefSMatthew G. Knepley 
57618e3a2eefSMatthew G. Knepley   Note:
57628e3a2eefSMatthew G. Knepley   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
57638e3a2eefSMatthew G. Knepley   like a GPU, or vectorize on a multicore machine.
57648e3a2eefSMatthew G. Knepley */
576506ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user)
57668e3a2eefSMatthew G. Knepley {
57678e3a2eefSMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *) dm->data;
57688e3a2eefSMatthew G. Knepley   const char     *name  = "Jacobian";
57698e3a2eefSMatthew G. Knepley   DM              dmAux = NULL, plex, plexAux = NULL;
57708e3a2eefSMatthew G. Knepley   DMEnclosureType encAux;
57718e3a2eefSMatthew G. Knepley   Vec             A;
57728e3a2eefSMatthew G. Knepley   DMField         coordField;
57738e3a2eefSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
57748e3a2eefSMatthew G. Knepley   PetscQuadrature quad;
57758e3a2eefSMatthew G. Knepley   PetscSection    section, globalSection, sectionAux;
57768e3a2eefSMatthew G. Knepley   PetscScalar    *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
57778e3a2eefSMatthew G. Knepley   const PetscInt *cells;
57788e3a2eefSMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
57798e3a2eefSMatthew G. Knepley   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
57808e3a2eefSMatthew G. Knepley   PetscBool       hasDyn;
57818e3a2eefSMatthew G. Knepley   PetscErrorCode  ierr;
57828e3a2eefSMatthew G. Knepley 
57838e3a2eefSMatthew G. Knepley   PetscFunctionBegin;
57848e3a2eefSMatthew G. Knepley   ierr = PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
57858e3a2eefSMatthew G. Knepley   ierr = DMConvert(dm, DMPLEX, &plex);CHKERRQ(ierr);
57868e3a2eefSMatthew G. Knepley   if (!cellIS) {
57878e3a2eefSMatthew G. Knepley     PetscInt depth;
57888e3a2eefSMatthew G. Knepley 
57898e3a2eefSMatthew G. Knepley     ierr = DMPlexGetDepth(plex, &depth);CHKERRQ(ierr);
57908e3a2eefSMatthew G. Knepley     ierr = DMGetStratumIS(plex, "dim", depth, &cellIS);CHKERRQ(ierr);
57918e3a2eefSMatthew G. Knepley     if (!cellIS) {ierr = DMGetStratumIS(plex, "depth", depth, &cellIS);CHKERRQ(ierr);}
57928e3a2eefSMatthew G. Knepley   } else {
57938e3a2eefSMatthew G. Knepley     ierr = PetscObjectReference((PetscObject) cellIS);CHKERRQ(ierr);
57948e3a2eefSMatthew G. Knepley   }
57958e3a2eefSMatthew G. Knepley   ierr = ISGetLocalSize(cellIS, &numCells);CHKERRQ(ierr);
57968e3a2eefSMatthew G. Knepley   ierr = ISGetPointRange(cellIS, &cStart, &cEnd, &cells);CHKERRQ(ierr);
57978e3a2eefSMatthew G. Knepley   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
57988e3a2eefSMatthew G. Knepley   ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);
57998e3a2eefSMatthew G. Knepley   ierr = DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob);CHKERRQ(ierr);
58008e3a2eefSMatthew G. Knepley   ierr = PetscDSGetNumFields(prob, &Nf);CHKERRQ(ierr);
58018e3a2eefSMatthew G. Knepley   ierr = PetscDSGetTotalDimension(prob, &totDim);CHKERRQ(ierr);
58028e3a2eefSMatthew G. Knepley   ierr = PetscDSHasDynamicJacobian(prob, &hasDyn);CHKERRQ(ierr);
58038e3a2eefSMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
5804e7225ec2SMatthew G. Knepley   ierr = DMGetAuxiliaryVec(dm, key.label, key.value, &A);CHKERRQ(ierr);
58058e3a2eefSMatthew G. Knepley   if (A) {
58068e3a2eefSMatthew G. Knepley     ierr = VecGetDM(A, &dmAux);CHKERRQ(ierr);
58078e3a2eefSMatthew G. Knepley     ierr = DMGetEnclosureRelation(dmAux, dm, &encAux);CHKERRQ(ierr);
58088e3a2eefSMatthew G. Knepley     ierr = DMConvert(dmAux, DMPLEX, &plexAux);CHKERRQ(ierr);
58098e3a2eefSMatthew G. Knepley     ierr = DMGetLocalSection(plexAux, &sectionAux);CHKERRQ(ierr);
58108e3a2eefSMatthew G. Knepley     ierr = DMGetDS(dmAux, &probAux);CHKERRQ(ierr);
58118e3a2eefSMatthew G. Knepley     ierr = PetscDSGetTotalDimension(probAux, &totDimAux);CHKERRQ(ierr);
58128e3a2eefSMatthew G. Knepley   }
58138e3a2eefSMatthew G. Knepley   ierr = VecSet(Z, 0.0);CHKERRQ(ierr);
58148e3a2eefSMatthew G. Knepley   ierr = PetscMalloc6(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,numCells*totDim*totDim,&elemMat,hasDyn ? numCells*totDim*totDim : 0, &elemMatD,numCells*totDim,&y,totDim,&z);CHKERRQ(ierr);
58158e3a2eefSMatthew G. Knepley   if (dmAux) {ierr = PetscMalloc1(numCells*totDimAux, &a);CHKERRQ(ierr);}
58168e3a2eefSMatthew G. Knepley   ierr = DMGetCoordinateField(dm, &coordField);CHKERRQ(ierr);
58178e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
58188e3a2eefSMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
58198e3a2eefSMatthew G. Knepley     const PetscInt cind = c - cStart;
58208e3a2eefSMatthew G. Knepley     PetscScalar   *x = NULL,  *x_t = NULL;
58218e3a2eefSMatthew G. Knepley     PetscInt       i;
58228e3a2eefSMatthew G. Knepley 
58238e3a2eefSMatthew G. Knepley     ierr = DMPlexVecGetClosure(plex, section, X, cell, NULL, &x);CHKERRQ(ierr);
58248e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
58258e3a2eefSMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x);CHKERRQ(ierr);
58268e3a2eefSMatthew G. Knepley     if (X_t) {
58278e3a2eefSMatthew G. Knepley       ierr = DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t);CHKERRQ(ierr);
58288e3a2eefSMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
58298e3a2eefSMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t);CHKERRQ(ierr);
58308e3a2eefSMatthew G. Knepley     }
58318e3a2eefSMatthew G. Knepley     if (dmAux) {
58328e3a2eefSMatthew G. Knepley       PetscInt subcell;
58338e3a2eefSMatthew G. Knepley       ierr = DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell);CHKERRQ(ierr);
58348e3a2eefSMatthew G. Knepley       ierr = DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x);CHKERRQ(ierr);
58358e3a2eefSMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
58368e3a2eefSMatthew G. Knepley       ierr = DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x);CHKERRQ(ierr);
58378e3a2eefSMatthew G. Knepley     }
58388e3a2eefSMatthew G. Knepley     ierr = DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x);CHKERRQ(ierr);
58398e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) y[cind*totDim+i] = x[i];
58408e3a2eefSMatthew G. Knepley     ierr = DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x);CHKERRQ(ierr);
58418e3a2eefSMatthew G. Knepley   }
58428e3a2eefSMatthew G. Knepley   ierr = PetscArrayzero(elemMat, numCells*totDim*totDim);CHKERRQ(ierr);
58438e3a2eefSMatthew G. Knepley   if (hasDyn)  {ierr = PetscArrayzero(elemMatD, numCells*totDim*totDim);CHKERRQ(ierr);}
58448e3a2eefSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
58458e3a2eefSMatthew G. Knepley     PetscFE  fe;
58468e3a2eefSMatthew G. Knepley     PetscInt Nb;
58478e3a2eefSMatthew G. Knepley     /* Conforming batches */
58488e3a2eefSMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
58498e3a2eefSMatthew G. Knepley     /* Remainder */
58508e3a2eefSMatthew G. Knepley     PetscInt Nr, offset, Nq;
58518e3a2eefSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
58528e3a2eefSMatthew G. Knepley     PetscInt    maxDegree;
58538e3a2eefSMatthew G. Knepley     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
58548e3a2eefSMatthew G. Knepley 
58558e3a2eefSMatthew G. Knepley     ierr = PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);CHKERRQ(ierr);
58568e3a2eefSMatthew G. Knepley     ierr = PetscFEGetQuadrature(fe, &quad);CHKERRQ(ierr);
58578e3a2eefSMatthew G. Knepley     ierr = PetscFEGetDimension(fe, &Nb);CHKERRQ(ierr);
58588e3a2eefSMatthew G. Knepley     ierr = PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);CHKERRQ(ierr);
58598e3a2eefSMatthew G. Knepley     ierr = DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);CHKERRQ(ierr);
58608e3a2eefSMatthew G. Knepley     if (maxDegree <= 1) {ierr = DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom);CHKERRQ(ierr);}
58618e3a2eefSMatthew G. Knepley     if (!qGeom) {
58628e3a2eefSMatthew G. Knepley       ierr = PetscFEGetQuadrature(fe,&qGeom);CHKERRQ(ierr);
58638e3a2eefSMatthew G. Knepley       ierr = PetscObjectReference((PetscObject)qGeom);CHKERRQ(ierr);
58648e3a2eefSMatthew G. Knepley     }
58658e3a2eefSMatthew G. Knepley     ierr = PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);CHKERRQ(ierr);
58668e3a2eefSMatthew G. Knepley     ierr = DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);CHKERRQ(ierr);
58678e3a2eefSMatthew G. Knepley     blockSize = Nb;
58688e3a2eefSMatthew G. Knepley     batchSize = numBlocks * blockSize;
58698e3a2eefSMatthew G. Knepley     ierr = PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);CHKERRQ(ierr);
58708e3a2eefSMatthew G. Knepley     numChunks = numCells / (numBatches*batchSize);
58718e3a2eefSMatthew G. Knepley     Ne        = numChunks*numBatches*batchSize;
58728e3a2eefSMatthew G. Knepley     Nr        = numCells % (numBatches*batchSize);
58738e3a2eefSMatthew G. Knepley     offset    = numCells - Nr;
58748e3a2eefSMatthew G. Knepley     ierr = PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);CHKERRQ(ierr);
58758e3a2eefSMatthew G. Knepley     ierr = PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom);CHKERRQ(ierr);
58768e3a2eefSMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
58778e3a2eefSMatthew G. Knepley       key.field = fieldI*Nf + fieldJ;
58788e3a2eefSMatthew G. Knepley       ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat);CHKERRQ(ierr);
58798e3a2eefSMatthew G. Knepley       ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]);CHKERRQ(ierr);
58808e3a2eefSMatthew G. Knepley       if (hasDyn) {
58818e3a2eefSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD);CHKERRQ(ierr);
58828e3a2eefSMatthew G. Knepley         ierr = PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]);CHKERRQ(ierr);
58838e3a2eefSMatthew G. Knepley       }
58848e3a2eefSMatthew G. Knepley     }
58858e3a2eefSMatthew G. Knepley     ierr = PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom);CHKERRQ(ierr);
58868e3a2eefSMatthew G. Knepley     ierr = PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom);CHKERRQ(ierr);
58878e3a2eefSMatthew G. Knepley     ierr = DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);CHKERRQ(ierr);
58888e3a2eefSMatthew G. Knepley     ierr = PetscQuadratureDestroy(&qGeom);CHKERRQ(ierr);
58898e3a2eefSMatthew G. Knepley   }
58908e3a2eefSMatthew G. Knepley   if (hasDyn) {
58918e3a2eefSMatthew G. Knepley     for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];
58928e3a2eefSMatthew G. Knepley   }
58938e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
58948e3a2eefSMatthew G. Knepley     const PetscInt     cell = cells ? cells[c] : c;
58958e3a2eefSMatthew G. Knepley     const PetscInt     cind = c - cStart;
58968e3a2eefSMatthew G. Knepley     const PetscBLASInt M = totDim, one = 1;
58978e3a2eefSMatthew G. Knepley     const PetscScalar  a = 1.0, b = 0.0;
58988e3a2eefSMatthew G. Knepley 
58998e3a2eefSMatthew G. Knepley     PetscStackCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind*totDim*totDim], &M, &y[cind*totDim], &one, &b, z, &one));
59008e3a2eefSMatthew G. Knepley     if (mesh->printFEM > 1) {
59018e3a2eefSMatthew G. Knepley       ierr = DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind*totDim*totDim]);CHKERRQ(ierr);
59028e3a2eefSMatthew G. Knepley       ierr = DMPrintCellVector(c, "Y",  totDim, &y[cind*totDim]);CHKERRQ(ierr);
59038e3a2eefSMatthew G. Knepley       ierr = DMPrintCellVector(c, "Z",  totDim, z);CHKERRQ(ierr);
59048e3a2eefSMatthew G. Knepley     }
59058e3a2eefSMatthew G. Knepley     ierr = DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES);CHKERRQ(ierr);
59068e3a2eefSMatthew G. Knepley   }
59078e3a2eefSMatthew G. Knepley   ierr = PetscFree6(u,u_t,elemMat,elemMatD,y,z);CHKERRQ(ierr);
59088e3a2eefSMatthew G. Knepley   if (mesh->printFEM) {
59098e3a2eefSMatthew G. Knepley     ierr = PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n");CHKERRQ(ierr);
59108e3a2eefSMatthew G. Knepley     ierr = VecView(Z, NULL);CHKERRQ(ierr);
59118e3a2eefSMatthew G. Knepley   }
59128e3a2eefSMatthew G. Knepley   ierr = PetscFree(a);CHKERRQ(ierr);
59138e3a2eefSMatthew G. Knepley   ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
59148e3a2eefSMatthew G. Knepley   ierr = DMDestroy(&plexAux);CHKERRQ(ierr);
59158e3a2eefSMatthew G. Knepley   ierr = DMDestroy(&plex);CHKERRQ(ierr);
59168e3a2eefSMatthew G. Knepley   ierr = PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);CHKERRQ(ierr);
59178e3a2eefSMatthew G. Knepley   PetscFunctionReturn(0);
59188e3a2eefSMatthew G. Knepley }
5919