xref: /petsc/src/dm/impls/plex/plex.c (revision 4211eb483909a9a6403702befc20456a00a91081)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     PetscReal    scale         = 2.0;
675     PetscReal    tikzscale     = 1.0;
676     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
677     double       tcoords[3];
678     PetscScalar *coords;
679     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
680     PetscMPIInt  rank, size;
681     char         **names, **colors, **lcolors;
682     PetscBool    plotEdges, flg;
683 
684     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
685     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
686     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
687     numLabels  = PetscMax(numLabels, 10);
688     numColors  = 10;
689     numLColors = 10;
690     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
691     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
692     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
693     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
694     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
695     if (!useLabels) numLabels = 0;
696     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
697     if (!useColors) {
698       numColors = 3;
699       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
700     }
701     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
702     if (!useColors) {
703       numLColors = 4;
704       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
705     }
706     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
707     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
708     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
709     if (depth < dim) plotEdges = PETSC_FALSE;
710     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
711     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
712     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
713     ierr = PetscViewerASCIIPrintf(viewer, "\
714 \\documentclass[tikz]{standalone}\n\n\
715 \\usepackage{pgflibraryshapes}\n\
716 \\usetikzlibrary{backgrounds}\n\
717 \\usetikzlibrary{arrows}\n\
718 \\begin{document}\n");CHKERRQ(ierr);
719     if (size > 1) {
720       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
721       for (p = 0; p < size; ++p) {
722         if (p > 0 && p == size-1) {
723           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
724         } else if (p > 0) {
725           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
726         }
727         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
728       }
729       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
730     }
731     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", tikzscale);CHKERRQ(ierr);
732     /* Plot vertices */
733     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
734     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
735     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
736     for (v = vStart; v < vEnd; ++v) {
737       PetscInt  off, dof, d;
738       PetscBool isLabeled = PETSC_FALSE;
739 
740       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
741       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
742       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
743       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
744       for (d = 0; d < dof; ++d) {
745         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
746         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
747       }
748       /* Rotate coordinates since PGF makes z point out of the page instead of up */
749       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
750       for (d = 0; d < dof; ++d) {
751         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
752         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", tcoords[d]);CHKERRQ(ierr);
753       }
754       color = colors[rank%numColors];
755       for (l = 0; l < numLabels; ++l) {
756         PetscInt val;
757         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
758         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
759       }
760       if (useNumbers) {
761         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
762       } else {
763         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
764       }
765     }
766     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
767     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
768     /* Plot cells */
769     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
770     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
771     if (dim == 3 || !useNumbers) {
772       for (e = eStart; e < eEnd; ++e) {
773         const PetscInt *cone;
774 
775         color = colors[rank%numColors];
776         for (l = 0; l < numLabels; ++l) {
777           PetscInt val;
778           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
779           if (val >= 0) {color = lcolors[l%numLColors]; break;}
780         }
781         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
782         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
783       }
784     } else {
785       for (c = cStart; c < cEnd; ++c) {
786         PetscInt *closure = NULL;
787         PetscInt  closureSize, firstPoint = -1;
788 
789         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
790         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
791         for (p = 0; p < closureSize*2; p += 2) {
792           const PetscInt point = closure[p];
793 
794           if ((point < vStart) || (point >= vEnd)) continue;
795           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
796           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
797           if (firstPoint < 0) firstPoint = point;
798         }
799         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
800         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
801         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
802       }
803     }
804     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
805     for (c = cStart; c < cEnd; ++c) {
806       double    ccoords[3] = {0.0, 0.0, 0.0};
807       PetscBool isLabeled  = PETSC_FALSE;
808       PetscInt *closure    = NULL;
809       PetscInt  closureSize, dof, d, n = 0;
810 
811       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
812       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
813       for (p = 0; p < closureSize*2; p += 2) {
814         const PetscInt point = closure[p];
815         PetscInt       off;
816 
817         if ((point < vStart) || (point >= vEnd)) continue;
818         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
819         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
820         for (d = 0; d < dof; ++d) {
821           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
822           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
823         }
824         /* Rotate coordinates since PGF makes z point out of the page instead of up */
825         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
826         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
827         ++n;
828       }
829       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
830       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
831       for (d = 0; d < dof; ++d) {
832         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
833         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", ccoords[d]);CHKERRQ(ierr);
834       }
835       color = colors[rank%numColors];
836       for (l = 0; l < numLabels; ++l) {
837         PetscInt val;
838         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
839         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
840       }
841       if (useNumbers) {
842         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
843       } else {
844         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
845       }
846     }
847     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
848     /* Plot edges */
849     if (plotEdges) {
850       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
851       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
852       for (e = eStart; e < eEnd; ++e) {
853         const PetscInt *cone;
854         PetscInt        coneSize, offA, offB, dof, d;
855 
856         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
857         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
858         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
859         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
860         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
861         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
862         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
863         for (d = 0; d < dof; ++d) {
864           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
865           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
866         }
867         /* Rotate coordinates since PGF makes z point out of the page instead of up */
868         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
869         for (d = 0; d < dof; ++d) {
870           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
871           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
872         }
873         color = colors[rank%numColors];
874         for (l = 0; l < numLabels; ++l) {
875           PetscInt val;
876           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
877           if (val >= 0) {color = lcolors[l%numLColors]; break;}
878         }
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
880       }
881       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
882       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
883       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
884     }
885     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
886     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
887     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
888     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
889     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
890     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
891     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
892     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
893   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
894     Vec                    cown,acown;
895     VecScatter             sct;
896     ISLocalToGlobalMapping g2l;
897     IS                     gid,acis;
898     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
899     MPI_Group              ggroup,ngroup;
900     PetscScalar            *array,nid;
901     const PetscInt         *idxs;
902     PetscInt               *idxs2,*start,*adjacency,*work;
903     PetscInt64             lm[3],gm[3];
904     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
905     PetscMPIInt            d1,d2,rank;
906 
907     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
908     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
909 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
910     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
911 #endif
912     if (ncomm != MPI_COMM_NULL) {
913       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
914       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
915       d1   = 0;
916       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
917       nid  = d2;
918       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
919       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
920       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
921     } else nid = 0.0;
922 
923     /* Get connectivity */
924     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
925     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
926 
927     /* filter overlapped local cells */
928     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
929     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
930     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
931     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
932     for (c = cStart, cum = 0; c < cEnd; c++) {
933       if (idxs[c-cStart] < 0) continue;
934       idxs2[cum++] = idxs[c-cStart];
935     }
936     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
937     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
938     ierr = ISDestroy(&gid);CHKERRQ(ierr);
939     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
940 
941     /* support for node-aware cell locality */
942     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
943     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
944     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
945     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
946     for (c = 0; c < numVertices; c++) array[c] = nid;
947     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
948     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
949     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
950     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
951     ierr = ISDestroy(&acis);CHKERRQ(ierr);
952     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
953     ierr = VecDestroy(&cown);CHKERRQ(ierr);
954 
955     /* compute edgeCut */
956     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
957     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
958     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
959     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
960     ierr = ISDestroy(&gid);CHKERRQ(ierr);
961     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
962     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
963       PetscInt totl;
964 
965       totl = start[c+1]-start[c];
966       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
967       for (i = 0; i < totl; i++) {
968         if (work[i] < 0) {
969           ect  += 1;
970           ectn += (array[i + start[c]] != nid) ? 0 : 1;
971         }
972       }
973     }
974     ierr  = PetscFree(work);CHKERRQ(ierr);
975     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
976     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
977     lm[1] = -numVertices;
978     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
979     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
980     lm[0] = ect; /* edgeCut */
981     lm[1] = ectn; /* node-aware edgeCut */
982     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
983     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
984     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
985 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
986     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
987 #else
988     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
989 #endif
990     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
991     ierr  = PetscFree(start);CHKERRQ(ierr);
992     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
993     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
994   } else {
995     MPI_Comm    comm;
996     PetscInt   *sizes, *hybsizes;
997     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
998     PetscInt    pStart, pEnd, p;
999     PetscInt    numLabels, l;
1000     const char *name;
1001     PetscMPIInt size;
1002 
1003     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1004     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1005     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1006     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1007     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1008     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1009     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1010     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1011     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1012     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1013     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1014     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1015     if (depth == 1) {
1016       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1017       pEnd = pEnd - pStart;
1018       pMax[0] -= pStart;
1019       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1020       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1021       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1022       for (p = 0; p < size; ++p) {
1023         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1024         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1025       }
1026       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1027       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1028       pEnd = pEnd - pStart;
1029       pMax[depth] -= pStart;
1030       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1031       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1032       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1033       for (p = 0; p < size; ++p) {
1034         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1035         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1036       }
1037       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1038     } else {
1039       PetscMPIInt rank;
1040       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1041       for (d = 0; d <= dim; d++) {
1042         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1043         pEnd    -= pStart;
1044         pMax[d] -= pStart;
1045         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1046         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1047         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1048         for (p = 0; p < size; ++p) {
1049           if (!rank) {
1050             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1051             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1052           }
1053         }
1054         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1055       }
1056     }
1057     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1058     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1059     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1060     for (l = 0; l < numLabels; ++l) {
1061       DMLabel         label;
1062       const char     *name;
1063       IS              valueIS;
1064       const PetscInt *values;
1065       PetscInt        numValues, v;
1066 
1067       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1068       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1069       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1070       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1071       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1072       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1073       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1074       for (v = 0; v < numValues; ++v) {
1075         PetscInt size;
1076 
1077         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1078         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1079         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1080       }
1081       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1082       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1083       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1084       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1085     }
1086     /* If no fields are specified, people do not want to see adjacency */
1087     if (dm->Nf) {
1088       PetscInt f;
1089 
1090       for (f = 0; f < dm->Nf; ++f) {
1091         const char *name;
1092 
1093         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1094         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1095         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1096         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1097         if (dm->fields[f].adjacency[0]) {
1098           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1099           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1100         } else {
1101           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1102           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1103         }
1104         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1105       }
1106     }
1107     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1108     if (cdm) {
1109       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1110       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1111       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1112     }
1113   }
1114   PetscFunctionReturn(0);
1115 }
1116 
1117 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1118 {
1119   PetscDraw          draw;
1120   DM                 cdm;
1121   PetscSection       coordSection;
1122   Vec                coordinates;
1123   const PetscScalar *coords;
1124   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1125   PetscBool          isnull;
1126   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1127   PetscMPIInt        rank;
1128   PetscErrorCode     ierr;
1129 
1130   PetscFunctionBegin;
1131   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1132   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1133   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1134   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
1135   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1136   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1137   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1138 
1139   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1140   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1141   if (isnull) PetscFunctionReturn(0);
1142   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1143 
1144   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1145   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1146   for (c = 0; c < N; c += dim) {
1147     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1148     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1149   }
1150   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1151   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1152   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1153   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1154   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1155 
1156   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1157   for (c = cStart; c < cEnd; ++c) {
1158     PetscScalar *coords = NULL;
1159     PetscInt     numCoords,coneSize;
1160 
1161     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1162     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1163     switch (coneSize) {
1164     case 3:
1165       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1166                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1167                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1168                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1169       break;
1170     case 4:
1171       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1172                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1173                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1174                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1175                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1176       break;
1177     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1178     }
1179     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1180   }
1181   for (c = cStart; c < cEnd; ++c) {
1182     PetscScalar *coords = NULL;
1183     PetscInt     numCoords,coneSize;
1184 
1185     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1186     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1187     switch (coneSize) {
1188     case 3:
1189       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1190       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1191       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1192       break;
1193     case 4:
1194       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1195       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1196       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1197       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1198       break;
1199     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1200     }
1201     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202   }
1203   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1204   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1205   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1206   PetscFunctionReturn(0);
1207 }
1208 
1209 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1210 {
1211   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1212   char           name[PETSC_MAX_PATH_LEN];
1213   PetscErrorCode ierr;
1214 
1215   PetscFunctionBegin;
1216   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1217   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1219   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1220   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1221   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1222   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1223   if (iascii) {
1224     PetscViewerFormat format;
1225     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1226     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1227       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1228     } else {
1229       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1230     }
1231   } else if (ishdf5) {
1232 #if defined(PETSC_HAVE_HDF5)
1233     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1234 #else
1235     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1236 #endif
1237   } else if (isvtk) {
1238     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1239   } else if (isdraw) {
1240     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1241   } else if (isglvis) {
1242     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1243   } else {
1244     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1245   }
1246   /* Optionally view the partition */
1247   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1248   if (flg) {
1249     Vec ranks;
1250     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1251     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1252     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1253   }
1254   /* Optionally view a label */
1255   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1256   if (flg) {
1257     DMLabel label;
1258     Vec     val;
1259 
1260     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1261     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1262     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1263     ierr = VecView(val, viewer);CHKERRQ(ierr);
1264     ierr = VecDestroy(&val);CHKERRQ(ierr);
1265   }
1266   PetscFunctionReturn(0);
1267 }
1268 
1269 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1270 {
1271   PetscBool      ishdf5;
1272   PetscErrorCode ierr;
1273 
1274   PetscFunctionBegin;
1275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1276   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1277   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1278   if (ishdf5) {
1279 #if defined(PETSC_HAVE_HDF5)
1280     PetscViewerFormat format;
1281     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1282     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1283       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1284     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1285       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1286     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1287 #else
1288     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1289 #endif
1290   } else {
1291     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1292   }
1293   PetscFunctionReturn(0);
1294 }
1295 
1296 PetscErrorCode DMDestroy_Plex(DM dm)
1297 {
1298   DM_Plex       *mesh = (DM_Plex*) dm->data;
1299   PetscErrorCode ierr;
1300 
1301   PetscFunctionBegin;
1302   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1303   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1304   if (--mesh->refct > 0) PetscFunctionReturn(0);
1305   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1306   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1307   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1308   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1309   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1310   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1311   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1312   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1313   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1314   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1315   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1316   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1317   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1318   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1319   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1320   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1321   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1322   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1323   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1324   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1325   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1326   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1327   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1328   ierr = PetscFree(mesh);CHKERRQ(ierr);
1329   PetscFunctionReturn(0);
1330 }
1331 
1332 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1333 {
1334   PetscSection           sectionGlobal;
1335   PetscInt               bs = -1, mbs;
1336   PetscInt               localSize;
1337   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1338   PetscErrorCode         ierr;
1339   MatType                mtype;
1340   ISLocalToGlobalMapping ltog;
1341 
1342   PetscFunctionBegin;
1343   ierr = MatInitializePackage();CHKERRQ(ierr);
1344   mtype = dm->mattype;
1345   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1346   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1347   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1348   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1349   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1350   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1351   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1352   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1353   if (mbs > 1) bs = mbs;
1354   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1355   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1356   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1357   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1358   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1359   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1360   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1361   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1362   if (!isShell) {
1363     PetscSection subSection;
1364     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1365     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1366     PetscInt     pStart, pEnd, p, dof, cdof;
1367 
1368     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1369     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1370       PetscSection section;
1371       PetscInt     size;
1372 
1373       ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
1374       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1375       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1376       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1377     } else {
1378       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1379     }
1380     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1381     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1382       PetscInt bdof;
1383 
1384       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1385       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1386       dof  = dof < 0 ? -(dof+1) : dof;
1387       bdof = cdof && (dof-cdof) ? 1 : dof;
1388       if (dof) {
1389         if (bs < 0)          {bs = bdof;}
1390         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1391       }
1392       if (isMatIS) {
1393         PetscInt loff,c,off;
1394         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1395         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1396         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1397       }
1398     }
1399     /* Must have same blocksize on all procs (some might have no points) */
1400     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1401     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1402     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1403     else                            {bs = bsMinMax[0];}
1404     bs = bs < 0 ? 1 : bs;
1405     if (isMatIS) {
1406       PetscInt l;
1407       /* Must reduce indices by blocksize */
1408       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] /= bs;
1409       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1410     }
1411     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1412     if (isMatIS) {
1413       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1414     }
1415     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1416     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1417     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1418   }
1419   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1420   PetscFunctionReturn(0);
1421 }
1422 
1423 /*@
1424   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1425 
1426   Not collective
1427 
1428   Input Parameter:
1429 . mesh - The DMPlex
1430 
1431   Output Parameters:
1432 . subsection - The subdomain section
1433 
1434   Level: developer
1435 
1436 .seealso:
1437 @*/
1438 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1439 {
1440   DM_Plex       *mesh = (DM_Plex*) dm->data;
1441   PetscErrorCode ierr;
1442 
1443   PetscFunctionBegin;
1444   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1445   if (!mesh->subdomainSection) {
1446     PetscSection section;
1447     PetscSF      sf;
1448 
1449     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1450     ierr = DMGetSection(dm,&section);CHKERRQ(ierr);
1451     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1452     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1453   }
1454   *subsection = mesh->subdomainSection;
1455   PetscFunctionReturn(0);
1456 }
1457 
1458 /*@
1459   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1460 
1461   Not collective
1462 
1463   Input Parameter:
1464 . mesh - The DMPlex
1465 
1466   Output Parameters:
1467 + pStart - The first mesh point
1468 - pEnd   - The upper bound for mesh points
1469 
1470   Level: beginner
1471 
1472 .seealso: DMPlexCreate(), DMPlexSetChart()
1473 @*/
1474 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1475 {
1476   DM_Plex       *mesh = (DM_Plex*) dm->data;
1477   PetscErrorCode ierr;
1478 
1479   PetscFunctionBegin;
1480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1481   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1482   PetscFunctionReturn(0);
1483 }
1484 
1485 /*@
1486   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1487 
1488   Not collective
1489 
1490   Input Parameters:
1491 + mesh - The DMPlex
1492 . pStart - The first mesh point
1493 - pEnd   - The upper bound for mesh points
1494 
1495   Output Parameters:
1496 
1497   Level: beginner
1498 
1499 .seealso: DMPlexCreate(), DMPlexGetChart()
1500 @*/
1501 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1502 {
1503   DM_Plex       *mesh = (DM_Plex*) dm->data;
1504   PetscErrorCode ierr;
1505 
1506   PetscFunctionBegin;
1507   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1508   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1509   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1510   PetscFunctionReturn(0);
1511 }
1512 
1513 /*@
1514   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1515 
1516   Not collective
1517 
1518   Input Parameters:
1519 + mesh - The DMPlex
1520 - p - The point, which must lie in the chart set with DMPlexSetChart()
1521 
1522   Output Parameter:
1523 . size - The cone size for point p
1524 
1525   Level: beginner
1526 
1527 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1528 @*/
1529 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1530 {
1531   DM_Plex       *mesh = (DM_Plex*) dm->data;
1532   PetscErrorCode ierr;
1533 
1534   PetscFunctionBegin;
1535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1536   PetscValidPointer(size, 3);
1537   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1538   PetscFunctionReturn(0);
1539 }
1540 
1541 /*@
1542   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1543 
1544   Not collective
1545 
1546   Input Parameters:
1547 + mesh - The DMPlex
1548 . p - The point, which must lie in the chart set with DMPlexSetChart()
1549 - size - The cone size for point p
1550 
1551   Output Parameter:
1552 
1553   Note:
1554   This should be called after DMPlexSetChart().
1555 
1556   Level: beginner
1557 
1558 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1559 @*/
1560 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1561 {
1562   DM_Plex       *mesh = (DM_Plex*) dm->data;
1563   PetscErrorCode ierr;
1564 
1565   PetscFunctionBegin;
1566   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1567   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1568 
1569   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1570   PetscFunctionReturn(0);
1571 }
1572 
1573 /*@
1574   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1575 
1576   Not collective
1577 
1578   Input Parameters:
1579 + mesh - The DMPlex
1580 . p - The point, which must lie in the chart set with DMPlexSetChart()
1581 - size - The additional cone size for point p
1582 
1583   Output Parameter:
1584 
1585   Note:
1586   This should be called after DMPlexSetChart().
1587 
1588   Level: beginner
1589 
1590 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1591 @*/
1592 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1593 {
1594   DM_Plex       *mesh = (DM_Plex*) dm->data;
1595   PetscInt       csize;
1596   PetscErrorCode ierr;
1597 
1598   PetscFunctionBegin;
1599   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1600   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1601   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1602 
1603   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1604   PetscFunctionReturn(0);
1605 }
1606 
1607 /*@C
1608   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1609 
1610   Not collective
1611 
1612   Input Parameters:
1613 + dm - The DMPlex
1614 - p - The point, which must lie in the chart set with DMPlexSetChart()
1615 
1616   Output Parameter:
1617 . cone - An array of points which are on the in-edges for point p
1618 
1619   Level: beginner
1620 
1621   Fortran Notes:
1622   Since it returns an array, this routine is only available in Fortran 90, and you must
1623   include petsc.h90 in your code.
1624   You must also call DMPlexRestoreCone() after you finish using the returned array.
1625   DMPlexRestoreCone() is not needed/available in C.
1626 
1627 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1628 @*/
1629 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1630 {
1631   DM_Plex       *mesh = (DM_Plex*) dm->data;
1632   PetscInt       off;
1633   PetscErrorCode ierr;
1634 
1635   PetscFunctionBegin;
1636   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1637   PetscValidPointer(cone, 3);
1638   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1639   *cone = &mesh->cones[off];
1640   PetscFunctionReturn(0);
1641 }
1642 
1643 /*@C
1644   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1645 
1646   Not collective
1647 
1648   Input Parameters:
1649 + dm - The DMPlex
1650 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1651 
1652   Output Parameter:
1653 + pConesSection - PetscSection describing the layout of pCones
1654 - pCones - An array of points which are on the in-edges for the point set p
1655 
1656   Level: intermediate
1657 
1658 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1659 @*/
1660 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1661 {
1662   PetscSection        cs, newcs;
1663   PetscInt            *cones;
1664   PetscInt            *newarr=NULL;
1665   PetscInt            n;
1666   PetscErrorCode      ierr;
1667 
1668   PetscFunctionBegin;
1669   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1670   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1671   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1672   if (pConesSection) *pConesSection = newcs;
1673   if (pCones) {
1674     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1675     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1676   }
1677   PetscFunctionReturn(0);
1678 }
1679 
1680 static PetscErrorCode DMPlexGetConeRecursive_Private(DM dm, PetscInt *n_inout, const PetscInt points[], PetscInt *offset_inout, PetscInt buf[])
1681 {
1682   PetscInt p, n, cn, i;
1683   const PetscInt *cone;
1684   PetscErrorCode ierr;
1685 
1686   PetscFunctionBegin;
1687   n = *n_inout;
1688   *n_inout = 0;
1689   for (i=0; i<n; i++) {
1690     p = points[i];
1691     ierr = DMPlexGetConeSize(dm, p, &cn);CHKERRQ(ierr);
1692     if (!cn) {
1693       cn = 1;
1694       if (buf) {
1695         buf[*offset_inout] = p;
1696         ++(*offset_inout);
1697       }
1698     } else {
1699       ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1700       ierr = DMPlexGetConeRecursive_Private(dm, &cn, cone, offset_inout, buf);CHKERRQ(ierr);
1701     }
1702     *n_inout += cn;
1703   }
1704   PetscFunctionReturn(0);
1705 }
1706 
1707 /*@C
1708   DMPlexGetConeRecursive - Like DMPlexGetConeTuple() but recursive, i.e. each cone point is expanded into a set of its own cone points until a vertex (DAG point with no cone) is reached.
1709 
1710   Not collective
1711 
1712   Input Parameters:
1713 + dm - The DMPlex
1714 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1715 
1716   Output Parameter:
1717 . pCones - An array of recursively expanded cones, i.e. containing only vertices, and each of them can be present multiple times
1718 
1719   Level: advanced
1720 
1721 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple()
1722 @*/
1723 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS p, IS *pCones)
1724 {
1725   const PetscInt      *arr=NULL;
1726   PetscInt            *cpoints=NULL;
1727   PetscInt            n, cn;
1728   PetscInt            zero;
1729   PetscErrorCode      ierr;
1730 
1731   PetscFunctionBegin;
1732   ierr = ISGetLocalSize(p, &n);CHKERRQ(ierr);
1733   ierr = ISGetIndices(p, &arr);CHKERRQ(ierr);
1734   zero = 0;
1735   /* first figure out the total number of returned points */
1736   cn = n;
1737   ierr = DMPlexGetConeRecursive_Private(dm, &cn, arr, &zero, NULL);CHKERRQ(ierr);
1738   ierr = PetscMalloc1(cn, &cpoints);CHKERRQ(ierr);
1739   /* now get recursive cones themselves */
1740   ierr = DMPlexGetConeRecursive_Private(dm, &n, arr, &zero, cpoints);CHKERRQ(ierr);
1741   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, cpoints, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1742   ierr = ISRestoreIndices(p, &arr);CHKERRQ(ierr);
1743   PetscFunctionReturn(0);
1744 }
1745 
1746 /*@
1747   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1748 
1749   Not collective
1750 
1751   Input Parameters:
1752 + mesh - The DMPlex
1753 . p - The point, which must lie in the chart set with DMPlexSetChart()
1754 - cone - An array of points which are on the in-edges for point p
1755 
1756   Output Parameter:
1757 
1758   Note:
1759   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1760 
1761   Developer Note: Why not call this DMPlexSetCover()
1762 
1763   Level: beginner
1764 
1765 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1766 @*/
1767 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1768 {
1769   DM_Plex       *mesh = (DM_Plex*) dm->data;
1770   PetscInt       pStart, pEnd;
1771   PetscInt       dof, off, c;
1772   PetscErrorCode ierr;
1773 
1774   PetscFunctionBegin;
1775   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1776   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1777   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1778   if (dof) PetscValidPointer(cone, 3);
1779   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1780   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1781   for (c = 0; c < dof; ++c) {
1782     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1783     mesh->cones[off+c] = cone[c];
1784   }
1785   PetscFunctionReturn(0);
1786 }
1787 
1788 /*@C
1789   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1790 
1791   Not collective
1792 
1793   Input Parameters:
1794 + mesh - The DMPlex
1795 - p - The point, which must lie in the chart set with DMPlexSetChart()
1796 
1797   Output Parameter:
1798 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1799                     integer giving the prescription for cone traversal. If it is negative, the cone is
1800                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1801                     the index of the cone point on which to start.
1802 
1803   Level: beginner
1804 
1805   Fortran Notes:
1806   Since it returns an array, this routine is only available in Fortran 90, and you must
1807   include petsc.h90 in your code.
1808   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1809   DMPlexRestoreConeOrientation() is not needed/available in C.
1810 
1811 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1812 @*/
1813 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1814 {
1815   DM_Plex       *mesh = (DM_Plex*) dm->data;
1816   PetscInt       off;
1817   PetscErrorCode ierr;
1818 
1819   PetscFunctionBegin;
1820   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1821 #if defined(PETSC_USE_DEBUG)
1822   {
1823     PetscInt dof;
1824     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1825     if (dof) PetscValidPointer(coneOrientation, 3);
1826   }
1827 #endif
1828   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1829 
1830   *coneOrientation = &mesh->coneOrientations[off];
1831   PetscFunctionReturn(0);
1832 }
1833 
1834 /*@
1835   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1836 
1837   Not collective
1838 
1839   Input Parameters:
1840 + mesh - The DMPlex
1841 . p - The point, which must lie in the chart set with DMPlexSetChart()
1842 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1843                     integer giving the prescription for cone traversal. If it is negative, the cone is
1844                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1845                     the index of the cone point on which to start.
1846 
1847   Output Parameter:
1848 
1849   Note:
1850   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1851 
1852   Level: beginner
1853 
1854 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1855 @*/
1856 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1857 {
1858   DM_Plex       *mesh = (DM_Plex*) dm->data;
1859   PetscInt       pStart, pEnd;
1860   PetscInt       dof, off, c;
1861   PetscErrorCode ierr;
1862 
1863   PetscFunctionBegin;
1864   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1865   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1866   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1867   if (dof) PetscValidPointer(coneOrientation, 3);
1868   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1869   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1870   for (c = 0; c < dof; ++c) {
1871     PetscInt cdof, o = coneOrientation[c];
1872 
1873     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1874     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1875     mesh->coneOrientations[off+c] = o;
1876   }
1877   PetscFunctionReturn(0);
1878 }
1879 
1880 /*@
1881   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
1882 
1883   Not collective
1884 
1885   Input Parameters:
1886 + mesh - The DMPlex
1887 . p - The point, which must lie in the chart set with DMPlexSetChart()
1888 . conePos - The local index in the cone where the point should be put
1889 - conePoint - The mesh point to insert
1890 
1891   Level: beginner
1892 
1893 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1894 @*/
1895 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1896 {
1897   DM_Plex       *mesh = (DM_Plex*) dm->data;
1898   PetscInt       pStart, pEnd;
1899   PetscInt       dof, off;
1900   PetscErrorCode ierr;
1901 
1902   PetscFunctionBegin;
1903   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1904   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1905   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1906   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1907   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1908   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1909   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1910   mesh->cones[off+conePos] = conePoint;
1911   PetscFunctionReturn(0);
1912 }
1913 
1914 /*@
1915   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
1916 
1917   Not collective
1918 
1919   Input Parameters:
1920 + mesh - The DMPlex
1921 . p - The point, which must lie in the chart set with DMPlexSetChart()
1922 . conePos - The local index in the cone where the point should be put
1923 - coneOrientation - The point orientation to insert
1924 
1925   Level: beginner
1926 
1927 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1928 @*/
1929 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1930 {
1931   DM_Plex       *mesh = (DM_Plex*) dm->data;
1932   PetscInt       pStart, pEnd;
1933   PetscInt       dof, off;
1934   PetscErrorCode ierr;
1935 
1936   PetscFunctionBegin;
1937   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1938   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1939   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1940   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1941   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1942   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1943   mesh->coneOrientations[off+conePos] = coneOrientation;
1944   PetscFunctionReturn(0);
1945 }
1946 
1947 /*@
1948   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
1949 
1950   Not collective
1951 
1952   Input Parameters:
1953 + mesh - The DMPlex
1954 - p - The point, which must lie in the chart set with DMPlexSetChart()
1955 
1956   Output Parameter:
1957 . size - The support size for point p
1958 
1959   Level: beginner
1960 
1961 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1962 @*/
1963 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1964 {
1965   DM_Plex       *mesh = (DM_Plex*) dm->data;
1966   PetscErrorCode ierr;
1967 
1968   PetscFunctionBegin;
1969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1970   PetscValidPointer(size, 3);
1971   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1972   PetscFunctionReturn(0);
1973 }
1974 
1975 /*@
1976   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
1977 
1978   Not collective
1979 
1980   Input Parameters:
1981 + mesh - The DMPlex
1982 . p - The point, which must lie in the chart set with DMPlexSetChart()
1983 - size - The support size for point p
1984 
1985   Output Parameter:
1986 
1987   Note:
1988   This should be called after DMPlexSetChart().
1989 
1990   Level: beginner
1991 
1992 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1993 @*/
1994 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1995 {
1996   DM_Plex       *mesh = (DM_Plex*) dm->data;
1997   PetscErrorCode ierr;
1998 
1999   PetscFunctionBegin;
2000   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2001   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2002 
2003   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2004   PetscFunctionReturn(0);
2005 }
2006 
2007 /*@C
2008   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2009 
2010   Not collective
2011 
2012   Input Parameters:
2013 + mesh - The DMPlex
2014 - p - The point, which must lie in the chart set with DMPlexSetChart()
2015 
2016   Output Parameter:
2017 . support - An array of points which are on the out-edges for point p
2018 
2019   Level: beginner
2020 
2021   Fortran Notes:
2022   Since it returns an array, this routine is only available in Fortran 90, and you must
2023   include petsc.h90 in your code.
2024   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2025   DMPlexRestoreSupport() is not needed/available in C.
2026 
2027 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2028 @*/
2029 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2030 {
2031   DM_Plex       *mesh = (DM_Plex*) dm->data;
2032   PetscInt       off;
2033   PetscErrorCode ierr;
2034 
2035   PetscFunctionBegin;
2036   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2037   PetscValidPointer(support, 3);
2038   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2039   *support = &mesh->supports[off];
2040   PetscFunctionReturn(0);
2041 }
2042 
2043 /*@
2044   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2045 
2046   Not collective
2047 
2048   Input Parameters:
2049 + mesh - The DMPlex
2050 . p - The point, which must lie in the chart set with DMPlexSetChart()
2051 - support - An array of points which are on the out-edges for point p
2052 
2053   Output Parameter:
2054 
2055   Note:
2056   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2057 
2058   Level: beginner
2059 
2060 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2061 @*/
2062 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2063 {
2064   DM_Plex       *mesh = (DM_Plex*) dm->data;
2065   PetscInt       pStart, pEnd;
2066   PetscInt       dof, off, c;
2067   PetscErrorCode ierr;
2068 
2069   PetscFunctionBegin;
2070   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2071   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2072   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2073   if (dof) PetscValidPointer(support, 3);
2074   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2075   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2076   for (c = 0; c < dof; ++c) {
2077     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2078     mesh->supports[off+c] = support[c];
2079   }
2080   PetscFunctionReturn(0);
2081 }
2082 
2083 /*@
2084   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2085 
2086   Not collective
2087 
2088   Input Parameters:
2089 + mesh - The DMPlex
2090 . p - The point, which must lie in the chart set with DMPlexSetChart()
2091 . supportPos - The local index in the cone where the point should be put
2092 - supportPoint - The mesh point to insert
2093 
2094   Level: beginner
2095 
2096 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2097 @*/
2098 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2099 {
2100   DM_Plex       *mesh = (DM_Plex*) dm->data;
2101   PetscInt       pStart, pEnd;
2102   PetscInt       dof, off;
2103   PetscErrorCode ierr;
2104 
2105   PetscFunctionBegin;
2106   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2107   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2108   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2109   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2110   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2111   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2112   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2113   mesh->supports[off+supportPos] = supportPoint;
2114   PetscFunctionReturn(0);
2115 }
2116 
2117 /*@C
2118   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2119 
2120   Not collective
2121 
2122   Input Parameters:
2123 + mesh - The DMPlex
2124 . p - The point, which must lie in the chart set with DMPlexSetChart()
2125 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2126 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2127 
2128   Output Parameters:
2129 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2130 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2131 
2132   Note:
2133   If using internal storage (points is NULL on input), each call overwrites the last output.
2134 
2135   Fortran Notes:
2136   Since it returns an array, this routine is only available in Fortran 90, and you must
2137   include petsc.h90 in your code.
2138 
2139   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2140 
2141   Level: beginner
2142 
2143 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2144 @*/
2145 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2146 {
2147   DM_Plex        *mesh = (DM_Plex*) dm->data;
2148   PetscInt       *closure, *fifo;
2149   const PetscInt *tmp = NULL, *tmpO = NULL;
2150   PetscInt        tmpSize, t;
2151   PetscInt        depth       = 0, maxSize;
2152   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2153   PetscErrorCode  ierr;
2154 
2155   PetscFunctionBegin;
2156   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2157   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2158   /* This is only 1-level */
2159   if (useCone) {
2160     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2161     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2162     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2163   } else {
2164     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2165     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2166   }
2167   if (depth == 1) {
2168     if (*points) {
2169       closure = *points;
2170     } else {
2171       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2172       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2173     }
2174     closure[0] = p; closure[1] = 0;
2175     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2176       closure[closureSize]   = tmp[t];
2177       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2178     }
2179     if (numPoints) *numPoints = closureSize/2;
2180     if (points)    *points    = closure;
2181     PetscFunctionReturn(0);
2182   }
2183   {
2184     PetscInt c, coneSeries, s,supportSeries;
2185 
2186     c = mesh->maxConeSize;
2187     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2188     s = mesh->maxSupportSize;
2189     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2190     maxSize = 2*PetscMax(coneSeries,supportSeries);
2191   }
2192   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2193   if (*points) {
2194     closure = *points;
2195   } else {
2196     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2197   }
2198   closure[0] = p; closure[1] = 0;
2199   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2200     const PetscInt cp = tmp[t];
2201     const PetscInt co = tmpO ? tmpO[t] : 0;
2202 
2203     closure[closureSize]   = cp;
2204     closure[closureSize+1] = co;
2205     fifo[fifoSize]         = cp;
2206     fifo[fifoSize+1]       = co;
2207   }
2208   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2209   while (fifoSize - fifoStart) {
2210     const PetscInt q   = fifo[fifoStart];
2211     const PetscInt o   = fifo[fifoStart+1];
2212     const PetscInt rev = o >= 0 ? 0 : 1;
2213     const PetscInt off = rev ? -(o+1) : o;
2214 
2215     if (useCone) {
2216       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2217       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2218       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2219     } else {
2220       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2221       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2222       tmpO = NULL;
2223     }
2224     for (t = 0; t < tmpSize; ++t) {
2225       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2226       const PetscInt cp = tmp[i];
2227       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2228       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2229        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2230       PetscInt       co = tmpO ? tmpO[i] : 0;
2231       PetscInt       c;
2232 
2233       if (rev) {
2234         PetscInt childSize, coff;
2235         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2236         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2237         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2238       }
2239       /* Check for duplicate */
2240       for (c = 0; c < closureSize; c += 2) {
2241         if (closure[c] == cp) break;
2242       }
2243       if (c == closureSize) {
2244         closure[closureSize]   = cp;
2245         closure[closureSize+1] = co;
2246         fifo[fifoSize]         = cp;
2247         fifo[fifoSize+1]       = co;
2248         closureSize           += 2;
2249         fifoSize              += 2;
2250       }
2251     }
2252     fifoStart += 2;
2253   }
2254   if (numPoints) *numPoints = closureSize/2;
2255   if (points)    *points    = closure;
2256   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2257   PetscFunctionReturn(0);
2258 }
2259 
2260 /*@C
2261   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2262 
2263   Not collective
2264 
2265   Input Parameters:
2266 + mesh - The DMPlex
2267 . p - The point, which must lie in the chart set with DMPlexSetChart()
2268 . orientation - The orientation of the point
2269 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2270 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2271 
2272   Output Parameters:
2273 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2274 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2275 
2276   Note:
2277   If using internal storage (points is NULL on input), each call overwrites the last output.
2278 
2279   Fortran Notes:
2280   Since it returns an array, this routine is only available in Fortran 90, and you must
2281   include petsc.h90 in your code.
2282 
2283   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2284 
2285   Level: beginner
2286 
2287 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2288 @*/
2289 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2290 {
2291   DM_Plex        *mesh = (DM_Plex*) dm->data;
2292   PetscInt       *closure, *fifo;
2293   const PetscInt *tmp = NULL, *tmpO = NULL;
2294   PetscInt        tmpSize, t;
2295   PetscInt        depth       = 0, maxSize;
2296   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2297   PetscErrorCode  ierr;
2298 
2299   PetscFunctionBegin;
2300   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2301   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2302   /* This is only 1-level */
2303   if (useCone) {
2304     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2305     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2306     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2307   } else {
2308     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2309     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2310   }
2311   if (depth == 1) {
2312     if (*points) {
2313       closure = *points;
2314     } else {
2315       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2316       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2317     }
2318     closure[0] = p; closure[1] = ornt;
2319     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2320       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2321       closure[closureSize]   = tmp[i];
2322       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2323     }
2324     if (numPoints) *numPoints = closureSize/2;
2325     if (points)    *points    = closure;
2326     PetscFunctionReturn(0);
2327   }
2328   {
2329     PetscInt c, coneSeries, s,supportSeries;
2330 
2331     c = mesh->maxConeSize;
2332     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2333     s = mesh->maxSupportSize;
2334     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2335     maxSize = 2*PetscMax(coneSeries,supportSeries);
2336   }
2337   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2338   if (*points) {
2339     closure = *points;
2340   } else {
2341     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2342   }
2343   closure[0] = p; closure[1] = ornt;
2344   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2345     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2346     const PetscInt cp = tmp[i];
2347     PetscInt       co = tmpO ? tmpO[i] : 0;
2348 
2349     if (ornt < 0) {
2350       PetscInt childSize, coff;
2351       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2352       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2353       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2354     }
2355     closure[closureSize]   = cp;
2356     closure[closureSize+1] = co;
2357     fifo[fifoSize]         = cp;
2358     fifo[fifoSize+1]       = co;
2359   }
2360   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2361   while (fifoSize - fifoStart) {
2362     const PetscInt q   = fifo[fifoStart];
2363     const PetscInt o   = fifo[fifoStart+1];
2364     const PetscInt rev = o >= 0 ? 0 : 1;
2365     const PetscInt off = rev ? -(o+1) : o;
2366 
2367     if (useCone) {
2368       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2369       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2370       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2371     } else {
2372       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2373       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2374       tmpO = NULL;
2375     }
2376     for (t = 0; t < tmpSize; ++t) {
2377       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2378       const PetscInt cp = tmp[i];
2379       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2380       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2381        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2382       PetscInt       co = tmpO ? tmpO[i] : 0;
2383       PetscInt       c;
2384 
2385       if (rev) {
2386         PetscInt childSize, coff;
2387         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2388         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2389         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2390       }
2391       /* Check for duplicate */
2392       for (c = 0; c < closureSize; c += 2) {
2393         if (closure[c] == cp) break;
2394       }
2395       if (c == closureSize) {
2396         closure[closureSize]   = cp;
2397         closure[closureSize+1] = co;
2398         fifo[fifoSize]         = cp;
2399         fifo[fifoSize+1]       = co;
2400         closureSize           += 2;
2401         fifoSize              += 2;
2402       }
2403     }
2404     fifoStart += 2;
2405   }
2406   if (numPoints) *numPoints = closureSize/2;
2407   if (points)    *points    = closure;
2408   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2409   PetscFunctionReturn(0);
2410 }
2411 
2412 /*@C
2413   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2414 
2415   Not collective
2416 
2417   Input Parameters:
2418 + mesh - The DMPlex
2419 . p - The point, which must lie in the chart set with DMPlexSetChart()
2420 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2421 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2422 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2423 
2424   Note:
2425   If not using internal storage (points is not NULL on input), this call is unnecessary
2426 
2427   Fortran Notes:
2428   Since it returns an array, this routine is only available in Fortran 90, and you must
2429   include petsc.h90 in your code.
2430 
2431   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2432 
2433   Level: beginner
2434 
2435 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2436 @*/
2437 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2438 {
2439   PetscErrorCode ierr;
2440 
2441   PetscFunctionBegin;
2442   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2443   if (numPoints) PetscValidIntPointer(numPoints,4);
2444   if (points) PetscValidPointer(points,5);
2445   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2446   if (numPoints) *numPoints = 0;
2447   PetscFunctionReturn(0);
2448 }
2449 
2450 /*@
2451   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2452 
2453   Not collective
2454 
2455   Input Parameter:
2456 . mesh - The DMPlex
2457 
2458   Output Parameters:
2459 + maxConeSize - The maximum number of in-edges
2460 - maxSupportSize - The maximum number of out-edges
2461 
2462   Level: beginner
2463 
2464 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2465 @*/
2466 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2467 {
2468   DM_Plex *mesh = (DM_Plex*) dm->data;
2469 
2470   PetscFunctionBegin;
2471   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2472   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2473   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2474   PetscFunctionReturn(0);
2475 }
2476 
2477 PetscErrorCode DMSetUp_Plex(DM dm)
2478 {
2479   DM_Plex       *mesh = (DM_Plex*) dm->data;
2480   PetscInt       size;
2481   PetscErrorCode ierr;
2482 
2483   PetscFunctionBegin;
2484   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2485   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2486   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2487   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2488   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2489   if (mesh->maxSupportSize) {
2490     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2491     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2492     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2493   }
2494   PetscFunctionReturn(0);
2495 }
2496 
2497 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2498 {
2499   PetscErrorCode ierr;
2500 
2501   PetscFunctionBegin;
2502   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2503   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2504   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2505   if (dm->useNatural && dm->sfMigration) {
2506     PetscSF        sfMigrationInv,sfNatural;
2507     PetscSection   section, sectionSeq;
2508 
2509     (*subdm)->sfMigration = dm->sfMigration;
2510     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2511     ierr = DMGetSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2512     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2513     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2514     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2515 
2516     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2517     (*subdm)->sfNatural = sfNatural;
2518     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2519     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2520   }
2521   PetscFunctionReturn(0);
2522 }
2523 
2524 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2525 {
2526   PetscErrorCode ierr;
2527   PetscInt       i = 0;
2528 
2529   PetscFunctionBegin;
2530   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2531   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2532   (*superdm)->useNatural = PETSC_FALSE;
2533   for (i = 0; i < len; i++){
2534     if (dms[i]->useNatural && dms[i]->sfMigration) {
2535       PetscSF        sfMigrationInv,sfNatural;
2536       PetscSection   section, sectionSeq;
2537 
2538       (*superdm)->sfMigration = dms[i]->sfMigration;
2539       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2540       (*superdm)->useNatural = PETSC_TRUE;
2541       ierr = DMGetSection((*superdm), &section);CHKERRQ(ierr);
2542       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2543       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2544       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2545 
2546       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2547       (*superdm)->sfNatural = sfNatural;
2548       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2549       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2550       break;
2551     }
2552   }
2553   PetscFunctionReturn(0);
2554 }
2555 
2556 /*@
2557   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2558 
2559   Not collective
2560 
2561   Input Parameter:
2562 . mesh - The DMPlex
2563 
2564   Output Parameter:
2565 
2566   Note:
2567   This should be called after all calls to DMPlexSetCone()
2568 
2569   Level: beginner
2570 
2571 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2572 @*/
2573 PetscErrorCode DMPlexSymmetrize(DM dm)
2574 {
2575   DM_Plex       *mesh = (DM_Plex*) dm->data;
2576   PetscInt      *offsets;
2577   PetscInt       supportSize;
2578   PetscInt       pStart, pEnd, p;
2579   PetscErrorCode ierr;
2580 
2581   PetscFunctionBegin;
2582   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2583   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2584   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2585   /* Calculate support sizes */
2586   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2587   for (p = pStart; p < pEnd; ++p) {
2588     PetscInt dof, off, c;
2589 
2590     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2591     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2592     for (c = off; c < off+dof; ++c) {
2593       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2594     }
2595   }
2596   for (p = pStart; p < pEnd; ++p) {
2597     PetscInt dof;
2598 
2599     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2600 
2601     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2602   }
2603   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2604   /* Calculate supports */
2605   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2606   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2607   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2608   for (p = pStart; p < pEnd; ++p) {
2609     PetscInt dof, off, c;
2610 
2611     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2612     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2613     for (c = off; c < off+dof; ++c) {
2614       const PetscInt q = mesh->cones[c];
2615       PetscInt       offS;
2616 
2617       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2618 
2619       mesh->supports[offS+offsets[q]] = p;
2620       ++offsets[q];
2621     }
2622   }
2623   ierr = PetscFree(offsets);CHKERRQ(ierr);
2624   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2625   PetscFunctionReturn(0);
2626 }
2627 
2628 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2629 {
2630   IS             stratumIS;
2631   PetscErrorCode ierr;
2632 
2633   PetscFunctionBegin;
2634   if (pStart >= pEnd) PetscFunctionReturn(0);
2635 #if defined(PETSC_USE_DEBUG)
2636   {
2637     PetscInt  qStart, qEnd, numLevels, level;
2638     PetscBool overlap = PETSC_FALSE;
2639     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2640     for (level = 0; level < numLevels; level++) {
2641       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2642       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2643     }
2644     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2645   }
2646 #endif
2647   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2648   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2649   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2650   PetscFunctionReturn(0);
2651 }
2652 
2653 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2654 
2655 /*@
2656   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2657   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2658   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2659   the DAG.
2660 
2661   Collective on dm
2662 
2663   Input Parameter:
2664 . mesh - The DMPlex
2665 
2666   Output Parameter:
2667 
2668   Notes:
2669   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2670   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2671   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2672   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2673 
2674   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2675 
2676   Level: beginner
2677 
2678 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2679 @*/
2680 PetscErrorCode DMPlexStratify(DM dm)
2681 {
2682   DM_Plex       *mesh = (DM_Plex*) dm->data;
2683   DMLabel        label;
2684   PetscInt       pStart, pEnd, p;
2685   PetscInt       numRoots = 0, numLeaves = 0;
2686   PetscInt       cMax, fMax, eMax, vMax;
2687   PetscErrorCode ierr;
2688 
2689   PetscFunctionBegin;
2690   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2691   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2692 
2693   /* Create depth label */
2694   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2695   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2696   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2697 
2698   {
2699     /* Initialize roots and count leaves */
2700     PetscInt sMin = PETSC_MAX_INT;
2701     PetscInt sMax = PETSC_MIN_INT;
2702     PetscInt coneSize, supportSize;
2703 
2704     for (p = pStart; p < pEnd; ++p) {
2705       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2706       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2707       if (!coneSize && supportSize) {
2708         sMin = PetscMin(p, sMin);
2709         sMax = PetscMax(p, sMax);
2710         ++numRoots;
2711       } else if (!supportSize && coneSize) {
2712         ++numLeaves;
2713       } else if (!supportSize && !coneSize) {
2714         /* Isolated points */
2715         sMin = PetscMin(p, sMin);
2716         sMax = PetscMax(p, sMax);
2717       }
2718     }
2719     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2720   }
2721 
2722   if (numRoots + numLeaves == (pEnd - pStart)) {
2723     PetscInt sMin = PETSC_MAX_INT;
2724     PetscInt sMax = PETSC_MIN_INT;
2725     PetscInt coneSize, supportSize;
2726 
2727     for (p = pStart; p < pEnd; ++p) {
2728       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2729       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2730       if (!supportSize && coneSize) {
2731         sMin = PetscMin(p, sMin);
2732         sMax = PetscMax(p, sMax);
2733       }
2734     }
2735     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2736   } else {
2737     PetscInt level = 0;
2738     PetscInt qStart, qEnd, q;
2739 
2740     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2741     while (qEnd > qStart) {
2742       PetscInt sMin = PETSC_MAX_INT;
2743       PetscInt sMax = PETSC_MIN_INT;
2744 
2745       for (q = qStart; q < qEnd; ++q) {
2746         const PetscInt *support;
2747         PetscInt        supportSize, s;
2748 
2749         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2750         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2751         for (s = 0; s < supportSize; ++s) {
2752           sMin = PetscMin(support[s], sMin);
2753           sMax = PetscMax(support[s], sMax);
2754         }
2755       }
2756       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2757       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2758       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2759     }
2760   }
2761   { /* just in case there is an empty process */
2762     PetscInt numValues, maxValues = 0, v;
2763 
2764     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2765     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2766     for (v = numValues; v < maxValues; v++) {
2767       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2768     }
2769   }
2770   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2771 
2772   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2773   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2774     PetscInt dim;
2775     DMLabel  dimLabel;
2776 
2777     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2778     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2779     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2780     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2781     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2782     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2783     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2784   }
2785   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2786   PetscFunctionReturn(0);
2787 }
2788 
2789 /*@C
2790   DMPlexGetJoin - Get an array for the join of the set of points
2791 
2792   Not Collective
2793 
2794   Input Parameters:
2795 + dm - The DMPlex object
2796 . numPoints - The number of input points for the join
2797 - points - The input points
2798 
2799   Output Parameters:
2800 + numCoveredPoints - The number of points in the join
2801 - coveredPoints - The points in the join
2802 
2803   Level: intermediate
2804 
2805   Note: Currently, this is restricted to a single level join
2806 
2807   Fortran Notes:
2808   Since it returns an array, this routine is only available in Fortran 90, and you must
2809   include petsc.h90 in your code.
2810 
2811   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2812 
2813 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2814 @*/
2815 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2816 {
2817   DM_Plex       *mesh = (DM_Plex*) dm->data;
2818   PetscInt      *join[2];
2819   PetscInt       joinSize, i = 0;
2820   PetscInt       dof, off, p, c, m;
2821   PetscErrorCode ierr;
2822 
2823   PetscFunctionBegin;
2824   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2825   PetscValidIntPointer(points, 3);
2826   PetscValidIntPointer(numCoveredPoints, 4);
2827   PetscValidPointer(coveredPoints, 5);
2828   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2829   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2830   /* Copy in support of first point */
2831   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2832   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2833   for (joinSize = 0; joinSize < dof; ++joinSize) {
2834     join[i][joinSize] = mesh->supports[off+joinSize];
2835   }
2836   /* Check each successive support */
2837   for (p = 1; p < numPoints; ++p) {
2838     PetscInt newJoinSize = 0;
2839 
2840     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2841     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2842     for (c = 0; c < dof; ++c) {
2843       const PetscInt point = mesh->supports[off+c];
2844 
2845       for (m = 0; m < joinSize; ++m) {
2846         if (point == join[i][m]) {
2847           join[1-i][newJoinSize++] = point;
2848           break;
2849         }
2850       }
2851     }
2852     joinSize = newJoinSize;
2853     i        = 1-i;
2854   }
2855   *numCoveredPoints = joinSize;
2856   *coveredPoints    = join[i];
2857   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
2858   PetscFunctionReturn(0);
2859 }
2860 
2861 /*@C
2862   DMPlexRestoreJoin - Restore an array for the join of the set of points
2863 
2864   Not Collective
2865 
2866   Input Parameters:
2867 + dm - The DMPlex object
2868 . numPoints - The number of input points for the join
2869 - points - The input points
2870 
2871   Output Parameters:
2872 + numCoveredPoints - The number of points in the join
2873 - coveredPoints - The points in the join
2874 
2875   Fortran Notes:
2876   Since it returns an array, this routine is only available in Fortran 90, and you must
2877   include petsc.h90 in your code.
2878 
2879   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2880 
2881   Level: intermediate
2882 
2883 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2884 @*/
2885 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2886 {
2887   PetscErrorCode ierr;
2888 
2889   PetscFunctionBegin;
2890   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2891   if (points) PetscValidIntPointer(points,3);
2892   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2893   PetscValidPointer(coveredPoints, 5);
2894   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
2895   if (numCoveredPoints) *numCoveredPoints = 0;
2896   PetscFunctionReturn(0);
2897 }
2898 
2899 /*@C
2900   DMPlexGetFullJoin - Get an array for the join of the set of points
2901 
2902   Not Collective
2903 
2904   Input Parameters:
2905 + dm - The DMPlex object
2906 . numPoints - The number of input points for the join
2907 - points - The input points
2908 
2909   Output Parameters:
2910 + numCoveredPoints - The number of points in the join
2911 - coveredPoints - The points in the join
2912 
2913   Fortran Notes:
2914   Since it returns an array, this routine is only available in Fortran 90, and you must
2915   include petsc.h90 in your code.
2916 
2917   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2918 
2919   Level: intermediate
2920 
2921 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2922 @*/
2923 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2924 {
2925   DM_Plex       *mesh = (DM_Plex*) dm->data;
2926   PetscInt      *offsets, **closures;
2927   PetscInt      *join[2];
2928   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2929   PetscInt       p, d, c, m, ms;
2930   PetscErrorCode ierr;
2931 
2932   PetscFunctionBegin;
2933   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2934   PetscValidIntPointer(points, 3);
2935   PetscValidIntPointer(numCoveredPoints, 4);
2936   PetscValidPointer(coveredPoints, 5);
2937 
2938   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2939   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2940   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2941   ms      = mesh->maxSupportSize;
2942   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
2943   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2944   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2945 
2946   for (p = 0; p < numPoints; ++p) {
2947     PetscInt closureSize;
2948 
2949     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2950 
2951     offsets[p*(depth+2)+0] = 0;
2952     for (d = 0; d < depth+1; ++d) {
2953       PetscInt pStart, pEnd, i;
2954 
2955       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2956       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2957         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2958           offsets[p*(depth+2)+d+1] = i;
2959           break;
2960         }
2961       }
2962       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2963     }
2964     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2965   }
2966   for (d = 0; d < depth+1; ++d) {
2967     PetscInt dof;
2968 
2969     /* Copy in support of first point */
2970     dof = offsets[d+1] - offsets[d];
2971     for (joinSize = 0; joinSize < dof; ++joinSize) {
2972       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2973     }
2974     /* Check each successive cone */
2975     for (p = 1; p < numPoints && joinSize; ++p) {
2976       PetscInt newJoinSize = 0;
2977 
2978       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2979       for (c = 0; c < dof; ++c) {
2980         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2981 
2982         for (m = 0; m < joinSize; ++m) {
2983           if (point == join[i][m]) {
2984             join[1-i][newJoinSize++] = point;
2985             break;
2986           }
2987         }
2988       }
2989       joinSize = newJoinSize;
2990       i        = 1-i;
2991     }
2992     if (joinSize) break;
2993   }
2994   *numCoveredPoints = joinSize;
2995   *coveredPoints    = join[i];
2996   for (p = 0; p < numPoints; ++p) {
2997     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2998   }
2999   ierr = PetscFree(closures);CHKERRQ(ierr);
3000   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3001   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3002   PetscFunctionReturn(0);
3003 }
3004 
3005 /*@C
3006   DMPlexGetMeet - Get an array for the meet of the set of points
3007 
3008   Not Collective
3009 
3010   Input Parameters:
3011 + dm - The DMPlex object
3012 . numPoints - The number of input points for the meet
3013 - points - The input points
3014 
3015   Output Parameters:
3016 + numCoveredPoints - The number of points in the meet
3017 - coveredPoints - The points in the meet
3018 
3019   Level: intermediate
3020 
3021   Note: Currently, this is restricted to a single level meet
3022 
3023   Fortran Notes:
3024   Since it returns an array, this routine is only available in Fortran 90, and you must
3025   include petsc.h90 in your code.
3026 
3027   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3028 
3029 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3030 @*/
3031 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3032 {
3033   DM_Plex       *mesh = (DM_Plex*) dm->data;
3034   PetscInt      *meet[2];
3035   PetscInt       meetSize, i = 0;
3036   PetscInt       dof, off, p, c, m;
3037   PetscErrorCode ierr;
3038 
3039   PetscFunctionBegin;
3040   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3041   PetscValidPointer(points, 2);
3042   PetscValidPointer(numCoveringPoints, 3);
3043   PetscValidPointer(coveringPoints, 4);
3044   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3045   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3046   /* Copy in cone of first point */
3047   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3048   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3049   for (meetSize = 0; meetSize < dof; ++meetSize) {
3050     meet[i][meetSize] = mesh->cones[off+meetSize];
3051   }
3052   /* Check each successive cone */
3053   for (p = 1; p < numPoints; ++p) {
3054     PetscInt newMeetSize = 0;
3055 
3056     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3057     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3058     for (c = 0; c < dof; ++c) {
3059       const PetscInt point = mesh->cones[off+c];
3060 
3061       for (m = 0; m < meetSize; ++m) {
3062         if (point == meet[i][m]) {
3063           meet[1-i][newMeetSize++] = point;
3064           break;
3065         }
3066       }
3067     }
3068     meetSize = newMeetSize;
3069     i        = 1-i;
3070   }
3071   *numCoveringPoints = meetSize;
3072   *coveringPoints    = meet[i];
3073   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3074   PetscFunctionReturn(0);
3075 }
3076 
3077 /*@C
3078   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3079 
3080   Not Collective
3081 
3082   Input Parameters:
3083 + dm - The DMPlex object
3084 . numPoints - The number of input points for the meet
3085 - points - The input points
3086 
3087   Output Parameters:
3088 + numCoveredPoints - The number of points in the meet
3089 - coveredPoints - The points in the meet
3090 
3091   Level: intermediate
3092 
3093   Fortran Notes:
3094   Since it returns an array, this routine is only available in Fortran 90, and you must
3095   include petsc.h90 in your code.
3096 
3097   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3098 
3099 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3100 @*/
3101 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3102 {
3103   PetscErrorCode ierr;
3104 
3105   PetscFunctionBegin;
3106   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3107   if (points) PetscValidIntPointer(points,3);
3108   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3109   PetscValidPointer(coveredPoints,5);
3110   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3111   if (numCoveredPoints) *numCoveredPoints = 0;
3112   PetscFunctionReturn(0);
3113 }
3114 
3115 /*@C
3116   DMPlexGetFullMeet - Get an array for the meet of the set of points
3117 
3118   Not Collective
3119 
3120   Input Parameters:
3121 + dm - The DMPlex object
3122 . numPoints - The number of input points for the meet
3123 - points - The input points
3124 
3125   Output Parameters:
3126 + numCoveredPoints - The number of points in the meet
3127 - coveredPoints - The points in the meet
3128 
3129   Level: intermediate
3130 
3131   Fortran Notes:
3132   Since it returns an array, this routine is only available in Fortran 90, and you must
3133   include petsc.h90 in your code.
3134 
3135   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3136 
3137 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3138 @*/
3139 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3140 {
3141   DM_Plex       *mesh = (DM_Plex*) dm->data;
3142   PetscInt      *offsets, **closures;
3143   PetscInt      *meet[2];
3144   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3145   PetscInt       p, h, c, m, mc;
3146   PetscErrorCode ierr;
3147 
3148   PetscFunctionBegin;
3149   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3150   PetscValidPointer(points, 2);
3151   PetscValidPointer(numCoveredPoints, 3);
3152   PetscValidPointer(coveredPoints, 4);
3153 
3154   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3155   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3156   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3157   mc      = mesh->maxConeSize;
3158   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3159   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3160   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3161 
3162   for (p = 0; p < numPoints; ++p) {
3163     PetscInt closureSize;
3164 
3165     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3166 
3167     offsets[p*(height+2)+0] = 0;
3168     for (h = 0; h < height+1; ++h) {
3169       PetscInt pStart, pEnd, i;
3170 
3171       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3172       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3173         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3174           offsets[p*(height+2)+h+1] = i;
3175           break;
3176         }
3177       }
3178       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3179     }
3180     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3181   }
3182   for (h = 0; h < height+1; ++h) {
3183     PetscInt dof;
3184 
3185     /* Copy in cone of first point */
3186     dof = offsets[h+1] - offsets[h];
3187     for (meetSize = 0; meetSize < dof; ++meetSize) {
3188       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3189     }
3190     /* Check each successive cone */
3191     for (p = 1; p < numPoints && meetSize; ++p) {
3192       PetscInt newMeetSize = 0;
3193 
3194       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3195       for (c = 0; c < dof; ++c) {
3196         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3197 
3198         for (m = 0; m < meetSize; ++m) {
3199           if (point == meet[i][m]) {
3200             meet[1-i][newMeetSize++] = point;
3201             break;
3202           }
3203         }
3204       }
3205       meetSize = newMeetSize;
3206       i        = 1-i;
3207     }
3208     if (meetSize) break;
3209   }
3210   *numCoveredPoints = meetSize;
3211   *coveredPoints    = meet[i];
3212   for (p = 0; p < numPoints; ++p) {
3213     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3214   }
3215   ierr = PetscFree(closures);CHKERRQ(ierr);
3216   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3217   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3218   PetscFunctionReturn(0);
3219 }
3220 
3221 /*@C
3222   DMPlexEqual - Determine if two DMs have the same topology
3223 
3224   Not Collective
3225 
3226   Input Parameters:
3227 + dmA - A DMPlex object
3228 - dmB - A DMPlex object
3229 
3230   Output Parameters:
3231 . equal - PETSC_TRUE if the topologies are identical
3232 
3233   Level: intermediate
3234 
3235   Notes:
3236   We are not solving graph isomorphism, so we do not permutation.
3237 
3238 .seealso: DMPlexGetCone()
3239 @*/
3240 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3241 {
3242   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3243   PetscErrorCode ierr;
3244 
3245   PetscFunctionBegin;
3246   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3247   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3248   PetscValidPointer(equal, 3);
3249 
3250   *equal = PETSC_FALSE;
3251   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3252   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3253   if (depth != depthB) PetscFunctionReturn(0);
3254   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3255   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3256   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3257   for (p = pStart; p < pEnd; ++p) {
3258     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3259     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3260 
3261     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3262     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3263     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3264     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3265     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3266     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3267     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3268     for (c = 0; c < coneSize; ++c) {
3269       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3270       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3271     }
3272     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3273     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3274     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3275     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3276     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3277     for (s = 0; s < supportSize; ++s) {
3278       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3279     }
3280   }
3281   *equal = PETSC_TRUE;
3282   PetscFunctionReturn(0);
3283 }
3284 
3285 /*@C
3286   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3287 
3288   Not Collective
3289 
3290   Input Parameters:
3291 + dm         - The DMPlex
3292 . cellDim    - The cell dimension
3293 - numCorners - The number of vertices on a cell
3294 
3295   Output Parameters:
3296 . numFaceVertices - The number of vertices on a face
3297 
3298   Level: developer
3299 
3300   Notes:
3301   Of course this can only work for a restricted set of symmetric shapes
3302 
3303 .seealso: DMPlexGetCone()
3304 @*/
3305 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3306 {
3307   MPI_Comm       comm;
3308   PetscErrorCode ierr;
3309 
3310   PetscFunctionBegin;
3311   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3312   PetscValidPointer(numFaceVertices,3);
3313   switch (cellDim) {
3314   case 0:
3315     *numFaceVertices = 0;
3316     break;
3317   case 1:
3318     *numFaceVertices = 1;
3319     break;
3320   case 2:
3321     switch (numCorners) {
3322     case 3: /* triangle */
3323       *numFaceVertices = 2; /* Edge has 2 vertices */
3324       break;
3325     case 4: /* quadrilateral */
3326       *numFaceVertices = 2; /* Edge has 2 vertices */
3327       break;
3328     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3329       *numFaceVertices = 3; /* Edge has 3 vertices */
3330       break;
3331     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3332       *numFaceVertices = 3; /* Edge has 3 vertices */
3333       break;
3334     default:
3335       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3336     }
3337     break;
3338   case 3:
3339     switch (numCorners) {
3340     case 4: /* tetradehdron */
3341       *numFaceVertices = 3; /* Face has 3 vertices */
3342       break;
3343     case 6: /* tet cohesive cells */
3344       *numFaceVertices = 4; /* Face has 4 vertices */
3345       break;
3346     case 8: /* hexahedron */
3347       *numFaceVertices = 4; /* Face has 4 vertices */
3348       break;
3349     case 9: /* tet cohesive Lagrange cells */
3350       *numFaceVertices = 6; /* Face has 6 vertices */
3351       break;
3352     case 10: /* quadratic tetrahedron */
3353       *numFaceVertices = 6; /* Face has 6 vertices */
3354       break;
3355     case 12: /* hex cohesive Lagrange cells */
3356       *numFaceVertices = 6; /* Face has 6 vertices */
3357       break;
3358     case 18: /* quadratic tet cohesive Lagrange cells */
3359       *numFaceVertices = 6; /* Face has 6 vertices */
3360       break;
3361     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3362       *numFaceVertices = 9; /* Face has 9 vertices */
3363       break;
3364     default:
3365       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3366     }
3367     break;
3368   default:
3369     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3370   }
3371   PetscFunctionReturn(0);
3372 }
3373 
3374 /*@
3375   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3376 
3377   Not Collective
3378 
3379   Input Parameter:
3380 . dm    - The DMPlex object
3381 
3382   Output Parameter:
3383 . depthLabel - The DMLabel recording point depth
3384 
3385   Level: developer
3386 
3387 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3388 @*/
3389 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3390 {
3391   PetscErrorCode ierr;
3392 
3393   PetscFunctionBegin;
3394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3395   PetscValidPointer(depthLabel, 2);
3396   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3397   *depthLabel = dm->depthLabel;
3398   PetscFunctionReturn(0);
3399 }
3400 
3401 /*@
3402   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3403 
3404   Not Collective
3405 
3406   Input Parameter:
3407 . dm    - The DMPlex object
3408 
3409   Output Parameter:
3410 . depth - The number of strata (breadth first levels) in the DAG
3411 
3412   Level: developer
3413 
3414 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3415 @*/
3416 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3417 {
3418   DMLabel        label;
3419   PetscInt       d = 0;
3420   PetscErrorCode ierr;
3421 
3422   PetscFunctionBegin;
3423   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3424   PetscValidPointer(depth, 2);
3425   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3426   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3427   *depth = d-1;
3428   PetscFunctionReturn(0);
3429 }
3430 
3431 /*@
3432   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3433 
3434   Not Collective
3435 
3436   Input Parameters:
3437 + dm           - The DMPlex object
3438 - stratumValue - The requested depth
3439 
3440   Output Parameters:
3441 + start - The first point at this depth
3442 - end   - One beyond the last point at this depth
3443 
3444   Notes:
3445   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3446   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3447   higher dimension, e.g., "edges".
3448 
3449   Level: developer
3450 
3451 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3452 @*/
3453 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3454 {
3455   DMLabel        label;
3456   PetscInt       pStart, pEnd;
3457   PetscErrorCode ierr;
3458 
3459   PetscFunctionBegin;
3460   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3461   if (start) {PetscValidPointer(start, 3); *start = 0;}
3462   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3463   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3464   if (pStart == pEnd) PetscFunctionReturn(0);
3465   if (stratumValue < 0) {
3466     if (start) *start = pStart;
3467     if (end)   *end   = pEnd;
3468     PetscFunctionReturn(0);
3469   }
3470   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3471   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3472   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3473   PetscFunctionReturn(0);
3474 }
3475 
3476 /*@
3477   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3478 
3479   Not Collective
3480 
3481   Input Parameters:
3482 + dm           - The DMPlex object
3483 - stratumValue - The requested height
3484 
3485   Output Parameters:
3486 + start - The first point at this height
3487 - end   - One beyond the last point at this height
3488 
3489   Notes:
3490   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3491   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3492   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3493 
3494   Level: developer
3495 
3496 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3497 @*/
3498 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3499 {
3500   DMLabel        label;
3501   PetscInt       depth, pStart, pEnd;
3502   PetscErrorCode ierr;
3503 
3504   PetscFunctionBegin;
3505   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3506   if (start) {PetscValidPointer(start, 3); *start = 0;}
3507   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3508   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3509   if (pStart == pEnd) PetscFunctionReturn(0);
3510   if (stratumValue < 0) {
3511     if (start) *start = pStart;
3512     if (end)   *end   = pEnd;
3513     PetscFunctionReturn(0);
3514   }
3515   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3516   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3517   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3518   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3519   PetscFunctionReturn(0);
3520 }
3521 
3522 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3523 {
3524   PetscSection   section, s;
3525   Mat            m;
3526   PetscInt       maxHeight;
3527   PetscErrorCode ierr;
3528 
3529   PetscFunctionBegin;
3530   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3531   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3532   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3533   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3534   ierr = DMSetSection(*cdm, section);CHKERRQ(ierr);
3535   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3536   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3537   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3538   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3539   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3540   ierr = MatDestroy(&m);CHKERRQ(ierr);
3541 
3542   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3543   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3544   PetscFunctionReturn(0);
3545 }
3546 
3547 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3548 {
3549   Vec            coordsLocal;
3550   DM             coordsDM;
3551   PetscErrorCode ierr;
3552 
3553   PetscFunctionBegin;
3554   *field = NULL;
3555   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3556   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3557   if (coordsLocal && coordsDM) {
3558     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3559   }
3560   PetscFunctionReturn(0);
3561 }
3562 
3563 /*@C
3564   DMPlexGetConeSection - Return a section which describes the layout of cone data
3565 
3566   Not Collective
3567 
3568   Input Parameters:
3569 . dm        - The DMPlex object
3570 
3571   Output Parameter:
3572 . section - The PetscSection object
3573 
3574   Level: developer
3575 
3576 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3577 @*/
3578 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3579 {
3580   DM_Plex *mesh = (DM_Plex*) dm->data;
3581 
3582   PetscFunctionBegin;
3583   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3584   if (section) *section = mesh->coneSection;
3585   PetscFunctionReturn(0);
3586 }
3587 
3588 /*@C
3589   DMPlexGetSupportSection - Return a section which describes the layout of support data
3590 
3591   Not Collective
3592 
3593   Input Parameters:
3594 . dm        - The DMPlex object
3595 
3596   Output Parameter:
3597 . section - The PetscSection object
3598 
3599   Level: developer
3600 
3601 .seealso: DMPlexGetConeSection()
3602 @*/
3603 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3604 {
3605   DM_Plex *mesh = (DM_Plex*) dm->data;
3606 
3607   PetscFunctionBegin;
3608   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3609   if (section) *section = mesh->supportSection;
3610   PetscFunctionReturn(0);
3611 }
3612 
3613 /*@C
3614   DMPlexGetCones - Return cone data
3615 
3616   Not Collective
3617 
3618   Input Parameters:
3619 . dm        - The DMPlex object
3620 
3621   Output Parameter:
3622 . cones - The cone for each point
3623 
3624   Level: developer
3625 
3626 .seealso: DMPlexGetConeSection()
3627 @*/
3628 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3629 {
3630   DM_Plex *mesh = (DM_Plex*) dm->data;
3631 
3632   PetscFunctionBegin;
3633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3634   if (cones) *cones = mesh->cones;
3635   PetscFunctionReturn(0);
3636 }
3637 
3638 /*@C
3639   DMPlexGetConeOrientations - Return cone orientation data
3640 
3641   Not Collective
3642 
3643   Input Parameters:
3644 . dm        - The DMPlex object
3645 
3646   Output Parameter:
3647 . coneOrientations - The cone orientation for each point
3648 
3649   Level: developer
3650 
3651 .seealso: DMPlexGetConeSection()
3652 @*/
3653 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3654 {
3655   DM_Plex *mesh = (DM_Plex*) dm->data;
3656 
3657   PetscFunctionBegin;
3658   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3659   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3660   PetscFunctionReturn(0);
3661 }
3662 
3663 /******************************** FEM Support **********************************/
3664 
3665 /*@
3666 
3667   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3668   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3669   section provided (or default section of the DM).
3670 
3671   Input Parameters:
3672 + dm      - The DM
3673 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3674 - section - The PetscSection to reorder, or NULL for the default section
3675 
3676   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3677   degree of the basis.
3678 
3679   Example:
3680   A typical interpolated single-quad mesh might order points as
3681 .vb
3682   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3683 
3684   v4 -- e6 -- v3
3685   |           |
3686   e7    c0    e8
3687   |           |
3688   v1 -- e5 -- v2
3689 .ve
3690 
3691   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3692   dofs in the order of points, e.g.,
3693 .vb
3694     c0 -> [0,1,2,3]
3695     v1 -> [4]
3696     ...
3697     e5 -> [8, 9]
3698 .ve
3699 
3700   which corresponds to the dofs
3701 .vb
3702     6   10  11  7
3703     13  2   3   15
3704     12  0   1   14
3705     4   8   9   5
3706 .ve
3707 
3708   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3709 .vb
3710   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3711 .ve
3712 
3713   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3714 .vb
3715    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3716 .ve
3717 
3718   Level: developer
3719 
3720 .seealso: DMGetSection(), PetscSectionSetClosurePermutation()
3721 @*/
3722 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3723 {
3724   DMLabel        label;
3725   PetscInt      *perm;
3726   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3727   PetscErrorCode ierr;
3728 
3729   PetscFunctionBegin;
3730   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3731   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3732   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3733   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3734   if (depth == 1) {eStart = point;}
3735   else if  (depth == dim) {
3736     const PetscInt *cone;
3737 
3738     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3739     if (dim == 2) eStart = cone[0];
3740     else if (dim == 3) {
3741       const PetscInt *cone2;
3742       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3743       eStart = cone2[0];
3744     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3745   } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3746   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
3747   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3748   if (dim < 1) PetscFunctionReturn(0);
3749   for (f = 0; f < Nf; ++f) {
3750     /* An order k SEM disc has k-1 dofs on an edge */
3751     ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3752     ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3753     k = k/Nc + 1;
3754     size += PetscPowInt(k+1, dim)*Nc;
3755   }
3756   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3757   for (f = 0; f < Nf; ++f) {
3758     switch (dim) {
3759     case 1:
3760       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3761       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3762       k = k/Nc + 1;
3763       /*
3764         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3765         We want              [ vtx0; edge of length k-1; vtx1 ]
3766       */
3767       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3768       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3769       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3770       foffset = offset;
3771       break;
3772     case 2:
3773       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3774       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3775       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3776       k = k/Nc + 1;
3777       /* The SEM order is
3778 
3779          v_lb, {e_b}, v_rb,
3780          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3781          v_lt, reverse {e_t}, v_rt
3782       */
3783       {
3784         const PetscInt of   = 0;
3785         const PetscInt oeb  = of   + PetscSqr(k-1);
3786         const PetscInt oer  = oeb  + (k-1);
3787         const PetscInt oet  = oer  + (k-1);
3788         const PetscInt oel  = oet  + (k-1);
3789         const PetscInt ovlb = oel  + (k-1);
3790         const PetscInt ovrb = ovlb + 1;
3791         const PetscInt ovrt = ovrb + 1;
3792         const PetscInt ovlt = ovrt + 1;
3793         PetscInt       o;
3794 
3795         /* bottom */
3796         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3797         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3798         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3799         /* middle */
3800         for (i = 0; i < k-1; ++i) {
3801           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3802           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3803           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3804         }
3805         /* top */
3806         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3807         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3808         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3809         foffset = offset;
3810       }
3811       break;
3812     case 3:
3813       /* The original hex closure is
3814 
3815          {c,
3816           f_b, f_t, f_f, f_b, f_r, f_l,
3817           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3818           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3819       */
3820       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3821       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3822       k = k/Nc + 1;
3823       /* The SEM order is
3824          Bottom Slice
3825          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3826          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3827          v_blb, {e_bb}, v_brb,
3828 
3829          Middle Slice (j)
3830          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3831          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3832          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3833 
3834          Top Slice
3835          v_tlf, {e_tf}, v_trf,
3836          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
3837          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
3838       */
3839       {
3840         const PetscInt oc    = 0;
3841         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
3842         const PetscInt oft   = ofb   + PetscSqr(k-1);
3843         const PetscInt off   = oft   + PetscSqr(k-1);
3844         const PetscInt ofk   = off   + PetscSqr(k-1);
3845         const PetscInt ofr   = ofk   + PetscSqr(k-1);
3846         const PetscInt ofl   = ofr   + PetscSqr(k-1);
3847         const PetscInt oebl  = ofl   + PetscSqr(k-1);
3848         const PetscInt oebb  = oebl  + (k-1);
3849         const PetscInt oebr  = oebb  + (k-1);
3850         const PetscInt oebf  = oebr  + (k-1);
3851         const PetscInt oetf  = oebf  + (k-1);
3852         const PetscInt oetr  = oetf  + (k-1);
3853         const PetscInt oetb  = oetr  + (k-1);
3854         const PetscInt oetl  = oetb  + (k-1);
3855         const PetscInt oerf  = oetl  + (k-1);
3856         const PetscInt oelf  = oerf  + (k-1);
3857         const PetscInt oelb  = oelf  + (k-1);
3858         const PetscInt oerb  = oelb  + (k-1);
3859         const PetscInt ovblf = oerb  + (k-1);
3860         const PetscInt ovblb = ovblf + 1;
3861         const PetscInt ovbrb = ovblb + 1;
3862         const PetscInt ovbrf = ovbrb + 1;
3863         const PetscInt ovtlf = ovbrf + 1;
3864         const PetscInt ovtrf = ovtlf + 1;
3865         const PetscInt ovtrb = ovtrf + 1;
3866         const PetscInt ovtlb = ovtrb + 1;
3867         PetscInt       o, n;
3868 
3869         /* Bottom Slice */
3870         /*   bottom */
3871         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
3872         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3873         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
3874         /*   middle */
3875         for (i = 0; i < k-1; ++i) {
3876           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
3877           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
3878           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
3879         }
3880         /*   top */
3881         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
3882         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3883         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
3884 
3885         /* Middle Slice */
3886         for (j = 0; j < k-1; ++j) {
3887           /*   bottom */
3888           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
3889           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3890           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
3891           /*   middle */
3892           for (i = 0; i < k-1; ++i) {
3893             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
3894             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
3895             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
3896           }
3897           /*   top */
3898           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
3899           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3900           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
3901         }
3902 
3903         /* Top Slice */
3904         /*   bottom */
3905         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
3906         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3907         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
3908         /*   middle */
3909         for (i = 0; i < k-1; ++i) {
3910           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
3911           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
3912           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
3913         }
3914         /*   top */
3915         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
3916         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3917         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
3918 
3919         foffset = offset;
3920       }
3921       break;
3922     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
3923     }
3924   }
3925   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
3926   /* Check permutation */
3927   {
3928     PetscInt *check;
3929 
3930     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
3931     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
3932     for (i = 0; i < size; ++i) check[perm[i]] = i;
3933     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
3934     ierr = PetscFree(check);CHKERRQ(ierr);
3935   }
3936   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
3937   PetscFunctionReturn(0);
3938 }
3939 
3940 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
3941 {
3942   PetscDS        prob;
3943   PetscInt       depth, Nf, h;
3944   DMLabel        label;
3945   PetscErrorCode ierr;
3946 
3947   PetscFunctionBeginHot;
3948   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
3949   Nf      = prob->Nf;
3950   label   = dm->depthLabel;
3951   *dspace = NULL;
3952   if (field < Nf) {
3953     PetscObject disc = prob->disc[field];
3954 
3955     if (disc->classid == PETSCFE_CLASSID) {
3956       PetscDualSpace dsp;
3957 
3958       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
3959       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
3960       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
3961       h    = depth - 1 - h;
3962       if (h) {
3963         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
3964       } else {
3965         *dspace = dsp;
3966       }
3967     }
3968   }
3969   PetscFunctionReturn(0);
3970 }
3971 
3972 
3973 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
3974 {
3975   PetscScalar    *array, *vArray;
3976   const PetscInt *cone, *coneO;
3977   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
3978   PetscErrorCode  ierr;
3979 
3980   PetscFunctionBeginHot;
3981   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
3982   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
3983   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3984   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
3985   if (!values || !*values) {
3986     if ((point >= pStart) && (point < pEnd)) {
3987       PetscInt dof;
3988 
3989       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
3990       size += dof;
3991     }
3992     for (p = 0; p < numPoints; ++p) {
3993       const PetscInt cp = cone[p];
3994       PetscInt       dof;
3995 
3996       if ((cp < pStart) || (cp >= pEnd)) continue;
3997       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
3998       size += dof;
3999     }
4000     if (!values) {
4001       if (csize) *csize = size;
4002       PetscFunctionReturn(0);
4003     }
4004     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4005   } else {
4006     array = *values;
4007   }
4008   size = 0;
4009   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4010   if ((point >= pStart) && (point < pEnd)) {
4011     PetscInt     dof, off, d;
4012     PetscScalar *varr;
4013 
4014     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4015     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4016     varr = &vArray[off];
4017     for (d = 0; d < dof; ++d, ++offset) {
4018       array[offset] = varr[d];
4019     }
4020     size += dof;
4021   }
4022   for (p = 0; p < numPoints; ++p) {
4023     const PetscInt cp = cone[p];
4024     PetscInt       o  = coneO[p];
4025     PetscInt       dof, off, d;
4026     PetscScalar   *varr;
4027 
4028     if ((cp < pStart) || (cp >= pEnd)) continue;
4029     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4030     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4031     varr = &vArray[off];
4032     if (o >= 0) {
4033       for (d = 0; d < dof; ++d, ++offset) {
4034         array[offset] = varr[d];
4035       }
4036     } else {
4037       for (d = dof-1; d >= 0; --d, ++offset) {
4038         array[offset] = varr[d];
4039       }
4040     }
4041     size += dof;
4042   }
4043   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4044   if (!*values) {
4045     if (csize) *csize = size;
4046     *values = array;
4047   } else {
4048     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4049     *csize = size;
4050   }
4051   PetscFunctionReturn(0);
4052 }
4053 
4054 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4055 {
4056   const PetscInt *cla;
4057   PetscInt       np, *pts = NULL;
4058   PetscErrorCode ierr;
4059 
4060   PetscFunctionBeginHot;
4061   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4062   if (!*clPoints) {
4063     PetscInt pStart, pEnd, p, q;
4064 
4065     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4066     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4067     /* Compress out points not in the section */
4068     for (p = 0, q = 0; p < np; p++) {
4069       PetscInt r = pts[2*p];
4070       if ((r >= pStart) && (r < pEnd)) {
4071         pts[q*2]   = r;
4072         pts[q*2+1] = pts[2*p+1];
4073         ++q;
4074       }
4075     }
4076     np = q;
4077     cla = NULL;
4078   } else {
4079     PetscInt dof, off;
4080 
4081     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4082     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4083     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4084     np   = dof/2;
4085     pts  = (PetscInt *) &cla[off];
4086   }
4087   *numPoints = np;
4088   *points    = pts;
4089   *clp       = cla;
4090 
4091   PetscFunctionReturn(0);
4092 }
4093 
4094 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4095 {
4096   PetscErrorCode ierr;
4097 
4098   PetscFunctionBeginHot;
4099   if (!*clPoints) {
4100     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4101   } else {
4102     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4103   }
4104   *numPoints = 0;
4105   *points    = NULL;
4106   *clSec     = NULL;
4107   *clPoints  = NULL;
4108   *clp       = NULL;
4109   PetscFunctionReturn(0);
4110 }
4111 
4112 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4113 {
4114   PetscInt          offset = 0, p;
4115   const PetscInt    **perms = NULL;
4116   const PetscScalar **flips = NULL;
4117   PetscErrorCode    ierr;
4118 
4119   PetscFunctionBeginHot;
4120   *size = 0;
4121   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4122   for (p = 0; p < numPoints; p++) {
4123     const PetscInt    point = points[2*p];
4124     const PetscInt    *perm = perms ? perms[p] : NULL;
4125     const PetscScalar *flip = flips ? flips[p] : NULL;
4126     PetscInt          dof, off, d;
4127     const PetscScalar *varr;
4128 
4129     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4130     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4131     varr = &vArray[off];
4132     if (clperm) {
4133       if (perm) {
4134         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4135       } else {
4136         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4137       }
4138       if (flip) {
4139         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4140       }
4141     } else {
4142       if (perm) {
4143         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4144       } else {
4145         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4146       }
4147       if (flip) {
4148         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4149       }
4150     }
4151     offset += dof;
4152   }
4153   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4154   *size = offset;
4155   PetscFunctionReturn(0);
4156 }
4157 
4158 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4159 {
4160   PetscInt          offset = 0, f;
4161   PetscErrorCode    ierr;
4162 
4163   PetscFunctionBeginHot;
4164   *size = 0;
4165   for (f = 0; f < numFields; ++f) {
4166     PetscInt          p;
4167     const PetscInt    **perms = NULL;
4168     const PetscScalar **flips = NULL;
4169 
4170     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4171     for (p = 0; p < numPoints; p++) {
4172       const PetscInt    point = points[2*p];
4173       PetscInt          fdof, foff, b;
4174       const PetscScalar *varr;
4175       const PetscInt    *perm = perms ? perms[p] : NULL;
4176       const PetscScalar *flip = flips ? flips[p] : NULL;
4177 
4178       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4179       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4180       varr = &vArray[foff];
4181       if (clperm) {
4182         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4183         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4184         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4185       } else {
4186         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4187         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4188         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4189       }
4190       offset += fdof;
4191     }
4192     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4193   }
4194   *size = offset;
4195   PetscFunctionReturn(0);
4196 }
4197 
4198 /*@C
4199   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4200 
4201   Not collective
4202 
4203   Input Parameters:
4204 + dm - The DM
4205 . section - The section describing the layout in v, or NULL to use the default section
4206 . v - The local vector
4207 . point - The point in the DM
4208 . csize - The size of the input values array, or NULL
4209 - values - An array to use for the values, or NULL to have it allocated automatically
4210 
4211   Output Parameters:
4212 + csize - The number of values in the closure
4213 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4214 
4215 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4216 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4217 $ assembly function, and a user may already have allocated storage for this operation.
4218 $
4219 $ A typical use could be
4220 $
4221 $  values = NULL;
4222 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4223 $  for (cl = 0; cl < clSize; ++cl) {
4224 $    <Compute on closure>
4225 $  }
4226 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4227 $
4228 $ or
4229 $
4230 $  PetscMalloc1(clMaxSize, &values);
4231 $  for (p = pStart; p < pEnd; ++p) {
4232 $    clSize = clMaxSize;
4233 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4234 $    for (cl = 0; cl < clSize; ++cl) {
4235 $      <Compute on closure>
4236 $    }
4237 $  }
4238 $  PetscFree(values);
4239 
4240   Fortran Notes:
4241   Since it returns an array, this routine is only available in Fortran 90, and you must
4242   include petsc.h90 in your code.
4243 
4244   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4245 
4246   Level: intermediate
4247 
4248 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4249 @*/
4250 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4251 {
4252   PetscSection       clSection;
4253   IS                 clPoints;
4254   PetscScalar       *array;
4255   const PetscScalar *vArray;
4256   PetscInt          *points = NULL;
4257   const PetscInt    *clp, *perm;
4258   PetscInt           depth, numFields, numPoints, size;
4259   PetscErrorCode     ierr;
4260 
4261   PetscFunctionBeginHot;
4262   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4263   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4264   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4265   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4266   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4267   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4268   if (depth == 1 && numFields < 2) {
4269     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4270     PetscFunctionReturn(0);
4271   }
4272   /* Get points */
4273   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4274   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4275   /* Get array */
4276   if (!values || !*values) {
4277     PetscInt asize = 0, dof, p;
4278 
4279     for (p = 0; p < numPoints*2; p += 2) {
4280       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4281       asize += dof;
4282     }
4283     if (!values) {
4284       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4285       if (csize) *csize = asize;
4286       PetscFunctionReturn(0);
4287     }
4288     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4289   } else {
4290     array = *values;
4291   }
4292   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4293   /* Get values */
4294   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4295   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4296   /* Cleanup points */
4297   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4298   /* Cleanup array */
4299   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4300   if (!*values) {
4301     if (csize) *csize = size;
4302     *values = array;
4303   } else {
4304     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4305     *csize = size;
4306   }
4307   PetscFunctionReturn(0);
4308 }
4309 
4310 /*@C
4311   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4312 
4313   Not collective
4314 
4315   Input Parameters:
4316 + dm - The DM
4317 . section - The section describing the layout in v, or NULL to use the default section
4318 . v - The local vector
4319 . point - The point in the DM
4320 . csize - The number of values in the closure, or NULL
4321 - values - The array of values, which is a borrowed array and should not be freed
4322 
4323   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4324 
4325   Fortran Notes:
4326   Since it returns an array, this routine is only available in Fortran 90, and you must
4327   include petsc.h90 in your code.
4328 
4329   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4330 
4331   Level: intermediate
4332 
4333 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4334 @*/
4335 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4336 {
4337   PetscInt       size = 0;
4338   PetscErrorCode ierr;
4339 
4340   PetscFunctionBegin;
4341   /* Should work without recalculating size */
4342   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4343   *values = NULL;
4344   PetscFunctionReturn(0);
4345 }
4346 
4347 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4348 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4349 
4350 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4351 {
4352   PetscInt        cdof;   /* The number of constraints on this point */
4353   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4354   PetscScalar    *a;
4355   PetscInt        off, cind = 0, k;
4356   PetscErrorCode  ierr;
4357 
4358   PetscFunctionBegin;
4359   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4360   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4361   a    = &array[off];
4362   if (!cdof || setBC) {
4363     if (clperm) {
4364       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4365       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4366     } else {
4367       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4368       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4369     }
4370   } else {
4371     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4372     if (clperm) {
4373       if (perm) {for (k = 0; k < dof; ++k) {
4374           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4375           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4376         }
4377       } else {
4378         for (k = 0; k < dof; ++k) {
4379           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4380           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4381         }
4382       }
4383     } else {
4384       if (perm) {
4385         for (k = 0; k < dof; ++k) {
4386           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4387           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4388         }
4389       } else {
4390         for (k = 0; k < dof; ++k) {
4391           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4392           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4393         }
4394       }
4395     }
4396   }
4397   PetscFunctionReturn(0);
4398 }
4399 
4400 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4401 {
4402   PetscInt        cdof;   /* The number of constraints on this point */
4403   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4404   PetscScalar    *a;
4405   PetscInt        off, cind = 0, k;
4406   PetscErrorCode  ierr;
4407 
4408   PetscFunctionBegin;
4409   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4410   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4411   a    = &array[off];
4412   if (cdof) {
4413     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4414     if (clperm) {
4415       if (perm) {
4416         for (k = 0; k < dof; ++k) {
4417           if ((cind < cdof) && (k == cdofs[cind])) {
4418             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4419             cind++;
4420           }
4421         }
4422       } else {
4423         for (k = 0; k < dof; ++k) {
4424           if ((cind < cdof) && (k == cdofs[cind])) {
4425             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4426             cind++;
4427           }
4428         }
4429       }
4430     } else {
4431       if (perm) {
4432         for (k = 0; k < dof; ++k) {
4433           if ((cind < cdof) && (k == cdofs[cind])) {
4434             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4435             cind++;
4436           }
4437         }
4438       } else {
4439         for (k = 0; k < dof; ++k) {
4440           if ((cind < cdof) && (k == cdofs[cind])) {
4441             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4442             cind++;
4443           }
4444         }
4445       }
4446     }
4447   }
4448   PetscFunctionReturn(0);
4449 }
4450 
4451 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4452 {
4453   PetscScalar    *a;
4454   PetscInt        fdof, foff, fcdof, foffset = *offset;
4455   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4456   PetscInt        cind = 0, b;
4457   PetscErrorCode  ierr;
4458 
4459   PetscFunctionBegin;
4460   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4461   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4462   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4463   a    = &array[foff];
4464   if (!fcdof || setBC) {
4465     if (clperm) {
4466       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4467       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4468     } else {
4469       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4470       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4471     }
4472   } else {
4473     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4474     if (clperm) {
4475       if (perm) {
4476         for (b = 0; b < fdof; b++) {
4477           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4478           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4479         }
4480       } else {
4481         for (b = 0; b < fdof; b++) {
4482           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4483           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4484         }
4485       }
4486     } else {
4487       if (perm) {
4488         for (b = 0; b < fdof; b++) {
4489           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4490           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4491         }
4492       } else {
4493         for (b = 0; b < fdof; b++) {
4494           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4495           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4496         }
4497       }
4498     }
4499   }
4500   *offset += fdof;
4501   PetscFunctionReturn(0);
4502 }
4503 
4504 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4505 {
4506   PetscScalar    *a;
4507   PetscInt        fdof, foff, fcdof, foffset = *offset;
4508   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4509   PetscInt        cind = 0, ncind = 0, b;
4510   PetscBool       ncSet, fcSet;
4511   PetscErrorCode  ierr;
4512 
4513   PetscFunctionBegin;
4514   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4515   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4516   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4517   a    = &array[foff];
4518   if (fcdof) {
4519     /* We just override fcdof and fcdofs with Ncc and comps */
4520     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4521     if (clperm) {
4522       if (perm) {
4523         if (comps) {
4524           for (b = 0; b < fdof; b++) {
4525             ncSet = fcSet = PETSC_FALSE;
4526             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4527             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4528             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4529           }
4530         } else {
4531           for (b = 0; b < fdof; b++) {
4532             if ((cind < fcdof) && (b == fcdofs[cind])) {
4533               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4534               ++cind;
4535             }
4536           }
4537         }
4538       } else {
4539         if (comps) {
4540           for (b = 0; b < fdof; b++) {
4541             ncSet = fcSet = PETSC_FALSE;
4542             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4543             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4544             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4545           }
4546         } else {
4547           for (b = 0; b < fdof; b++) {
4548             if ((cind < fcdof) && (b == fcdofs[cind])) {
4549               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4550               ++cind;
4551             }
4552           }
4553         }
4554       }
4555     } else {
4556       if (perm) {
4557         if (comps) {
4558           for (b = 0; b < fdof; b++) {
4559             ncSet = fcSet = PETSC_FALSE;
4560             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4561             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4562             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4563           }
4564         } else {
4565           for (b = 0; b < fdof; b++) {
4566             if ((cind < fcdof) && (b == fcdofs[cind])) {
4567               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4568               ++cind;
4569             }
4570           }
4571         }
4572       } else {
4573         if (comps) {
4574           for (b = 0; b < fdof; b++) {
4575             ncSet = fcSet = PETSC_FALSE;
4576             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4577             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4578             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4579           }
4580         } else {
4581           for (b = 0; b < fdof; b++) {
4582             if ((cind < fcdof) && (b == fcdofs[cind])) {
4583               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4584               ++cind;
4585             }
4586           }
4587         }
4588       }
4589     }
4590   }
4591   *offset += fdof;
4592   PetscFunctionReturn(0);
4593 }
4594 
4595 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4596 {
4597   PetscScalar    *array;
4598   const PetscInt *cone, *coneO;
4599   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4600   PetscErrorCode  ierr;
4601 
4602   PetscFunctionBeginHot;
4603   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4604   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4605   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4606   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4607   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4608   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4609     const PetscInt cp = !p ? point : cone[p-1];
4610     const PetscInt o  = !p ? 0     : coneO[p-1];
4611 
4612     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4613     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4614     /* ADD_VALUES */
4615     {
4616       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4617       PetscScalar    *a;
4618       PetscInt        cdof, coff, cind = 0, k;
4619 
4620       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4621       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4622       a    = &array[coff];
4623       if (!cdof) {
4624         if (o >= 0) {
4625           for (k = 0; k < dof; ++k) {
4626             a[k] += values[off+k];
4627           }
4628         } else {
4629           for (k = 0; k < dof; ++k) {
4630             a[k] += values[off+dof-k-1];
4631           }
4632         }
4633       } else {
4634         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4635         if (o >= 0) {
4636           for (k = 0; k < dof; ++k) {
4637             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4638             a[k] += values[off+k];
4639           }
4640         } else {
4641           for (k = 0; k < dof; ++k) {
4642             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4643             a[k] += values[off+dof-k-1];
4644           }
4645         }
4646       }
4647     }
4648   }
4649   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4650   PetscFunctionReturn(0);
4651 }
4652 
4653 /*@C
4654   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4655 
4656   Not collective
4657 
4658   Input Parameters:
4659 + dm - The DM
4660 . section - The section describing the layout in v, or NULL to use the default section
4661 . v - The local vector
4662 . point - The point in the DM
4663 . values - The array of values
4664 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4665          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4666 
4667   Fortran Notes:
4668   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4669 
4670   Level: intermediate
4671 
4672 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4673 @*/
4674 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4675 {
4676   PetscSection    clSection;
4677   IS              clPoints;
4678   PetscScalar    *array;
4679   PetscInt       *points = NULL;
4680   const PetscInt *clp, *clperm;
4681   PetscInt        depth, numFields, numPoints, p;
4682   PetscErrorCode  ierr;
4683 
4684   PetscFunctionBeginHot;
4685   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4686   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4687   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4688   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4689   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4690   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4691   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4692     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4693     PetscFunctionReturn(0);
4694   }
4695   /* Get points */
4696   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4697   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4698   /* Get array */
4699   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4700   /* Get values */
4701   if (numFields > 0) {
4702     PetscInt offset = 0, f;
4703     for (f = 0; f < numFields; ++f) {
4704       const PetscInt    **perms = NULL;
4705       const PetscScalar **flips = NULL;
4706 
4707       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4708       switch (mode) {
4709       case INSERT_VALUES:
4710         for (p = 0; p < numPoints; p++) {
4711           const PetscInt    point = points[2*p];
4712           const PetscInt    *perm = perms ? perms[p] : NULL;
4713           const PetscScalar *flip = flips ? flips[p] : NULL;
4714           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4715         } break;
4716       case INSERT_ALL_VALUES:
4717         for (p = 0; p < numPoints; p++) {
4718           const PetscInt    point = points[2*p];
4719           const PetscInt    *perm = perms ? perms[p] : NULL;
4720           const PetscScalar *flip = flips ? flips[p] : NULL;
4721           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4722         } break;
4723       case INSERT_BC_VALUES:
4724         for (p = 0; p < numPoints; p++) {
4725           const PetscInt    point = points[2*p];
4726           const PetscInt    *perm = perms ? perms[p] : NULL;
4727           const PetscScalar *flip = flips ? flips[p] : NULL;
4728           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4729         } break;
4730       case ADD_VALUES:
4731         for (p = 0; p < numPoints; p++) {
4732           const PetscInt    point = points[2*p];
4733           const PetscInt    *perm = perms ? perms[p] : NULL;
4734           const PetscScalar *flip = flips ? flips[p] : NULL;
4735           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4736         } break;
4737       case ADD_ALL_VALUES:
4738         for (p = 0; p < numPoints; p++) {
4739           const PetscInt    point = points[2*p];
4740           const PetscInt    *perm = perms ? perms[p] : NULL;
4741           const PetscScalar *flip = flips ? flips[p] : NULL;
4742           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4743         } break;
4744       case ADD_BC_VALUES:
4745         for (p = 0; p < numPoints; p++) {
4746           const PetscInt    point = points[2*p];
4747           const PetscInt    *perm = perms ? perms[p] : NULL;
4748           const PetscScalar *flip = flips ? flips[p] : NULL;
4749           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4750         } break;
4751       default:
4752         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4753       }
4754       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4755     }
4756   } else {
4757     PetscInt dof, off;
4758     const PetscInt    **perms = NULL;
4759     const PetscScalar **flips = NULL;
4760 
4761     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4762     switch (mode) {
4763     case INSERT_VALUES:
4764       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4765         const PetscInt    point = points[2*p];
4766         const PetscInt    *perm = perms ? perms[p] : NULL;
4767         const PetscScalar *flip = flips ? flips[p] : NULL;
4768         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4769         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4770       } break;
4771     case INSERT_ALL_VALUES:
4772       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4773         const PetscInt    point = points[2*p];
4774         const PetscInt    *perm = perms ? perms[p] : NULL;
4775         const PetscScalar *flip = flips ? flips[p] : NULL;
4776         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4777         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4778       } break;
4779     case INSERT_BC_VALUES:
4780       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4781         const PetscInt    point = points[2*p];
4782         const PetscInt    *perm = perms ? perms[p] : NULL;
4783         const PetscScalar *flip = flips ? flips[p] : NULL;
4784         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4785         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4786       } break;
4787     case ADD_VALUES:
4788       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4789         const PetscInt    point = points[2*p];
4790         const PetscInt    *perm = perms ? perms[p] : NULL;
4791         const PetscScalar *flip = flips ? flips[p] : NULL;
4792         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4793         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4794       } break;
4795     case ADD_ALL_VALUES:
4796       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4797         const PetscInt    point = points[2*p];
4798         const PetscInt    *perm = perms ? perms[p] : NULL;
4799         const PetscScalar *flip = flips ? flips[p] : NULL;
4800         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4801         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4802       } break;
4803     case ADD_BC_VALUES:
4804       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4805         const PetscInt    point = points[2*p];
4806         const PetscInt    *perm = perms ? perms[p] : NULL;
4807         const PetscScalar *flip = flips ? flips[p] : NULL;
4808         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4809         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4810       } break;
4811     default:
4812       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4813     }
4814     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4815   }
4816   /* Cleanup points */
4817   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4818   /* Cleanup array */
4819   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4820   PetscFunctionReturn(0);
4821 }
4822 
4823 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4824 {
4825   PetscSection      clSection;
4826   IS                clPoints;
4827   PetscScalar       *array;
4828   PetscInt          *points = NULL;
4829   const PetscInt    *clp, *clperm;
4830   PetscInt          numFields, numPoints, p;
4831   PetscInt          offset = 0, f;
4832   PetscErrorCode    ierr;
4833 
4834   PetscFunctionBeginHot;
4835   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4836   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4837   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4838   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4839   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4840   /* Get points */
4841   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4842   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4843   /* Get array */
4844   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4845   /* Get values */
4846   for (f = 0; f < numFields; ++f) {
4847     const PetscInt    **perms = NULL;
4848     const PetscScalar **flips = NULL;
4849 
4850     if (!fieldActive[f]) {
4851       for (p = 0; p < numPoints*2; p += 2) {
4852         PetscInt fdof;
4853         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
4854         offset += fdof;
4855       }
4856       continue;
4857     }
4858     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4859     switch (mode) {
4860     case INSERT_VALUES:
4861       for (p = 0; p < numPoints; p++) {
4862         const PetscInt    point = points[2*p];
4863         const PetscInt    *perm = perms ? perms[p] : NULL;
4864         const PetscScalar *flip = flips ? flips[p] : NULL;
4865         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4866       } break;
4867     case INSERT_ALL_VALUES:
4868       for (p = 0; p < numPoints; p++) {
4869         const PetscInt    point = points[2*p];
4870         const PetscInt    *perm = perms ? perms[p] : NULL;
4871         const PetscScalar *flip = flips ? flips[p] : NULL;
4872         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4873         } break;
4874     case INSERT_BC_VALUES:
4875       for (p = 0; p < numPoints; p++) {
4876         const PetscInt    point = points[2*p];
4877         const PetscInt    *perm = perms ? perms[p] : NULL;
4878         const PetscScalar *flip = flips ? flips[p] : NULL;
4879         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
4880       } break;
4881     case ADD_VALUES:
4882       for (p = 0; p < numPoints; p++) {
4883         const PetscInt    point = points[2*p];
4884         const PetscInt    *perm = perms ? perms[p] : NULL;
4885         const PetscScalar *flip = flips ? flips[p] : NULL;
4886         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4887       } break;
4888     case ADD_ALL_VALUES:
4889       for (p = 0; p < numPoints; p++) {
4890         const PetscInt    point = points[2*p];
4891         const PetscInt    *perm = perms ? perms[p] : NULL;
4892         const PetscScalar *flip = flips ? flips[p] : NULL;
4893         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4894       } break;
4895     default:
4896       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4897     }
4898     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4899   }
4900   /* Cleanup points */
4901   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4902   /* Cleanup array */
4903   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4904   PetscFunctionReturn(0);
4905 }
4906 
4907 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
4908 {
4909   PetscMPIInt    rank;
4910   PetscInt       i, j;
4911   PetscErrorCode ierr;
4912 
4913   PetscFunctionBegin;
4914   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
4915   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
4916   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
4917   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
4918   numCIndices = numCIndices ? numCIndices : numRIndices;
4919   for (i = 0; i < numRIndices; i++) {
4920     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
4921     for (j = 0; j < numCIndices; j++) {
4922 #if defined(PETSC_USE_COMPLEX)
4923       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
4924 #else
4925       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
4926 #endif
4927     }
4928     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
4929   }
4930   PetscFunctionReturn(0);
4931 }
4932 
4933 /*
4934   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
4935 
4936   Input Parameters:
4937 + section - The section for this data layout
4938 . point   - The point contributing dofs with these indices
4939 . off     - The global offset of this point
4940 . loff    - The local offset of each field
4941 . setBC   - The flag determining whether to include indices of bounsary values
4942 . perm    - A permutation of the dofs on this point, or NULL
4943 - indperm - A permutation of the entire indices array, or NULL
4944 
4945   Output Parameter:
4946 . indices - Indices for dofs on this point
4947 
4948   Level: developer
4949 
4950   Note: The indices could be local or global, depending on the value of 'off'.
4951 */
4952 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
4953 {
4954   PetscInt        dof;   /* The number of unknowns on this point */
4955   PetscInt        cdof;  /* The number of constraints on this point */
4956   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4957   PetscInt        cind = 0, k;
4958   PetscErrorCode  ierr;
4959 
4960   PetscFunctionBegin;
4961   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4962   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4963   if (!cdof || setBC) {
4964     for (k = 0; k < dof; ++k) {
4965       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
4966       const PetscInt ind    = indperm ? indperm[preind] : preind;
4967 
4968       indices[ind] = off + k;
4969     }
4970   } else {
4971     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4972     for (k = 0; k < dof; ++k) {
4973       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
4974       const PetscInt ind    = indperm ? indperm[preind] : preind;
4975 
4976       if ((cind < cdof) && (k == cdofs[cind])) {
4977         /* Insert check for returning constrained indices */
4978         indices[ind] = -(off+k+1);
4979         ++cind;
4980       } else {
4981         indices[ind] = off+k-cind;
4982       }
4983     }
4984   }
4985   *loff += dof;
4986   PetscFunctionReturn(0);
4987 }
4988 
4989 /*
4990   This version only believes the point offset from the globalSection
4991 
4992  . off - The global offset of this point
4993 */
4994 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
4995 {
4996   PetscInt       numFields, foff, f;
4997   PetscErrorCode ierr;
4998 
4999   PetscFunctionBegin;
5000   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5001   for (f = 0, foff = 0; f < numFields; ++f) {
5002     PetscInt        fdof, cfdof;
5003     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5004     PetscInt        cind = 0, b;
5005     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5006 
5007     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5008     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5009     if (!cfdof || setBC) {
5010       for (b = 0; b < fdof; ++b) {
5011         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5012         const PetscInt ind    = indperm ? indperm[preind] : preind;
5013 
5014         indices[ind] = off+foff+b;
5015       }
5016     } else {
5017       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5018       for (b = 0; b < fdof; ++b) {
5019         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5020         const PetscInt ind    = indperm ? indperm[preind] : preind;
5021 
5022         if ((cind < cfdof) && (b == fcdofs[cind])) {
5023           indices[ind] = -(off+foff+b+1);
5024           ++cind;
5025         } else {
5026           indices[ind] = off+foff+b-cind;
5027         }
5028       }
5029     }
5030     foff     += (setBC ? fdof : (fdof - cfdof));
5031     foffs[f] += fdof;
5032   }
5033   PetscFunctionReturn(0);
5034 }
5035 
5036 /*
5037   This version believes the globalSection offsets for each field, rather than just the point offset
5038 
5039  . foffs - The offset into 'indices' for each field, since it is segregated by field
5040 */
5041 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5042 {
5043   PetscInt       numFields, foff, f;
5044   PetscErrorCode ierr;
5045 
5046   PetscFunctionBegin;
5047   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5048   for (f = 0; f < numFields; ++f) {
5049     PetscInt        fdof, cfdof;
5050     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5051     PetscInt        cind = 0, b;
5052     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5053 
5054     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5055     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5056     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5057     if (!cfdof || setBC) {
5058       for (b = 0; b < fdof; ++b) {
5059         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5060         const PetscInt ind    = indperm ? indperm[preind] : preind;
5061 
5062         indices[ind] = foff+b;
5063       }
5064     } else {
5065       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5066       for (b = 0; b < fdof; ++b) {
5067         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5068         const PetscInt ind    = indperm ? indperm[preind] : preind;
5069 
5070         if ((cind < cfdof) && (b == fcdofs[cind])) {
5071           indices[ind] = -(foff+b+1);
5072           ++cind;
5073         } else {
5074           indices[ind] = foff+b-cind;
5075         }
5076       }
5077     }
5078     foffs[f] += fdof;
5079   }
5080   PetscFunctionReturn(0);
5081 }
5082 
5083 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5084 {
5085   Mat             cMat;
5086   PetscSection    aSec, cSec;
5087   IS              aIS;
5088   PetscInt        aStart = -1, aEnd = -1;
5089   const PetscInt  *anchors;
5090   PetscInt        numFields, f, p, q, newP = 0;
5091   PetscInt        newNumPoints = 0, newNumIndices = 0;
5092   PetscInt        *newPoints, *indices, *newIndices;
5093   PetscInt        maxAnchor, maxDof;
5094   PetscInt        newOffsets[32];
5095   PetscInt        *pointMatOffsets[32];
5096   PetscInt        *newPointOffsets[32];
5097   PetscScalar     *pointMat[32];
5098   PetscScalar     *newValues=NULL,*tmpValues;
5099   PetscBool       anyConstrained = PETSC_FALSE;
5100   PetscErrorCode  ierr;
5101 
5102   PetscFunctionBegin;
5103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5104   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5105   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5106 
5107   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5108   /* if there are point-to-point constraints */
5109   if (aSec) {
5110     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5111     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5112     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5113     /* figure out how many points are going to be in the new element matrix
5114      * (we allow double counting, because it's all just going to be summed
5115      * into the global matrix anyway) */
5116     for (p = 0; p < 2*numPoints; p+=2) {
5117       PetscInt b    = points[p];
5118       PetscInt bDof = 0, bSecDof;
5119 
5120       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5121       if (!bSecDof) {
5122         continue;
5123       }
5124       if (b >= aStart && b < aEnd) {
5125         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5126       }
5127       if (bDof) {
5128         /* this point is constrained */
5129         /* it is going to be replaced by its anchors */
5130         PetscInt bOff, q;
5131 
5132         anyConstrained = PETSC_TRUE;
5133         newNumPoints  += bDof;
5134         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5135         for (q = 0; q < bDof; q++) {
5136           PetscInt a = anchors[bOff + q];
5137           PetscInt aDof;
5138 
5139           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5140           newNumIndices += aDof;
5141           for (f = 0; f < numFields; ++f) {
5142             PetscInt fDof;
5143 
5144             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5145             newOffsets[f+1] += fDof;
5146           }
5147         }
5148       }
5149       else {
5150         /* this point is not constrained */
5151         newNumPoints++;
5152         newNumIndices += bSecDof;
5153         for (f = 0; f < numFields; ++f) {
5154           PetscInt fDof;
5155 
5156           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5157           newOffsets[f+1] += fDof;
5158         }
5159       }
5160     }
5161   }
5162   if (!anyConstrained) {
5163     if (outNumPoints)  *outNumPoints  = 0;
5164     if (outNumIndices) *outNumIndices = 0;
5165     if (outPoints)     *outPoints     = NULL;
5166     if (outValues)     *outValues     = NULL;
5167     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5168     PetscFunctionReturn(0);
5169   }
5170 
5171   if (outNumPoints)  *outNumPoints  = newNumPoints;
5172   if (outNumIndices) *outNumIndices = newNumIndices;
5173 
5174   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5175 
5176   if (!outPoints && !outValues) {
5177     if (offsets) {
5178       for (f = 0; f <= numFields; f++) {
5179         offsets[f] = newOffsets[f];
5180       }
5181     }
5182     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5183     PetscFunctionReturn(0);
5184   }
5185 
5186   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5187 
5188   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5189 
5190   /* workspaces */
5191   if (numFields) {
5192     for (f = 0; f < numFields; f++) {
5193       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5194       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5195     }
5196   }
5197   else {
5198     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5199     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5200   }
5201 
5202   /* get workspaces for the point-to-point matrices */
5203   if (numFields) {
5204     PetscInt totalOffset, totalMatOffset;
5205 
5206     for (p = 0; p < numPoints; p++) {
5207       PetscInt b    = points[2*p];
5208       PetscInt bDof = 0, bSecDof;
5209 
5210       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5211       if (!bSecDof) {
5212         for (f = 0; f < numFields; f++) {
5213           newPointOffsets[f][p + 1] = 0;
5214           pointMatOffsets[f][p + 1] = 0;
5215         }
5216         continue;
5217       }
5218       if (b >= aStart && b < aEnd) {
5219         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5220       }
5221       if (bDof) {
5222         for (f = 0; f < numFields; f++) {
5223           PetscInt fDof, q, bOff, allFDof = 0;
5224 
5225           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5226           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5227           for (q = 0; q < bDof; q++) {
5228             PetscInt a = anchors[bOff + q];
5229             PetscInt aFDof;
5230 
5231             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5232             allFDof += aFDof;
5233           }
5234           newPointOffsets[f][p+1] = allFDof;
5235           pointMatOffsets[f][p+1] = fDof * allFDof;
5236         }
5237       }
5238       else {
5239         for (f = 0; f < numFields; f++) {
5240           PetscInt fDof;
5241 
5242           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5243           newPointOffsets[f][p+1] = fDof;
5244           pointMatOffsets[f][p+1] = 0;
5245         }
5246       }
5247     }
5248     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5249       newPointOffsets[f][0] = totalOffset;
5250       pointMatOffsets[f][0] = totalMatOffset;
5251       for (p = 0; p < numPoints; p++) {
5252         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5253         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5254       }
5255       totalOffset    = newPointOffsets[f][numPoints];
5256       totalMatOffset = pointMatOffsets[f][numPoints];
5257       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5258     }
5259   }
5260   else {
5261     for (p = 0; p < numPoints; p++) {
5262       PetscInt b    = points[2*p];
5263       PetscInt bDof = 0, bSecDof;
5264 
5265       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5266       if (!bSecDof) {
5267         newPointOffsets[0][p + 1] = 0;
5268         pointMatOffsets[0][p + 1] = 0;
5269         continue;
5270       }
5271       if (b >= aStart && b < aEnd) {
5272         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5273       }
5274       if (bDof) {
5275         PetscInt bOff, q, allDof = 0;
5276 
5277         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5278         for (q = 0; q < bDof; q++) {
5279           PetscInt a = anchors[bOff + q], aDof;
5280 
5281           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5282           allDof += aDof;
5283         }
5284         newPointOffsets[0][p+1] = allDof;
5285         pointMatOffsets[0][p+1] = bSecDof * allDof;
5286       }
5287       else {
5288         newPointOffsets[0][p+1] = bSecDof;
5289         pointMatOffsets[0][p+1] = 0;
5290       }
5291     }
5292     newPointOffsets[0][0] = 0;
5293     pointMatOffsets[0][0] = 0;
5294     for (p = 0; p < numPoints; p++) {
5295       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5296       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5297     }
5298     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5299   }
5300 
5301   /* output arrays */
5302   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5303 
5304   /* get the point-to-point matrices; construct newPoints */
5305   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5306   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5307   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5308   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5309   if (numFields) {
5310     for (p = 0, newP = 0; p < numPoints; p++) {
5311       PetscInt b    = points[2*p];
5312       PetscInt o    = points[2*p+1];
5313       PetscInt bDof = 0, bSecDof;
5314 
5315       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5316       if (!bSecDof) {
5317         continue;
5318       }
5319       if (b >= aStart && b < aEnd) {
5320         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5321       }
5322       if (bDof) {
5323         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5324 
5325         fStart[0] = 0;
5326         fEnd[0]   = 0;
5327         for (f = 0; f < numFields; f++) {
5328           PetscInt fDof;
5329 
5330           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5331           fStart[f+1] = fStart[f] + fDof;
5332           fEnd[f+1]   = fStart[f+1];
5333         }
5334         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5335         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5336 
5337         fAnchorStart[0] = 0;
5338         fAnchorEnd[0]   = 0;
5339         for (f = 0; f < numFields; f++) {
5340           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5341 
5342           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5343           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5344         }
5345         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5346         for (q = 0; q < bDof; q++) {
5347           PetscInt a = anchors[bOff + q], aOff;
5348 
5349           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5350           newPoints[2*(newP + q)]     = a;
5351           newPoints[2*(newP + q) + 1] = 0;
5352           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5353           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5354         }
5355         newP += bDof;
5356 
5357         if (outValues) {
5358           /* get the point-to-point submatrix */
5359           for (f = 0; f < numFields; f++) {
5360             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5361           }
5362         }
5363       }
5364       else {
5365         newPoints[2 * newP]     = b;
5366         newPoints[2 * newP + 1] = o;
5367         newP++;
5368       }
5369     }
5370   } else {
5371     for (p = 0; p < numPoints; p++) {
5372       PetscInt b    = points[2*p];
5373       PetscInt o    = points[2*p+1];
5374       PetscInt bDof = 0, bSecDof;
5375 
5376       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5377       if (!bSecDof) {
5378         continue;
5379       }
5380       if (b >= aStart && b < aEnd) {
5381         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5382       }
5383       if (bDof) {
5384         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5385 
5386         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5387         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5388 
5389         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5390         for (q = 0; q < bDof; q++) {
5391           PetscInt a = anchors[bOff + q], aOff;
5392 
5393           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5394 
5395           newPoints[2*(newP + q)]     = a;
5396           newPoints[2*(newP + q) + 1] = 0;
5397           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5398           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5399         }
5400         newP += bDof;
5401 
5402         /* get the point-to-point submatrix */
5403         if (outValues) {
5404           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5405         }
5406       }
5407       else {
5408         newPoints[2 * newP]     = b;
5409         newPoints[2 * newP + 1] = o;
5410         newP++;
5411       }
5412     }
5413   }
5414 
5415   if (outValues) {
5416     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5417     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5418     /* multiply constraints on the right */
5419     if (numFields) {
5420       for (f = 0; f < numFields; f++) {
5421         PetscInt oldOff = offsets[f];
5422 
5423         for (p = 0; p < numPoints; p++) {
5424           PetscInt cStart = newPointOffsets[f][p];
5425           PetscInt b      = points[2 * p];
5426           PetscInt c, r, k;
5427           PetscInt dof;
5428 
5429           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5430           if (!dof) {
5431             continue;
5432           }
5433           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5434             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5435             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5436 
5437             for (r = 0; r < numIndices; r++) {
5438               for (c = 0; c < nCols; c++) {
5439                 for (k = 0; k < dof; k++) {
5440                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5441                 }
5442               }
5443             }
5444           }
5445           else {
5446             /* copy this column as is */
5447             for (r = 0; r < numIndices; r++) {
5448               for (c = 0; c < dof; c++) {
5449                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5450               }
5451             }
5452           }
5453           oldOff += dof;
5454         }
5455       }
5456     }
5457     else {
5458       PetscInt oldOff = 0;
5459       for (p = 0; p < numPoints; p++) {
5460         PetscInt cStart = newPointOffsets[0][p];
5461         PetscInt b      = points[2 * p];
5462         PetscInt c, r, k;
5463         PetscInt dof;
5464 
5465         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5466         if (!dof) {
5467           continue;
5468         }
5469         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5470           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5471           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5472 
5473           for (r = 0; r < numIndices; r++) {
5474             for (c = 0; c < nCols; c++) {
5475               for (k = 0; k < dof; k++) {
5476                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5477               }
5478             }
5479           }
5480         }
5481         else {
5482           /* copy this column as is */
5483           for (r = 0; r < numIndices; r++) {
5484             for (c = 0; c < dof; c++) {
5485               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5486             }
5487           }
5488         }
5489         oldOff += dof;
5490       }
5491     }
5492 
5493     if (multiplyLeft) {
5494       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5495       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5496       /* multiply constraints transpose on the left */
5497       if (numFields) {
5498         for (f = 0; f < numFields; f++) {
5499           PetscInt oldOff = offsets[f];
5500 
5501           for (p = 0; p < numPoints; p++) {
5502             PetscInt rStart = newPointOffsets[f][p];
5503             PetscInt b      = points[2 * p];
5504             PetscInt c, r, k;
5505             PetscInt dof;
5506 
5507             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5508             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5509               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5510               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5511 
5512               for (r = 0; r < nRows; r++) {
5513                 for (c = 0; c < newNumIndices; c++) {
5514                   for (k = 0; k < dof; k++) {
5515                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5516                   }
5517                 }
5518               }
5519             }
5520             else {
5521               /* copy this row as is */
5522               for (r = 0; r < dof; r++) {
5523                 for (c = 0; c < newNumIndices; c++) {
5524                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5525                 }
5526               }
5527             }
5528             oldOff += dof;
5529           }
5530         }
5531       }
5532       else {
5533         PetscInt oldOff = 0;
5534 
5535         for (p = 0; p < numPoints; p++) {
5536           PetscInt rStart = newPointOffsets[0][p];
5537           PetscInt b      = points[2 * p];
5538           PetscInt c, r, k;
5539           PetscInt dof;
5540 
5541           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5542           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5543             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5544             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5545 
5546             for (r = 0; r < nRows; r++) {
5547               for (c = 0; c < newNumIndices; c++) {
5548                 for (k = 0; k < dof; k++) {
5549                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5550                 }
5551               }
5552             }
5553           }
5554           else {
5555             /* copy this row as is */
5556             for (r = 0; r < dof; r++) {
5557               for (c = 0; c < newNumIndices; c++) {
5558                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5559               }
5560             }
5561           }
5562           oldOff += dof;
5563         }
5564       }
5565 
5566       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5567     }
5568     else {
5569       newValues = tmpValues;
5570     }
5571   }
5572 
5573   /* clean up */
5574   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5575   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5576 
5577   if (numFields) {
5578     for (f = 0; f < numFields; f++) {
5579       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5580       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5581       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5582     }
5583   }
5584   else {
5585     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5586     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5587     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5588   }
5589   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5590 
5591   /* output */
5592   if (outPoints) {
5593     *outPoints = newPoints;
5594   }
5595   else {
5596     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5597   }
5598   if (outValues) {
5599     *outValues = newValues;
5600   }
5601   for (f = 0; f <= numFields; f++) {
5602     offsets[f] = newOffsets[f];
5603   }
5604   PetscFunctionReturn(0);
5605 }
5606 
5607 /*@C
5608   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5609 
5610   Not collective
5611 
5612   Input Parameters:
5613 + dm - The DM
5614 . section - The section describing the layout in v, or NULL to use the default section
5615 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5616 - point - The mesh point
5617 
5618   Output parameters:
5619 + numIndices - The number of indices
5620 . indices - The indices
5621 - outOffsets - Field offset if not NULL
5622 
5623   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5624 
5625   Level: advanced
5626 
5627 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5628 @*/
5629 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5630 {
5631   PetscSection    clSection;
5632   IS              clPoints;
5633   const PetscInt *clp, *clperm;
5634   const PetscInt  **perms[32] = {NULL};
5635   PetscInt       *points = NULL, *pointsNew;
5636   PetscInt        numPoints, numPointsNew;
5637   PetscInt        offsets[32];
5638   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5639   PetscErrorCode  ierr;
5640 
5641   PetscFunctionBegin;
5642   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5643   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5644   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5645   if (numIndices) PetscValidPointer(numIndices, 4);
5646   PetscValidPointer(indices, 5);
5647   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5648   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5649   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5650   /* Get points in closure */
5651   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5652   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5653   /* Get number of indices and indices per field */
5654   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5655     PetscInt dof, fdof;
5656 
5657     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5658     for (f = 0; f < Nf; ++f) {
5659       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5660       offsets[f+1] += fdof;
5661     }
5662     Nind += dof;
5663   }
5664   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5665   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5666   if (!Nf) offsets[1] = Nind;
5667   /* Get dual space symmetries */
5668   for (f = 0; f < PetscMax(1,Nf); f++) {
5669     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5670     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5671   }
5672   /* Correct for hanging node constraints */
5673   {
5674     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5675     if (numPointsNew) {
5676       for (f = 0; f < PetscMax(1,Nf); f++) {
5677         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5678         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5679       }
5680       for (f = 0; f < PetscMax(1,Nf); f++) {
5681         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5682         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5683       }
5684       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5685       numPoints = numPointsNew;
5686       Nind      = NindNew;
5687       points    = pointsNew;
5688     }
5689   }
5690   /* Calculate indices */
5691   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5692   if (Nf) {
5693     if (outOffsets) {
5694       PetscInt f;
5695 
5696       for (f = 0; f <= Nf; f++) {
5697         outOffsets[f] = offsets[f];
5698       }
5699     }
5700     for (p = 0; p < numPoints; p++) {
5701       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5702       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);
5703     }
5704   } else {
5705     for (p = 0, off = 0; p < numPoints; p++) {
5706       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5707 
5708       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5709       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);
5710     }
5711   }
5712   /* Cleanup points */
5713   for (f = 0; f < PetscMax(1,Nf); f++) {
5714     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5715     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5716   }
5717   if (numPointsNew) {
5718     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5719   } else {
5720     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5721   }
5722   if (numIndices) *numIndices = Nind;
5723   PetscFunctionReturn(0);
5724 }
5725 
5726 /*@C
5727   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5728 
5729   Not collective
5730 
5731   Input Parameters:
5732 + dm - The DM
5733 . section - The section describing the layout in v, or NULL to use the default section
5734 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5735 . point - The mesh point
5736 . numIndices - The number of indices
5737 . indices - The indices
5738 - outOffsets - Field offset if not NULL
5739 
5740   Level: advanced
5741 
5742 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5743 @*/
5744 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5745 {
5746   PetscErrorCode ierr;
5747 
5748   PetscFunctionBegin;
5749   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5750   PetscValidPointer(indices, 5);
5751   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5752   PetscFunctionReturn(0);
5753 }
5754 
5755 /*@C
5756   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5757 
5758   Not collective
5759 
5760   Input Parameters:
5761 + dm - The DM
5762 . section - The section describing the layout in v, or NULL to use the default section
5763 . globalSection - The section describing the layout in v, or NULL to use the default global section
5764 . A - The matrix
5765 . point - The point in the DM
5766 . values - The array of values
5767 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5768 
5769   Fortran Notes:
5770   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5771 
5772   Level: intermediate
5773 
5774 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5775 @*/
5776 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5777 {
5778   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5779   PetscSection        clSection;
5780   IS                  clPoints;
5781   PetscInt           *points = NULL, *newPoints;
5782   const PetscInt     *clp, *clperm;
5783   PetscInt           *indices;
5784   PetscInt            offsets[32];
5785   const PetscInt    **perms[32] = {NULL};
5786   const PetscScalar **flips[32] = {NULL};
5787   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5788   PetscScalar        *valCopy = NULL;
5789   PetscScalar        *newValues;
5790   PetscErrorCode      ierr;
5791 
5792   PetscFunctionBegin;
5793   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5794   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
5795   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5796   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5797   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5798   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5799   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5800   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5801   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5802   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5803   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5804   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5805     PetscInt fdof;
5806 
5807     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5808     for (f = 0; f < numFields; ++f) {
5809       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5810       offsets[f+1] += fdof;
5811     }
5812     numIndices += dof;
5813   }
5814   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5815 
5816   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5817   /* Get symmetries */
5818   for (f = 0; f < PetscMax(1,numFields); f++) {
5819     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5820     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5821     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5822       PetscInt foffset = offsets[f];
5823 
5824       for (p = 0; p < numPoints; p++) {
5825         PetscInt point          = points[2*p], fdof;
5826         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5827 
5828         if (!numFields) {
5829           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5830         } else {
5831           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5832         }
5833         if (flip) {
5834           PetscInt i, j, k;
5835 
5836           if (!valCopy) {
5837             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5838             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
5839             values = valCopy;
5840           }
5841           for (i = 0; i < fdof; i++) {
5842             PetscScalar fval = flip[i];
5843 
5844             for (k = 0; k < numIndices; k++) {
5845               valCopy[numIndices * (foffset + i) + k] *= fval;
5846               valCopy[numIndices * k + (foffset + i)] *= fval;
5847             }
5848           }
5849         }
5850         foffset += fdof;
5851       }
5852     }
5853   }
5854   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
5855   if (newNumPoints) {
5856     if (valCopy) {
5857       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5858     }
5859     for (f = 0; f < PetscMax(1,numFields); f++) {
5860       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5861       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5862     }
5863     for (f = 0; f < PetscMax(1,numFields); f++) {
5864       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5865       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5866     }
5867     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5868     numPoints  = newNumPoints;
5869     numIndices = newNumIndices;
5870     points     = newPoints;
5871     values     = newValues;
5872   }
5873   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5874   if (numFields) {
5875     PetscBool useFieldOffsets;
5876 
5877     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
5878     if (useFieldOffsets) {
5879       for (p = 0; p < numPoints; p++) {
5880         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
5881       }
5882     } else {
5883       for (p = 0; p < numPoints; p++) {
5884         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5885         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
5886       }
5887     }
5888   } else {
5889     for (p = 0, off = 0; p < numPoints; p++) {
5890       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5891       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5892       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
5893     }
5894   }
5895   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5896   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5897   if (mesh->printFEM > 1) {
5898     PetscInt i;
5899     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
5900     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
5901     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
5902   }
5903   if (ierr) {
5904     PetscMPIInt    rank;
5905     PetscErrorCode ierr2;
5906 
5907     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5908     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5909     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5910     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
5911     CHKERRQ(ierr);
5912   }
5913   for (f = 0; f < PetscMax(1,numFields); f++) {
5914     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5915     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5916   }
5917   if (newNumPoints) {
5918     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5919     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5920   }
5921   else {
5922     if (valCopy) {
5923       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5924     }
5925     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5926   }
5927   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5928   PetscFunctionReturn(0);
5929 }
5930 
5931 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5932 {
5933   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5934   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5935   PetscInt       *cpoints = NULL;
5936   PetscInt       *findices, *cindices;
5937   const PetscInt *fclperm, *cclperm;
5938   PetscInt        foffsets[32], coffsets[32];
5939   CellRefiner     cellRefiner;
5940   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5941   PetscErrorCode  ierr;
5942 
5943   PetscFunctionBegin;
5944   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5945   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5946   if (!fsection) {ierr = DMGetSection(dmf, &fsection);CHKERRQ(ierr);}
5947   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5948   if (!csection) {ierr = DMGetSection(dmc, &csection);CHKERRQ(ierr);}
5949   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5950   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5951   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5952   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5953   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5954   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5955   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5956   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5957   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
5958   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
5959   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
5960   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
5961   /* Column indices */
5962   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5963   maxFPoints = numCPoints;
5964   /* Compress out points not in the section */
5965   /*   TODO: Squeeze out points with 0 dof as well */
5966   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5967   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5968     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5969       cpoints[q*2]   = cpoints[p];
5970       cpoints[q*2+1] = cpoints[p+1];
5971       ++q;
5972     }
5973   }
5974   numCPoints = q;
5975   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5976     PetscInt fdof;
5977 
5978     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5979     if (!dof) continue;
5980     for (f = 0; f < numFields; ++f) {
5981       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5982       coffsets[f+1] += fdof;
5983     }
5984     numCIndices += dof;
5985   }
5986   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5987   /* Row indices */
5988   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5989   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5990   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
5991   for (r = 0, q = 0; r < numSubcells; ++r) {
5992     /* TODO Map from coarse to fine cells */
5993     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
5994     /* Compress out points not in the section */
5995     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
5996     for (p = 0; p < numFPoints*2; p += 2) {
5997       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
5998         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
5999         if (!dof) continue;
6000         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6001         if (s < q) continue;
6002         ftotpoints[q*2]   = fpoints[p];
6003         ftotpoints[q*2+1] = fpoints[p+1];
6004         ++q;
6005       }
6006     }
6007     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6008   }
6009   numFPoints = q;
6010   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6011     PetscInt fdof;
6012 
6013     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6014     if (!dof) continue;
6015     for (f = 0; f < numFields; ++f) {
6016       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6017       foffsets[f+1] += fdof;
6018     }
6019     numFIndices += dof;
6020   }
6021   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6022 
6023   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6024   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6025   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6026   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6027   if (numFields) {
6028     const PetscInt **permsF[32] = {NULL};
6029     const PetscInt **permsC[32] = {NULL};
6030 
6031     for (f = 0; f < numFields; f++) {
6032       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6033       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6034     }
6035     for (p = 0; p < numFPoints; p++) {
6036       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6037       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6038     }
6039     for (p = 0; p < numCPoints; p++) {
6040       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6041       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6042     }
6043     for (f = 0; f < numFields; f++) {
6044       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6045       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6046     }
6047   } else {
6048     const PetscInt **permsF = NULL;
6049     const PetscInt **permsC = NULL;
6050 
6051     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6052     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6053     for (p = 0, off = 0; p < numFPoints; p++) {
6054       const PetscInt *perm = permsF ? permsF[p] : NULL;
6055 
6056       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6057       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6058     }
6059     for (p = 0, off = 0; p < numCPoints; p++) {
6060       const PetscInt *perm = permsC ? permsC[p] : NULL;
6061 
6062       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6063       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6064     }
6065     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6066     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6067   }
6068   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6069   /* TODO: flips */
6070   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6071   if (ierr) {
6072     PetscMPIInt    rank;
6073     PetscErrorCode ierr2;
6074 
6075     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6076     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6077     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6078     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6079     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6080     CHKERRQ(ierr);
6081   }
6082   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6083   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6084   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6085   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6086   PetscFunctionReturn(0);
6087 }
6088 
6089 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6090 {
6091   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6092   PetscInt      *cpoints = NULL;
6093   PetscInt       foffsets[32], coffsets[32];
6094   const PetscInt *fclperm, *cclperm;
6095   CellRefiner    cellRefiner;
6096   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6097   PetscErrorCode ierr;
6098 
6099   PetscFunctionBegin;
6100   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6101   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6102   if (!fsection) {ierr = DMGetSection(dmf, &fsection);CHKERRQ(ierr);}
6103   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6104   if (!csection) {ierr = DMGetSection(dmc, &csection);CHKERRQ(ierr);}
6105   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6106   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6107   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6108   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6109   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6110   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6111   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6112   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6113   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6114   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6115   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6116   /* Column indices */
6117   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6118   maxFPoints = numCPoints;
6119   /* Compress out points not in the section */
6120   /*   TODO: Squeeze out points with 0 dof as well */
6121   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6122   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6123     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6124       cpoints[q*2]   = cpoints[p];
6125       cpoints[q*2+1] = cpoints[p+1];
6126       ++q;
6127     }
6128   }
6129   numCPoints = q;
6130   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6131     PetscInt fdof;
6132 
6133     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6134     if (!dof) continue;
6135     for (f = 0; f < numFields; ++f) {
6136       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6137       coffsets[f+1] += fdof;
6138     }
6139     numCIndices += dof;
6140   }
6141   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6142   /* Row indices */
6143   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6144   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6145   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6146   for (r = 0, q = 0; r < numSubcells; ++r) {
6147     /* TODO Map from coarse to fine cells */
6148     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6149     /* Compress out points not in the section */
6150     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6151     for (p = 0; p < numFPoints*2; p += 2) {
6152       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6153         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6154         if (!dof) continue;
6155         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6156         if (s < q) continue;
6157         ftotpoints[q*2]   = fpoints[p];
6158         ftotpoints[q*2+1] = fpoints[p+1];
6159         ++q;
6160       }
6161     }
6162     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6163   }
6164   numFPoints = q;
6165   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6166     PetscInt fdof;
6167 
6168     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6169     if (!dof) continue;
6170     for (f = 0; f < numFields; ++f) {
6171       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6172       foffsets[f+1] += fdof;
6173     }
6174     numFIndices += dof;
6175   }
6176   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6177 
6178   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6179   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6180   if (numFields) {
6181     const PetscInt **permsF[32] = {NULL};
6182     const PetscInt **permsC[32] = {NULL};
6183 
6184     for (f = 0; f < numFields; f++) {
6185       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6186       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6187     }
6188     for (p = 0; p < numFPoints; p++) {
6189       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6190       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6191     }
6192     for (p = 0; p < numCPoints; p++) {
6193       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6194       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6195     }
6196     for (f = 0; f < numFields; f++) {
6197       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6198       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6199     }
6200   } else {
6201     const PetscInt **permsF = NULL;
6202     const PetscInt **permsC = NULL;
6203 
6204     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6205     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6206     for (p = 0, off = 0; p < numFPoints; p++) {
6207       const PetscInt *perm = permsF ? permsF[p] : NULL;
6208 
6209       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6210       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6211     }
6212     for (p = 0, off = 0; p < numCPoints; p++) {
6213       const PetscInt *perm = permsC ? permsC[p] : NULL;
6214 
6215       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6216       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6217     }
6218     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6219     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6220   }
6221   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6222   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6223   PetscFunctionReturn(0);
6224 }
6225 
6226 /*@
6227   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6228 
6229   Input Parameter:
6230 . dm - The DMPlex object
6231 
6232   Output Parameters:
6233 + cMax - The first hybrid cell
6234 . fMax - The first hybrid face
6235 . eMax - The first hybrid edge
6236 - vMax - The first hybrid vertex
6237 
6238   Level: developer
6239 
6240 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6241 @*/
6242 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6243 {
6244   DM_Plex       *mesh = (DM_Plex*) dm->data;
6245   PetscInt       dim;
6246   PetscErrorCode ierr;
6247 
6248   PetscFunctionBegin;
6249   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6250   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6251   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6252   if (cMax) *cMax = mesh->hybridPointMax[dim];
6253   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6254   if (eMax) *eMax = mesh->hybridPointMax[1];
6255   if (vMax) *vMax = mesh->hybridPointMax[0];
6256   PetscFunctionReturn(0);
6257 }
6258 
6259 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6260 {
6261   IS             is, his;
6262   PetscInt       first = 0, stride;
6263   PetscBool      isStride;
6264   PetscErrorCode ierr;
6265 
6266   PetscFunctionBegin;
6267   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6268   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6269   if (isStride) {
6270     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6271   }
6272   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6273   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6274   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6275   ierr = ISDestroy(&his);CHKERRQ(ierr);
6276   ierr = ISDestroy(&is);CHKERRQ(ierr);
6277   PetscFunctionReturn(0);
6278 }
6279 
6280 /*@
6281   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6282 
6283   Input Parameters:
6284 . dm   - The DMPlex object
6285 . cMax - The first hybrid cell
6286 . fMax - The first hybrid face
6287 . eMax - The first hybrid edge
6288 - vMax - The first hybrid vertex
6289 
6290   Level: developer
6291 
6292 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6293 @*/
6294 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6295 {
6296   DM_Plex       *mesh = (DM_Plex*) dm->data;
6297   PetscInt       dim;
6298   PetscErrorCode ierr;
6299 
6300   PetscFunctionBegin;
6301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6302   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6303   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6304   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6305   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6306   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6307   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6308   PetscFunctionReturn(0);
6309 }
6310 
6311 /*@C
6312   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6313 
6314   Input Parameter:
6315 . dm   - The DMPlex object
6316 
6317   Output Parameter:
6318 . cellHeight - The height of a cell
6319 
6320   Level: developer
6321 
6322 .seealso DMPlexSetVTKCellHeight()
6323 @*/
6324 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6325 {
6326   DM_Plex *mesh = (DM_Plex*) dm->data;
6327 
6328   PetscFunctionBegin;
6329   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6330   PetscValidPointer(cellHeight, 2);
6331   *cellHeight = mesh->vtkCellHeight;
6332   PetscFunctionReturn(0);
6333 }
6334 
6335 /*@C
6336   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6337 
6338   Input Parameters:
6339 + dm   - The DMPlex object
6340 - cellHeight - The height of a cell
6341 
6342   Level: developer
6343 
6344 .seealso DMPlexGetVTKCellHeight()
6345 @*/
6346 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6347 {
6348   DM_Plex *mesh = (DM_Plex*) dm->data;
6349 
6350   PetscFunctionBegin;
6351   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6352   mesh->vtkCellHeight = cellHeight;
6353   PetscFunctionReturn(0);
6354 }
6355 
6356 /* We can easily have a form that takes an IS instead */
6357 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6358 {
6359   PetscSection   section, globalSection;
6360   PetscInt      *numbers, p;
6361   PetscErrorCode ierr;
6362 
6363   PetscFunctionBegin;
6364   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6365   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6366   for (p = pStart; p < pEnd; ++p) {
6367     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6368   }
6369   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6370   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6371   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6372   for (p = pStart; p < pEnd; ++p) {
6373     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6374     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6375     else                       numbers[p-pStart] += shift;
6376   }
6377   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6378   if (globalSize) {
6379     PetscLayout layout;
6380     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6381     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6382     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6383   }
6384   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6385   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6386   PetscFunctionReturn(0);
6387 }
6388 
6389 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6390 {
6391   PetscInt       cellHeight, cStart, cEnd, cMax;
6392   PetscErrorCode ierr;
6393 
6394   PetscFunctionBegin;
6395   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6396   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6397   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6398   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6399   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6400   PetscFunctionReturn(0);
6401 }
6402 
6403 /*@
6404   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6405 
6406   Input Parameter:
6407 . dm   - The DMPlex object
6408 
6409   Output Parameter:
6410 . globalCellNumbers - Global cell numbers for all cells on this process
6411 
6412   Level: developer
6413 
6414 .seealso DMPlexGetVertexNumbering()
6415 @*/
6416 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6417 {
6418   DM_Plex       *mesh = (DM_Plex*) dm->data;
6419   PetscErrorCode ierr;
6420 
6421   PetscFunctionBegin;
6422   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6423   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6424   *globalCellNumbers = mesh->globalCellNumbers;
6425   PetscFunctionReturn(0);
6426 }
6427 
6428 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6429 {
6430   PetscInt       vStart, vEnd, vMax;
6431   PetscErrorCode ierr;
6432 
6433   PetscFunctionBegin;
6434   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6435   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6436   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6437   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6438   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6439   PetscFunctionReturn(0);
6440 }
6441 
6442 /*@
6443   DMPlexGetVertexNumbering - Get a global certex numbering for all vertices on this process
6444 
6445   Input Parameter:
6446 . dm   - The DMPlex object
6447 
6448   Output Parameter:
6449 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6450 
6451   Level: developer
6452 
6453 .seealso DMPlexGetCellNumbering()
6454 @*/
6455 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6456 {
6457   DM_Plex       *mesh = (DM_Plex*) dm->data;
6458   PetscErrorCode ierr;
6459 
6460   PetscFunctionBegin;
6461   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6462   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6463   *globalVertexNumbers = mesh->globalVertexNumbers;
6464   PetscFunctionReturn(0);
6465 }
6466 
6467 /*@
6468   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6469 
6470   Input Parameter:
6471 . dm   - The DMPlex object
6472 
6473   Output Parameter:
6474 . globalPointNumbers - Global numbers for all points on this process
6475 
6476   Level: developer
6477 
6478 .seealso DMPlexGetCellNumbering()
6479 @*/
6480 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6481 {
6482   IS             nums[4];
6483   PetscInt       depths[4], gdepths[4], starts[4];
6484   PetscInt       depth, d, shift = 0;
6485   PetscErrorCode ierr;
6486 
6487   PetscFunctionBegin;
6488   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6489   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6490   /* For unstratified meshes use dim instead of depth */
6491   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6492   for (d = 0; d <= depth; ++d) {
6493     PetscInt end;
6494 
6495     depths[d] = depth-d;
6496     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6497     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6498   }
6499   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6500   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6501   for (d = 0; d <= depth; ++d) {
6502     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6503   }
6504   for (d = 0; d <= depth; ++d) {
6505     PetscInt pStart, pEnd, gsize;
6506 
6507     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6508     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6509     shift += gsize;
6510   }
6511   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6512   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6513   PetscFunctionReturn(0);
6514 }
6515 
6516 
6517 /*@
6518   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6519 
6520   Input Parameter:
6521 . dm - The DMPlex object
6522 
6523   Output Parameter:
6524 . ranks - The rank field
6525 
6526   Options Database Keys:
6527 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6528 
6529   Level: intermediate
6530 
6531 .seealso: DMView()
6532 @*/
6533 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6534 {
6535   DM             rdm;
6536   PetscFE        fe;
6537   PetscScalar   *r;
6538   PetscMPIInt    rank;
6539   PetscInt       dim, cStart, cEnd, c;
6540   PetscErrorCode ierr;
6541 
6542   PetscFunctionBeginUser;
6543   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6544   PetscValidPointer(ranks, 2);
6545   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6546   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6547   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6548   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6549   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6550   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6551   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6552   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6553   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6554   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6555   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6556   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6557   for (c = cStart; c < cEnd; ++c) {
6558     PetscScalar *lr;
6559 
6560     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6561     *lr = rank;
6562   }
6563   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6564   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6565   PetscFunctionReturn(0);
6566 }
6567 
6568 /*@
6569   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6570 
6571   Input Parameters:
6572 + dm    - The DMPlex
6573 - label - The DMLabel
6574 
6575   Output Parameter:
6576 . val - The label value field
6577 
6578   Options Database Keys:
6579 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6580 
6581   Level: intermediate
6582 
6583 .seealso: DMView()
6584 @*/
6585 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6586 {
6587   DM             rdm;
6588   PetscFE        fe;
6589   PetscScalar   *v;
6590   PetscInt       dim, cStart, cEnd, c;
6591   PetscErrorCode ierr;
6592 
6593   PetscFunctionBeginUser;
6594   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6595   PetscValidPointer(label, 2);
6596   PetscValidPointer(val, 3);
6597   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6598   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6599   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6600   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6601   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6602   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6603   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6604   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6605   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6606   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6607   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6608   for (c = cStart; c < cEnd; ++c) {
6609     PetscScalar *lv;
6610     PetscInt     cval;
6611 
6612     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6613     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6614     *lv = cval;
6615   }
6616   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6617   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6618   PetscFunctionReturn(0);
6619 }
6620 
6621 /*@
6622   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6623 
6624   Input Parameter:
6625 . dm - The DMPlex object
6626 
6627   Note: This is a useful diagnostic when creating meshes programmatically.
6628 
6629   Level: developer
6630 
6631 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6632 @*/
6633 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6634 {
6635   PetscSection    coneSection, supportSection;
6636   const PetscInt *cone, *support;
6637   PetscInt        coneSize, c, supportSize, s;
6638   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6639   PetscBool       storagecheck = PETSC_TRUE;
6640   PetscErrorCode  ierr;
6641 
6642   PetscFunctionBegin;
6643   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6644   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6645   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6646   /* Check that point p is found in the support of its cone points, and vice versa */
6647   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6648   for (p = pStart; p < pEnd; ++p) {
6649     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6650     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6651     for (c = 0; c < coneSize; ++c) {
6652       PetscBool dup = PETSC_FALSE;
6653       PetscInt  d;
6654       for (d = c-1; d >= 0; --d) {
6655         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6656       }
6657       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6658       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6659       for (s = 0; s < supportSize; ++s) {
6660         if (support[s] == p) break;
6661       }
6662       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6663         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6664         for (s = 0; s < coneSize; ++s) {
6665           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6666         }
6667         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6668         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6669         for (s = 0; s < supportSize; ++s) {
6670           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6671         }
6672         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6673         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6674         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6675       }
6676     }
6677     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6678     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6679     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6680     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6681     for (s = 0; s < supportSize; ++s) {
6682       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6683       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6684       for (c = 0; c < coneSize; ++c) {
6685         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6686         if (cone[c] != pp) { c = 0; break; }
6687         if (cone[c] == p) break;
6688       }
6689       if (c >= coneSize) {
6690         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6691         for (c = 0; c < supportSize; ++c) {
6692           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6693         }
6694         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6695         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6696         for (c = 0; c < coneSize; ++c) {
6697           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6698         }
6699         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6700         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6701       }
6702     }
6703   }
6704   if (storagecheck) {
6705     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6706     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6707     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6708   }
6709   PetscFunctionReturn(0);
6710 }
6711 
6712 /*@
6713   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6714 
6715   Input Parameters:
6716 + dm - The DMPlex object
6717 - cellHeight - Normally 0
6718 
6719   Note: This is a useful diagnostic when creating meshes programmatically.
6720   Currently applicable only to homogeneous simplex or tensor meshes.
6721 
6722   Level: developer
6723 
6724 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6725 @*/
6726 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6727 {
6728   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6729   PetscBool      isSimplex = PETSC_FALSE;
6730   PetscErrorCode ierr;
6731 
6732   PetscFunctionBegin;
6733   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6734   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6735   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6736   if (cStart < cEnd) {
6737     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6738     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6739   }
6740   switch (dim) {
6741   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6742   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6743   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6744   default:
6745     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6746   }
6747   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6748   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6749   cMax = cMax >= 0 ? cMax : cEnd;
6750   for (c = cStart; c < cMax; ++c) {
6751     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6752 
6753     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6754     for (cl = 0; cl < closureSize*2; cl += 2) {
6755       const PetscInt p = closure[cl];
6756       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6757     }
6758     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6759     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6760   }
6761   for (c = cMax; c < cEnd; ++c) {
6762     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6763 
6764     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6765     for (cl = 0; cl < closureSize*2; cl += 2) {
6766       const PetscInt p = closure[cl];
6767       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6768     }
6769     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6770     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6771   }
6772   PetscFunctionReturn(0);
6773 }
6774 
6775 /*@
6776   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6777 
6778   Input Parameters:
6779 + dm - The DMPlex object
6780 - cellHeight - Normally 0
6781 
6782   Note: This is a useful diagnostic when creating meshes programmatically.
6783 
6784   Level: developer
6785 
6786 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6787 @*/
6788 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6789 {
6790   PetscInt       pMax[4];
6791   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6792   PetscErrorCode ierr;
6793 
6794   PetscFunctionBegin;
6795   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6796   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6797   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6798   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6799   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6800   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6801     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6802     for (c = cStart; c < cEnd; ++c) {
6803       const PetscInt *cone, *ornt, *faces;
6804       PetscInt        numFaces, faceSize, coneSize,f;
6805       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6806 
6807       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6808       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6809       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6810       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6811       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6812       for (cl = 0; cl < closureSize*2; cl += 2) {
6813         const PetscInt p = closure[cl];
6814         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6815       }
6816       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6817       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6818       for (f = 0; f < numFaces; ++f) {
6819         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6820 
6821         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6822         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6823           const PetscInt p = fclosure[cl];
6824           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6825         }
6826         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6827         for (v = 0; v < fnumCorners; ++v) {
6828           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6829         }
6830         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6831       }
6832       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6833       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6834     }
6835   }
6836   PetscFunctionReturn(0);
6837 }
6838 
6839 /*@
6840   DMPlexCheckGeometry - Check the geometry of mesh cells
6841 
6842   Input Parameter:
6843 . dm - The DMPlex object
6844 
6845   Note: This is a useful diagnostic when creating meshes programmatically.
6846 
6847   Level: developer
6848 
6849 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
6850 @*/
6851 PetscErrorCode DMPlexCheckGeometry(DM dm)
6852 {
6853   PetscReal      detJ, J[9], refVol = 1.0;
6854   PetscReal      vol;
6855   PetscInt       dim, depth, d, cStart, cEnd, c;
6856   PetscErrorCode ierr;
6857 
6858   PetscFunctionBegin;
6859   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6860   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6861   for (d = 0; d < dim; ++d) refVol *= 2.0;
6862   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6863   for (c = cStart; c < cEnd; ++c) {
6864     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
6865     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
6866     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
6867     if (depth > 1) {
6868       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
6869       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
6870       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
6871     }
6872   }
6873   PetscFunctionReturn(0);
6874 }
6875 
6876 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
6877 {
6878   PetscInt i,l,n;
6879   const PetscInt *cone;
6880   PetscErrorCode ierr;
6881 
6882   PetscFunctionBegin;
6883   *missingPoint = -1;
6884   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
6885   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6886   for (i=0; i<n; i++) {
6887     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
6888     if (l < 0) {
6889       *missingPoint = cone[i];
6890       break;
6891     }
6892   }
6893   PetscFunctionReturn(0);
6894 }
6895 
6896 /*@
6897   DMPlexCheckPointSF - Check that several sufficient conditions are met for the point SF of this plex.
6898 
6899   Input Parameters:
6900 . dm - The DMPlex object
6901 
6902   Note: This is mainly intended for debugging/testing purposes.
6903 
6904   Level: developer
6905 
6906 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6907 @*/
6908 PetscErrorCode DMPlexCheckPointSF(DM dm)
6909 {
6910   PetscSF sf;
6911   PetscInt d,depth,i,nleaves,p,plo,phi,missingPoint;
6912   const PetscInt *locals;
6913   PetscErrorCode ierr;
6914 
6915   PetscFunctionBegin;
6916   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6917   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6918   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
6919   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
6920 
6921   /* 1) check there are no faces in 2D, cells in 3D, in interface */
6922   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
6923   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
6924   for (i=0; i<nleaves; i++) {
6925     p = locals[i];
6926     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
6927   }
6928 
6929   /* 2) if some point is in interface, then all its cone points must be also in interface  */
6930   for (i=0; i<nleaves; i++) {
6931     p = locals[i];
6932     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
6933     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
6934   }
6935   PetscFunctionReturn(0);
6936 }
6937 
6938 typedef struct cell_stats
6939 {
6940   PetscReal min, max, sum, squaresum;
6941   PetscInt  count;
6942 } cell_stats_t;
6943 
6944 static void cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
6945 {
6946   PetscInt i, N = *len;
6947 
6948   for (i = 0; i < N; i++) {
6949     cell_stats_t *A = (cell_stats_t *) a;
6950     cell_stats_t *B = (cell_stats_t *) b;
6951 
6952     B->min = PetscMin(A->min,B->min);
6953     B->max = PetscMax(A->max,B->max);
6954     B->sum += A->sum;
6955     B->squaresum += A->squaresum;
6956     B->count += A->count;
6957   }
6958 }
6959 
6960 /*@
6961   DMPlexCheckCellShape - Checks the Jacobian of the mapping and computes some minimal statistics.
6962 
6963   Input Parameters:
6964 + dm - The DMPlex object
6965 - output - If true, statistics will be displayed on stdout
6966 
6967   Note: This is mainly intended for debugging/testing purposes.
6968 
6969   Level: developer
6970 
6971 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6972 @*/
6973 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output)
6974 {
6975   PetscMPIInt    rank,size;
6976   PetscInt       dim, c, cStart, cEnd, cMax, count = 0;
6977   cell_stats_t   stats, globalStats;
6978   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
6979   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
6980   DM             dmCoarse;
6981   PetscErrorCode ierr;
6982 
6983   PetscFunctionBegin;
6984   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6985   stats.min   = PETSC_MAX_REAL;
6986   stats.max   = PETSC_MIN_REAL;
6987   stats.sum   = stats.squaresum = 0.;
6988   stats.count = 0;
6989 
6990   ierr = DMGetCoordinateDim(dm,&dim);CHKERRQ(ierr);
6991   ierr = PetscMalloc2(dim * dim, &J, dim * dim, &invJ);CHKERRQ(ierr);
6992   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
6993   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
6994   cMax = cMax < 0 ? cEnd : cMax;
6995   for (c = cStart; c < cMax; c++) {
6996     PetscInt  i;
6997     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
6998 
6999     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7000     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7001     for (i = 0; i < dim * dim; i++) {
7002       frobJ    += J[i] * J[i];
7003       frobInvJ += invJ[i] * invJ[i];
7004     }
7005     cond2 = frobJ * frobInvJ;
7006     cond  = PetscSqrtReal(cond2);
7007 
7008     stats.min        = PetscMin(stats.min,cond);
7009     stats.max        = PetscMax(stats.max,cond);
7010     stats.sum       += cond;
7011     stats.squaresum += cond2;
7012     stats.count++;
7013   }
7014 
7015   ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
7016   if (size > 1) {
7017     PetscMPIInt   blockLengths[2] = {4,1};
7018     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7019     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7020     MPI_Op        statReduce;
7021 
7022     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7023     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7024     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7025     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7026     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7027     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7028   } else {
7029     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7030   }
7031 
7032   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
7033   if (!rank) {
7034     count = globalStats.count;
7035     min   = globalStats.min;
7036     max   = globalStats.max;
7037     mean  = globalStats.sum / globalStats.count;
7038     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7039   }
7040 
7041   if (output) {
7042     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7043   }
7044   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7045 
7046   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7047   if (dmCoarse) {
7048     PetscBool isplex;
7049 
7050     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7051     if (isplex) {
7052       ierr = DMPlexCheckCellShape(dmCoarse,output);CHKERRQ(ierr);
7053     }
7054   }
7055   PetscFunctionReturn(0);
7056 }
7057 
7058 /* Pointwise interpolation
7059      Just code FEM for now
7060      u^f = I u^c
7061      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7062      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7063      I_{ij} = psi^f_i phi^c_j
7064 */
7065 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7066 {
7067   PetscSection   gsc, gsf;
7068   PetscInt       m, n;
7069   void          *ctx;
7070   DM             cdm;
7071   PetscBool      regular, ismatis;
7072   PetscErrorCode ierr;
7073 
7074   PetscFunctionBegin;
7075   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7076   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7077   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7078   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7079 
7080   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7081   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7082   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7083   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7084   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7085 
7086   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7087   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7088   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7089   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7090   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7091   if (scaling) {
7092     /* Use naive scaling */
7093     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7094   }
7095   PetscFunctionReturn(0);
7096 }
7097 
7098 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7099 {
7100   PetscErrorCode ierr;
7101   VecScatter     ctx;
7102 
7103   PetscFunctionBegin;
7104   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7105   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7106   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7107   PetscFunctionReturn(0);
7108 }
7109 
7110 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7111 {
7112   PetscSection   gsc, gsf;
7113   PetscInt       m, n;
7114   void          *ctx;
7115   DM             cdm;
7116   PetscBool      regular;
7117   PetscErrorCode ierr;
7118 
7119   PetscFunctionBegin;
7120   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7121   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7122   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7123   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7124 
7125   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7126   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7127   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7128   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7129 
7130   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7131   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7132   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7133   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7134   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7135   PetscFunctionReturn(0);
7136 }
7137 
7138 /*@
7139   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7140 
7141   Input Parameter:
7142 . dm - The DMPlex object
7143 
7144   Output Parameter:
7145 . regular - The flag
7146 
7147   Level: intermediate
7148 
7149 .seealso: DMPlexSetRegularRefinement()
7150 @*/
7151 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7152 {
7153   PetscFunctionBegin;
7154   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7155   PetscValidPointer(regular, 2);
7156   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7157   PetscFunctionReturn(0);
7158 }
7159 
7160 /*@
7161   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7162 
7163   Input Parameters:
7164 + dm - The DMPlex object
7165 - regular - The flag
7166 
7167   Level: intermediate
7168 
7169 .seealso: DMPlexGetRegularRefinement()
7170 @*/
7171 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7172 {
7173   PetscFunctionBegin;
7174   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7175   ((DM_Plex *) dm->data)->regularRefinement = regular;
7176   PetscFunctionReturn(0);
7177 }
7178 
7179 /* anchors */
7180 /*@
7181   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7182   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7183 
7184   not collective
7185 
7186   Input Parameters:
7187 . dm - The DMPlex object
7188 
7189   Output Parameters:
7190 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7191 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7192 
7193 
7194   Level: intermediate
7195 
7196 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7197 @*/
7198 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7199 {
7200   DM_Plex *plex = (DM_Plex *)dm->data;
7201   PetscErrorCode ierr;
7202 
7203   PetscFunctionBegin;
7204   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7205   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7206   if (anchorSection) *anchorSection = plex->anchorSection;
7207   if (anchorIS) *anchorIS = plex->anchorIS;
7208   PetscFunctionReturn(0);
7209 }
7210 
7211 /*@
7212   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7213   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7214   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7215 
7216   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7217   DMGetConstraints() and filling in the entries in the constraint matrix.
7218 
7219   collective on dm
7220 
7221   Input Parameters:
7222 + dm - The DMPlex object
7223 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7224 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7225 
7226   The reference counts of anchorSection and anchorIS are incremented.
7227 
7228   Level: intermediate
7229 
7230 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7231 @*/
7232 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7233 {
7234   DM_Plex        *plex = (DM_Plex *)dm->data;
7235   PetscMPIInt    result;
7236   PetscErrorCode ierr;
7237 
7238   PetscFunctionBegin;
7239   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7240   if (anchorSection) {
7241     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7242     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7243     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7244   }
7245   if (anchorIS) {
7246     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7247     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7248     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7249   }
7250 
7251   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7252   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7253   plex->anchorSection = anchorSection;
7254 
7255   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7256   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7257   plex->anchorIS = anchorIS;
7258 
7259 #if defined(PETSC_USE_DEBUG)
7260   if (anchorIS && anchorSection) {
7261     PetscInt size, a, pStart, pEnd;
7262     const PetscInt *anchors;
7263 
7264     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7265     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7266     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7267     for (a = 0; a < size; a++) {
7268       PetscInt p;
7269 
7270       p = anchors[a];
7271       if (p >= pStart && p < pEnd) {
7272         PetscInt dof;
7273 
7274         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7275         if (dof) {
7276           PetscErrorCode ierr2;
7277 
7278           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7279           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7280         }
7281       }
7282     }
7283     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7284   }
7285 #endif
7286   /* reset the generic constraints */
7287   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7288   PetscFunctionReturn(0);
7289 }
7290 
7291 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7292 {
7293   PetscSection anchorSection;
7294   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7295   PetscErrorCode ierr;
7296 
7297   PetscFunctionBegin;
7298   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7299   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7300   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7301   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7302   if (numFields) {
7303     PetscInt f;
7304     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7305 
7306     for (f = 0; f < numFields; f++) {
7307       PetscInt numComp;
7308 
7309       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7310       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7311     }
7312   }
7313   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7314   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7315   pStart = PetscMax(pStart,sStart);
7316   pEnd   = PetscMin(pEnd,sEnd);
7317   pEnd   = PetscMax(pStart,pEnd);
7318   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7319   for (p = pStart; p < pEnd; p++) {
7320     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7321     if (dof) {
7322       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7323       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7324       for (f = 0; f < numFields; f++) {
7325         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7326         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7327       }
7328     }
7329   }
7330   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7331   PetscFunctionReturn(0);
7332 }
7333 
7334 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7335 {
7336   PetscSection aSec;
7337   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7338   const PetscInt *anchors;
7339   PetscInt numFields, f;
7340   IS aIS;
7341   PetscErrorCode ierr;
7342 
7343   PetscFunctionBegin;
7344   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7345   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7346   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7347   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7348   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7349   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7350   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7351   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7352   /* cSec will be a subset of aSec and section */
7353   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7354   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7355   i[0] = 0;
7356   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7357   for (p = pStart; p < pEnd; p++) {
7358     PetscInt rDof, rOff, r;
7359 
7360     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7361     if (!rDof) continue;
7362     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7363     if (numFields) {
7364       for (f = 0; f < numFields; f++) {
7365         annz = 0;
7366         for (r = 0; r < rDof; r++) {
7367           a = anchors[rOff + r];
7368           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7369           annz += aDof;
7370         }
7371         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7372         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7373         for (q = 0; q < dof; q++) {
7374           i[off + q + 1] = i[off + q] + annz;
7375         }
7376       }
7377     }
7378     else {
7379       annz = 0;
7380       for (q = 0; q < dof; q++) {
7381         a = anchors[off + q];
7382         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7383         annz += aDof;
7384       }
7385       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7386       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7387       for (q = 0; q < dof; q++) {
7388         i[off + q + 1] = i[off + q] + annz;
7389       }
7390     }
7391   }
7392   nnz = i[m];
7393   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7394   offset = 0;
7395   for (p = pStart; p < pEnd; p++) {
7396     if (numFields) {
7397       for (f = 0; f < numFields; f++) {
7398         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7399         for (q = 0; q < dof; q++) {
7400           PetscInt rDof, rOff, r;
7401           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7402           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7403           for (r = 0; r < rDof; r++) {
7404             PetscInt s;
7405 
7406             a = anchors[rOff + r];
7407             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7408             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7409             for (s = 0; s < aDof; s++) {
7410               j[offset++] = aOff + s;
7411             }
7412           }
7413         }
7414       }
7415     }
7416     else {
7417       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7418       for (q = 0; q < dof; q++) {
7419         PetscInt rDof, rOff, r;
7420         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7421         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7422         for (r = 0; r < rDof; r++) {
7423           PetscInt s;
7424 
7425           a = anchors[rOff + r];
7426           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7427           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7428           for (s = 0; s < aDof; s++) {
7429             j[offset++] = aOff + s;
7430           }
7431         }
7432       }
7433     }
7434   }
7435   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7436   ierr = PetscFree(i);CHKERRQ(ierr);
7437   ierr = PetscFree(j);CHKERRQ(ierr);
7438   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7439   PetscFunctionReturn(0);
7440 }
7441 
7442 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7443 {
7444   DM_Plex        *plex = (DM_Plex *)dm->data;
7445   PetscSection   anchorSection, section, cSec;
7446   Mat            cMat;
7447   PetscErrorCode ierr;
7448 
7449   PetscFunctionBegin;
7450   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7451   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7452   if (anchorSection) {
7453     PetscInt Nf;
7454 
7455     ierr = DMGetSection(dm,&section);CHKERRQ(ierr);
7456     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7457     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7458     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7459     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7460     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7461     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7462     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7463   }
7464   PetscFunctionReturn(0);
7465 }
7466 
7467 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7468 {
7469   IS             subis;
7470   PetscSection   section, subsection;
7471   PetscErrorCode ierr;
7472 
7473   PetscFunctionBegin;
7474   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
7475   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7476   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7477   /* Create subdomain */
7478   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7479   /* Create submodel */
7480   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7481   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7482   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7483   ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
7484   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7485   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7486   /* Create map from submodel to global model */
7487   if (is) {
7488     PetscSection    sectionGlobal, subsectionGlobal;
7489     IS              spIS;
7490     const PetscInt *spmap;
7491     PetscInt       *subIndices;
7492     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7493     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7494 
7495     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7496     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7497     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7498     ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7499     ierr = DMGetDefaultGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7500     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7501     for (p = pStart; p < pEnd; ++p) {
7502       PetscInt gdof, pSubSize  = 0;
7503 
7504       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7505       if (gdof > 0) {
7506         for (f = 0; f < Nf; ++f) {
7507           PetscInt fdof, fcdof;
7508 
7509           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7510           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7511           pSubSize += fdof-fcdof;
7512         }
7513         subSize += pSubSize;
7514         if (pSubSize) {
7515           if (bs < 0) {
7516             bs = pSubSize;
7517           } else if (bs != pSubSize) {
7518             /* Layout does not admit a pointwise block size */
7519             bs = 1;
7520           }
7521         }
7522       }
7523     }
7524     /* Must have same blocksize on all procs (some might have no points) */
7525     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7526     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7527     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7528     else                            {bs = bsMinMax[0];}
7529     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7530     for (p = pStart; p < pEnd; ++p) {
7531       PetscInt gdof, goff;
7532 
7533       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7534       if (gdof > 0) {
7535         const PetscInt point = spmap[p];
7536 
7537         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7538         for (f = 0; f < Nf; ++f) {
7539           PetscInt fdof, fcdof, fc, f2, poff = 0;
7540 
7541           /* Can get rid of this loop by storing field information in the global section */
7542           for (f2 = 0; f2 < f; ++f2) {
7543             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7544             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7545             poff += fdof-fcdof;
7546           }
7547           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7548           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7549           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7550             subIndices[subOff] = goff+poff+fc;
7551           }
7552         }
7553       }
7554     }
7555     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7556     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7557     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7558     if (bs > 1) {
7559       /* We need to check that the block size does not come from non-contiguous fields */
7560       PetscInt i, j, set = 1;
7561       for (i = 0; i < subSize; i += bs) {
7562         for (j = 0; j < bs; ++j) {
7563           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7564         }
7565       }
7566       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7567     }
7568     /* Attach nullspace */
7569     for (f = 0; f < Nf; ++f) {
7570       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7571       if ((*subdm)->nullspaceConstructors[f]) break;
7572     }
7573     if (f < Nf) {
7574       MatNullSpace nullSpace;
7575 
7576       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7577       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7578       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7579     }
7580   }
7581   PetscFunctionReturn(0);
7582 }
7583