xref: /petsc/src/dm/impls/plex/plex.c (revision 0d3b6ff2d777fc53ae5f1aa453274beef4146c2b) 
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cMax;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_INVALID;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cMax < 0 ? cEnd : cMax;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else {
135     if (field >= 0) {
136       const char *fieldname;
137 
138       ierr = PetscSectionGetFieldName(section, field, &fieldname);CHKERRQ(ierr);
139       ierr = PetscInfo2((PetscObject) dm, "Could not classify VTK output type of section field %D \"%s\"\n", field, fieldname);CHKERRQ(ierr);
140     } else {
141       ierr = PetscInfo((PetscObject) dm, "Could not classify VTK output typp of section\"%s\"\n");CHKERRQ(ierr);
142     }
143   }
144   PetscFunctionReturn(0);
145 }
146 
147 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
148 {
149   DM                 dm;
150   PetscSection       s;
151   PetscDraw          draw, popup;
152   DM                 cdm;
153   PetscSection       coordSection;
154   Vec                coordinates;
155   const PetscScalar *coords, *array;
156   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
157   PetscReal          vbound[2], time;
158   PetscBool          isnull, flg;
159   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
160   const char        *name;
161   char               title[PETSC_MAX_PATH_LEN];
162   PetscErrorCode     ierr;
163 
164   PetscFunctionBegin;
165   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
166   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
167   if (isnull) PetscFunctionReturn(0);
168 
169   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
170   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
171   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
172   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
173   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
174   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
175   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
176   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
177   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
178   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
179   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
180 
181   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
182   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
183 
184   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
185   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
186   for (c = 0; c < N; c += dim) {
187     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
188     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
189   }
190   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
191   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
192 
193   /* Could implement something like DMDASelectFields() */
194   for (f = 0; f < Nf; ++f) {
195     DM   fdm = dm;
196     Vec  fv  = v;
197     IS   fis;
198     char prefix[PETSC_MAX_PATH_LEN];
199     const char *fname;
200 
201     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
202     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
203 
204     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
205     else               {prefix[0] = '\0';}
206     if (Nf > 1) {
207       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
208       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
209       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
210       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
211     }
212     for (comp = 0; comp < Nc; ++comp, ++w) {
213       PetscInt nmax = 2;
214 
215       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
216       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
217       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
218       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
219 
220       /* TODO Get max and min only for this component */
221       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
222       if (!flg) {
223         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
224         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
225         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
226       }
227       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
228       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
229       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
230 
231       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
232       for (c = cStart; c < cEnd; ++c) {
233         PetscScalar *coords = NULL, *a = NULL;
234         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
235 
236         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
237         if (a) {
238           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
239           color[1] = color[2] = color[3] = color[0];
240         } else {
241           PetscScalar *vals = NULL;
242           PetscInt     numVals, va;
243 
244           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
245           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
246           switch (numVals/Nc) {
247           case 3: /* P1 Triangle */
248           case 4: /* P1 Quadrangle */
249             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
250             break;
251           case 6: /* P2 Triangle */
252           case 8: /* P2 Quadrangle */
253             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
254             break;
255           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
256           }
257           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
258         }
259         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
260         switch (numCoords) {
261         case 6:
262           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
263           break;
264         case 8:
265           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
266           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
267           break;
268         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
269         }
270         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
271       }
272       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
273       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
274       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
275       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
276     }
277     if (Nf > 1) {
278       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
279       ierr = ISDestroy(&fis);CHKERRQ(ierr);
280       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
281     }
282   }
283   PetscFunctionReturn(0);
284 }
285 
286 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
287 {
288   DM                      dm;
289   Vec                     locv;
290   const char              *name;
291   PetscSection            section;
292   PetscInt                pStart, pEnd;
293   PetscInt                numFields;
294   PetscViewerVTKFieldType ft;
295   PetscErrorCode          ierr;
296 
297   PetscFunctionBegin;
298   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
299   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
300   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
301   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
302   ierr = VecCopy(v, locv);CHKERRQ(ierr);
303   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
304   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
305   if (!numFields) {
306     ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
307     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
308   } else {
309     PetscInt f;
310 
311     for (f = 0; f < numFields; f++) {
312       ierr = DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);CHKERRQ(ierr);
313       if (ft == PETSC_VTK_INVALID) continue;
314       ierr = PetscObjectReference((PetscObject)locv);CHKERRQ(ierr);
315       ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
316     }
317     ierr = VecDestroy(&locv);CHKERRQ(ierr);
318   }
319   PetscFunctionReturn(0);
320 }
321 
322 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
323 {
324   DM             dm;
325   PetscBool      isvtk, ishdf5, isdraw, isglvis;
326   PetscErrorCode ierr;
327 
328   PetscFunctionBegin;
329   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
330   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
331   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
332   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
333   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
334   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
335   if (isvtk || ishdf5 || isdraw || isglvis) {
336     PetscInt    i,numFields;
337     PetscObject fe;
338     PetscBool   fem = PETSC_FALSE;
339     Vec         locv = v;
340     const char  *name;
341     PetscInt    step;
342     PetscReal   time;
343 
344     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
345     for (i=0; i<numFields; i++) {
346       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
347       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
348     }
349     if (fem) {
350       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
351       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
352       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
353       ierr = VecCopy(v, locv);CHKERRQ(ierr);
354       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
355       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
356     }
357     if (isvtk) {
358       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
359     } else if (ishdf5) {
360 #if defined(PETSC_HAVE_HDF5)
361       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
362 #else
363       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
364 #endif
365     } else if (isdraw) {
366       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
367     } else if (isglvis) {
368       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
369       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
370       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
371     }
372     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
373   } else {
374     PetscBool isseq;
375 
376     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
377     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
378     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
379   }
380   PetscFunctionReturn(0);
381 }
382 
383 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
384 {
385   DM             dm;
386   PetscBool      isvtk, ishdf5, isdraw, isglvis;
387   PetscErrorCode ierr;
388 
389   PetscFunctionBegin;
390   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
391   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
392   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
393   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
394   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
395   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
396   if (isvtk || isdraw || isglvis) {
397     Vec         locv;
398     const char *name;
399 
400     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
401     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
402     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
403     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
404     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
405     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
406     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
407   } else if (ishdf5) {
408 #if defined(PETSC_HAVE_HDF5)
409     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
410 #else
411     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
412 #endif
413   } else {
414     PetscBool isseq;
415 
416     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
417     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
418     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
419   }
420   PetscFunctionReturn(0);
421 }
422 
423 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
424 {
425   DM                dm;
426   MPI_Comm          comm;
427   PetscViewerFormat format;
428   Vec               v;
429   PetscBool         isvtk, ishdf5;
430   PetscErrorCode    ierr;
431 
432   PetscFunctionBegin;
433   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
434   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
435   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
436   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
437   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
438   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
439   if (format == PETSC_VIEWER_NATIVE) {
440     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
441     /* this need a better fix */
442     if (dm->useNatural) {
443       if (dm->sfNatural) {
444         const char *vecname;
445         PetscInt    n, nroots;
446 
447         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
448         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
449         if (n == nroots) {
450           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
451           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
452           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
453           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
454           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
455         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
456       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
457     } else v = originalv;
458   } else v = originalv;
459 
460   if (ishdf5) {
461 #if defined(PETSC_HAVE_HDF5)
462     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
463 #else
464     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
465 #endif
466   } else if (isvtk) {
467     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
468   } else {
469     PetscBool isseq;
470 
471     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
472     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
473     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
474   }
475   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
476   PetscFunctionReturn(0);
477 }
478 
479 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
480 {
481   DM             dm;
482   PetscBool      ishdf5;
483   PetscErrorCode ierr;
484 
485   PetscFunctionBegin;
486   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
487   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
488   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
489   if (ishdf5) {
490     DM          dmBC;
491     Vec         gv;
492     const char *name;
493 
494     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
495     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
496     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
497     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
498     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
499     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
500     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
501     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
502   } else {
503     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
504   }
505   PetscFunctionReturn(0);
506 }
507 
508 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
509 {
510   DM             dm;
511   PetscBool      ishdf5;
512   PetscErrorCode ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
518   if (ishdf5) {
519 #if defined(PETSC_HAVE_HDF5)
520     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
521 #else
522     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
523 #endif
524   } else {
525     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
526   }
527   PetscFunctionReturn(0);
528 }
529 
530 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
531 {
532   DM                dm;
533   PetscViewerFormat format;
534   PetscBool         ishdf5;
535   PetscErrorCode    ierr;
536 
537   PetscFunctionBegin;
538   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
539   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
540   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
541   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
542   if (format == PETSC_VIEWER_NATIVE) {
543     if (dm->useNatural) {
544       if (dm->sfNatural) {
545         if (ishdf5) {
546 #if defined(PETSC_HAVE_HDF5)
547           Vec         v;
548           const char *vecname;
549 
550           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
551           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
552           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
553           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
554           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
555           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
556           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
557 #else
558           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
559 #endif
560         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
561       }
562     } else {
563       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
564     }
565   }
566   PetscFunctionReturn(0);
567 }
568 
569 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
570 {
571   PetscSection       coordSection;
572   Vec                coordinates;
573   DMLabel            depthLabel;
574   const char        *name[4];
575   const PetscScalar *a;
576   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
577   PetscErrorCode     ierr;
578 
579   PetscFunctionBegin;
580   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
581   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
582   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
583   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
584   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
585   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
586   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
587   name[0]     = "vertex";
588   name[1]     = "edge";
589   name[dim-1] = "face";
590   name[dim]   = "cell";
591   for (c = cStart; c < cEnd; ++c) {
592     PetscInt *closure = NULL;
593     PetscInt  closureSize, cl;
594 
595     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
596     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
597     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
598     for (cl = 0; cl < closureSize*2; cl += 2) {
599       PetscInt point = closure[cl], depth, dof, off, d, p;
600 
601       if ((point < pStart) || (point >= pEnd)) continue;
602       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
603       if (!dof) continue;
604       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
605       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
606       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
607       for (p = 0; p < dof/dim; ++p) {
608         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
609         for (d = 0; d < dim; ++d) {
610           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
611           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
612         }
613         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
614       }
615       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
616     }
617     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
618     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
619   }
620   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
621   PetscFunctionReturn(0);
622 }
623 
624 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
625 {
626   DM_Plex          *mesh = (DM_Plex*) dm->data;
627   DM                cdm;
628   DMLabel           markers;
629   PetscSection      coordSection;
630   Vec               coordinates;
631   PetscViewerFormat format;
632   PetscErrorCode    ierr;
633 
634   PetscFunctionBegin;
635   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
636   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
637   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
638   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
639   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
640     const char *name;
641     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
642     PetscInt    pStart, pEnd, p;
643     PetscMPIInt rank, size;
644 
645     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
646     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
647     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
648     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
649     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
650     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
651     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
652     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
653     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
654     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
655     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
656     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
658     for (p = pStart; p < pEnd; ++p) {
659       PetscInt dof, off, s;
660 
661       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
662       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
663       for (s = off; s < off+dof; ++s) {
664         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
665       }
666     }
667     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
668     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
669     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
670     for (p = pStart; p < pEnd; ++p) {
671       PetscInt dof, off, c;
672 
673       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
674       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
675       for (c = off; c < off+dof; ++c) {
676         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
677       }
678     }
679     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
680     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
681     if (coordSection && coordinates) {
682       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
683     }
684     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
685     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
686     if (size > 1) {
687       PetscSF sf;
688 
689       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
690       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
691     }
692     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
693   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
694     const char  *name, *color;
695     const char  *defcolors[3]  = {"gray", "orange", "green"};
696     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
697     char         lname[PETSC_MAX_PATH_LEN];
698     PetscReal    scale         = 2.0;
699     PetscReal    tikzscale     = 1.0;
700     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
701     double       tcoords[3];
702     PetscScalar *coords;
703     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
704     PetscMPIInt  rank, size;
705     char         **names, **colors, **lcolors;
706     PetscBool    plotEdges, flg, lflg;
707     PetscBT      wp = NULL;
708     PetscInt     pEnd, pStart;
709 
710     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
711     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
712     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
713     numLabels  = PetscMax(numLabels, 10);
714     numColors  = 10;
715     numLColors = 10;
716     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
717     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
718     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
719     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
720     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
721     if (!useLabels) numLabels = 0;
722     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
723     if (!useColors) {
724       numColors = 3;
725       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
726     }
727     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
728     if (!useColors) {
729       numLColors = 4;
730       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
731     }
732     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
733     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
734     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
735     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
736     if (depth < dim) plotEdges = PETSC_FALSE;
737 
738     /* filter points with labelvalue != labeldefaultvalue */
739     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
740     if (lflg) {
741       DMLabel lbl;
742 
743       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
744       if (lbl) {
745         PetscInt val, defval;
746 
747         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
748         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
749         for (c = pStart;  c < pEnd; c++) {
750           PetscInt *closure = NULL;
751           PetscInt  closureSize;
752 
753           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
754           if (val == defval) continue;
755 
756           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
757           for (p = 0; p < closureSize*2; p += 2) {
758             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
759           }
760           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
761         }
762       }
763     }
764 
765     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
766     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
767     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
768     ierr = PetscViewerASCIIPrintf(viewer, "\
769 \\documentclass[tikz]{standalone}\n\n\
770 \\usepackage{pgflibraryshapes}\n\
771 \\usetikzlibrary{backgrounds}\n\
772 \\usetikzlibrary{arrows}\n\
773 \\begin{document}\n");CHKERRQ(ierr);
774     if (size > 1) {
775       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
776       for (p = 0; p < size; ++p) {
777         if (p > 0 && p == size-1) {
778           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
779         } else if (p > 0) {
780           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
781         }
782         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
783       }
784       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
785     }
786     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
787 
788     /* Plot vertices */
789     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
790     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
791     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
792     for (v = vStart; v < vEnd; ++v) {
793       PetscInt  off, dof, d;
794       PetscBool isLabeled = PETSC_FALSE;
795 
796       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
797       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
798       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
799       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
800       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
801       for (d = 0; d < dof; ++d) {
802         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
803         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
804       }
805       /* Rotate coordinates since PGF makes z point out of the page instead of up */
806       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
807       for (d = 0; d < dof; ++d) {
808         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
809         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
810       }
811       color = colors[rank%numColors];
812       for (l = 0; l < numLabels; ++l) {
813         PetscInt val;
814         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
815         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
816       }
817       if (useNumbers) {
818         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
819       } else {
820         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
821       }
822     }
823     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
824     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
825     /* Plot cells */
826     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
827     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
828     if (dim == 3 || !useNumbers) {
829       for (e = eStart; e < eEnd; ++e) {
830         const PetscInt *cone;
831 
832         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
833         color = colors[rank%numColors];
834         for (l = 0; l < numLabels; ++l) {
835           PetscInt val;
836           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
837           if (val >= 0) {color = lcolors[l%numLColors]; break;}
838         }
839         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
840         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
841       }
842     } else {
843       for (c = cStart; c < cEnd; ++c) {
844         PetscInt *closure = NULL;
845         PetscInt  closureSize, firstPoint = -1;
846 
847         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
848         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
849         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
850         for (p = 0; p < closureSize*2; p += 2) {
851           const PetscInt point = closure[p];
852 
853           if ((point < vStart) || (point >= vEnd)) continue;
854           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
855           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
856           if (firstPoint < 0) firstPoint = point;
857         }
858         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
859         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
860         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
861       }
862     }
863     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
864     for (c = cStart; c < cEnd; ++c) {
865       double    ccoords[3] = {0.0, 0.0, 0.0};
866       PetscBool isLabeled  = PETSC_FALSE;
867       PetscInt *closure    = NULL;
868       PetscInt  closureSize, dof, d, n = 0;
869 
870       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
871       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
872       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
873       for (p = 0; p < closureSize*2; p += 2) {
874         const PetscInt point = closure[p];
875         PetscInt       off;
876 
877         if ((point < vStart) || (point >= vEnd)) continue;
878         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
879         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
880         for (d = 0; d < dof; ++d) {
881           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
882           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
883         }
884         /* Rotate coordinates since PGF makes z point out of the page instead of up */
885         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
886         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
887         ++n;
888       }
889       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
890       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
891       for (d = 0; d < dof; ++d) {
892         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
893         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
894       }
895       color = colors[rank%numColors];
896       for (l = 0; l < numLabels; ++l) {
897         PetscInt val;
898         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
899         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
900       }
901       if (useNumbers) {
902         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
903       } else {
904         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
905       }
906     }
907     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
908     /* Plot edges */
909     if (plotEdges) {
910       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
911       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
912       for (e = eStart; e < eEnd; ++e) {
913         const PetscInt *cone;
914         PetscInt        coneSize, offA, offB, dof, d;
915 
916         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
917         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
918         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
919         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
920         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
921         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
922         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
923         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
924         for (d = 0; d < dof; ++d) {
925           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
926           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
927         }
928         /* Rotate coordinates since PGF makes z point out of the page instead of up */
929         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
930         for (d = 0; d < dof; ++d) {
931           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
932           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
933         }
934         color = colors[rank%numColors];
935         for (l = 0; l < numLabels; ++l) {
936           PetscInt val;
937           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
938           if (val >= 0) {color = lcolors[l%numLColors]; break;}
939         }
940         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
941       }
942       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
943       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
944       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
945     }
946     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
947     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
948     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
949     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
950     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
951     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
952     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
953     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
954     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
955   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
956     Vec                    cown,acown;
957     VecScatter             sct;
958     ISLocalToGlobalMapping g2l;
959     IS                     gid,acis;
960     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
961     MPI_Group              ggroup,ngroup;
962     PetscScalar            *array,nid;
963     const PetscInt         *idxs;
964     PetscInt               *idxs2,*start,*adjacency,*work;
965     PetscInt64             lm[3],gm[3];
966     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
967     PetscMPIInt            d1,d2,rank;
968 
969     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
970     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
971 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
972     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
973 #endif
974     if (ncomm != MPI_COMM_NULL) {
975       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
976       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
977       d1   = 0;
978       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
979       nid  = d2;
980       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
981       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
982       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
983     } else nid = 0.0;
984 
985     /* Get connectivity */
986     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
987     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
988 
989     /* filter overlapped local cells */
990     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
991     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
992     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
993     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
994     for (c = cStart, cum = 0; c < cEnd; c++) {
995       if (idxs[c-cStart] < 0) continue;
996       idxs2[cum++] = idxs[c-cStart];
997     }
998     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
999     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
1000     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1001     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
1002 
1003     /* support for node-aware cell locality */
1004     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
1005     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
1006     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
1007     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
1008     for (c = 0; c < numVertices; c++) array[c] = nid;
1009     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
1010     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
1011     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1012     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
1013     ierr = ISDestroy(&acis);CHKERRQ(ierr);
1014     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
1015     ierr = VecDestroy(&cown);CHKERRQ(ierr);
1016 
1017     /* compute edgeCut */
1018     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
1019     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
1020     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
1021     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
1022     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1023     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1024     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1025       PetscInt totl;
1026 
1027       totl = start[c+1]-start[c];
1028       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1029       for (i = 0; i < totl; i++) {
1030         if (work[i] < 0) {
1031           ect  += 1;
1032           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1033         }
1034       }
1035     }
1036     ierr  = PetscFree(work);CHKERRQ(ierr);
1037     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1038     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1039     lm[1] = -numVertices;
1040     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1041     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1042     lm[0] = ect; /* edgeCut */
1043     lm[1] = ectn; /* node-aware edgeCut */
1044     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1045     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1046     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1047 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1048     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1049 #else
1050     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1051 #endif
1052     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1053     ierr  = PetscFree(start);CHKERRQ(ierr);
1054     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1055     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1056   } else {
1057     MPI_Comm    comm;
1058     PetscInt   *sizes, *hybsizes, *ghostsizes;
1059     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
1060     PetscInt    pStart, pEnd, p, gcStart, gcEnd, gcNum;
1061     PetscInt    numLabels, l;
1062     const char *name;
1063     PetscMPIInt size;
1064 
1065     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1066     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1067     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1068     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1069     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1070     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1071     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1072     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1073     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1074     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1075     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1076     ierr = DMPlexGetGhostCellStratum(dm, &gcStart, &gcEnd);CHKERRQ(ierr);
1077     gcNum = gcEnd - gcStart;
1078     ierr = PetscCalloc3(size,&sizes,size,&hybsizes,size,&ghostsizes);CHKERRQ(ierr);
1079     if (depth == 1) {
1080       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1081       pEnd = pEnd - pStart;
1082       pMax[0] -= pStart;
1083       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1084       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1085       ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1086       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1087       for (p = 0; p < size; ++p) {
1088         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1089         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1090       }
1091       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1092       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1093       pEnd = pEnd - pStart;
1094       pMax[depth] -= pStart;
1095       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1096       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1097       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1098       for (p = 0; p < size; ++p) {
1099         ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);
1100         if (hybsizes[p] >= 0)   {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1101         if (ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1102       }
1103       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1104     } else {
1105       PetscMPIInt rank;
1106       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1107       for (d = 0; d <= dim; d++) {
1108         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1109         pEnd    -= pStart;
1110         pMax[d] -= pStart;
1111         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1112         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1113         if (d == dim) {ierr = MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);}
1114         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1115         for (p = 0; p < size; ++p) {
1116           if (!rank) {
1117             ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);
1118             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " (%D)", sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1119             if (d == dim && ghostsizes[p] > 0) {ierr = PetscViewerASCIIPrintf(viewer, " [%D]", ghostsizes[p]);CHKERRQ(ierr);}
1120           }
1121         }
1122         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1123       }
1124     }
1125     ierr = PetscFree3(sizes,hybsizes,ghostsizes);CHKERRQ(ierr);
1126     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1127     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1128     for (l = 0; l < numLabels; ++l) {
1129       DMLabel         label;
1130       const char     *name;
1131       IS              valueIS;
1132       const PetscInt *values;
1133       PetscInt        numValues, v;
1134 
1135       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1136       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1137       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1138       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1139       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1140       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1141       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1142       for (v = 0; v < numValues; ++v) {
1143         PetscInt size;
1144 
1145         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1146         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1147         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1148       }
1149       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1150       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1151       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1152       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1153     }
1154     /* If no fields are specified, people do not want to see adjacency */
1155     if (dm->Nf) {
1156       PetscInt f;
1157 
1158       for (f = 0; f < dm->Nf; ++f) {
1159         const char *name;
1160 
1161         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1162         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1163         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1164         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1165         if (dm->fields[f].adjacency[0]) {
1166           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1167           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1168         } else {
1169           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1170           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1171         }
1172         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1173       }
1174     }
1175     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1176     if (cdm) {
1177       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1178       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1179       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1180     }
1181   }
1182   PetscFunctionReturn(0);
1183 }
1184 
1185 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1186 {
1187   PetscDraw          draw;
1188   DM                 cdm;
1189   PetscSection       coordSection;
1190   Vec                coordinates;
1191   const PetscScalar *coords;
1192   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1193   PetscBool          isnull;
1194   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1195   PetscMPIInt        rank;
1196   PetscErrorCode     ierr;
1197 
1198   PetscFunctionBegin;
1199   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1200   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1201   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1202   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1203   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1204   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1205   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1206 
1207   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1208   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1209   if (isnull) PetscFunctionReturn(0);
1210   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1211 
1212   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1213   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1214   for (c = 0; c < N; c += dim) {
1215     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1216     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1217   }
1218   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1219   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1220   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1221   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1222   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1223 
1224   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1225   for (c = cStart; c < cEnd; ++c) {
1226     PetscScalar *coords = NULL;
1227     PetscInt     numCoords,coneSize;
1228 
1229     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1230     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1231     switch (coneSize) {
1232     case 3:
1233       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1234                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1235                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1236                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1237       break;
1238     case 4:
1239       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1240                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1241                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1242                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1243       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]),
1244                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1245                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1246                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1247       break;
1248     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1249     }
1250     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1251   }
1252   for (c = cStart; c < cEnd; ++c) {
1253     PetscScalar *coords = NULL;
1254     PetscInt     numCoords,coneSize;
1255 
1256     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1257     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1258     switch (coneSize) {
1259     case 3:
1260       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1261       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1262       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1263       break;
1264     case 4:
1265       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1266       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1267       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1268       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1269       break;
1270     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1271     }
1272     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1273   }
1274   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1275   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1276   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1277   PetscFunctionReturn(0);
1278 }
1279 
1280 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1281 {
1282   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1283   char           name[PETSC_MAX_PATH_LEN];
1284   PetscErrorCode ierr;
1285 
1286   PetscFunctionBegin;
1287   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1288   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1289   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1290   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1291   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1292   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1293   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1294   if (iascii) {
1295     PetscViewerFormat format;
1296     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1297     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1298       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1299     } else {
1300       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1301     }
1302   } else if (ishdf5) {
1303 #if defined(PETSC_HAVE_HDF5)
1304     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1305 #else
1306     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1307 #endif
1308   } else if (isvtk) {
1309     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1310   } else if (isdraw) {
1311     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1312   } else if (isglvis) {
1313     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1314   } else {
1315     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1316   }
1317   /* Optionally view the partition */
1318   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1319   if (flg) {
1320     Vec ranks;
1321     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1322     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1323     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1324   }
1325   /* Optionally view a label */
1326   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1327   if (flg) {
1328     DMLabel label;
1329     Vec     val;
1330 
1331     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1332     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1333     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1334     ierr = VecView(val, viewer);CHKERRQ(ierr);
1335     ierr = VecDestroy(&val);CHKERRQ(ierr);
1336   }
1337   PetscFunctionReturn(0);
1338 }
1339 
1340 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1341 {
1342   PetscBool      ishdf5;
1343   PetscErrorCode ierr;
1344 
1345   PetscFunctionBegin;
1346   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1347   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1348   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1349   if (ishdf5) {
1350 #if defined(PETSC_HAVE_HDF5)
1351     PetscViewerFormat format;
1352     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1353     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1354       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1355     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1356       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1357     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1358 #else
1359     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1360 #endif
1361   } else {
1362     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1363   }
1364   PetscFunctionReturn(0);
1365 }
1366 
1367 PetscErrorCode DMDestroy_Plex(DM dm)
1368 {
1369   DM_Plex       *mesh = (DM_Plex*) dm->data;
1370   PetscErrorCode ierr;
1371 
1372   PetscFunctionBegin;
1373   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1374   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1375   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
1376   if (--mesh->refct > 0) PetscFunctionReturn(0);
1377   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1378   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1379   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1380   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1381   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1382   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1383   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1384   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1385   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1386   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1387   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1388   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1389   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1390   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1391   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1392   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1393   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1394   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1395   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1396   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1397   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1398   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1399   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1400   ierr = PetscFree(mesh);CHKERRQ(ierr);
1401   PetscFunctionReturn(0);
1402 }
1403 
1404 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1405 {
1406   PetscSection           sectionGlobal;
1407   PetscInt               bs = -1, mbs;
1408   PetscInt               localSize;
1409   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1410   PetscErrorCode         ierr;
1411   MatType                mtype;
1412   ISLocalToGlobalMapping ltog;
1413 
1414   PetscFunctionBegin;
1415   ierr = MatInitializePackage();CHKERRQ(ierr);
1416   mtype = dm->mattype;
1417   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1418   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1419   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1420   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1421   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1422   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1423   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1424   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1425   if (mbs > 1) bs = mbs;
1426   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1427   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1428   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1429   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1430   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1431   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1432   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1433   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1434   if (!isShell) {
1435     PetscSection subSection;
1436     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1437     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1438     PetscInt     pStart, pEnd, p, dof, cdof;
1439 
1440     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1441     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1442       PetscSection section;
1443       PetscInt     size;
1444 
1445       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1446       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1447       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1448       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1449     } else {
1450       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1451     }
1452     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1453     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1454       PetscInt bdof;
1455 
1456       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1457       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1458       dof  = dof < 0 ? -(dof+1) : dof;
1459       bdof = cdof && (dof-cdof) ? 1 : dof;
1460       if (dof) {
1461         if (bs < 0)          {bs = bdof;}
1462         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1463       }
1464       if (isMatIS) {
1465         PetscInt loff,c,off;
1466         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1467         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1468         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1469       }
1470     }
1471     /* Must have same blocksize on all procs (some might have no points) */
1472     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1473     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1474     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1475     else                            {bs = bsMinMax[0];}
1476     bs = PetscMax(1,bs);
1477     if (isMatIS) { /* Must reduce indices by blocksize */
1478       PetscInt l;
1479 
1480       lsize = lsize/bs;
1481       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1482       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1483     }
1484     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1485     if (isMatIS) {
1486       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1487     }
1488     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1489     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1490     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1491   }
1492   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1493   PetscFunctionReturn(0);
1494 }
1495 
1496 /*@
1497   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1498 
1499   Not collective
1500 
1501   Input Parameter:
1502 . mesh - The DMPlex
1503 
1504   Output Parameters:
1505 . subsection - The subdomain section
1506 
1507   Level: developer
1508 
1509 .seealso:
1510 @*/
1511 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1512 {
1513   DM_Plex       *mesh = (DM_Plex*) dm->data;
1514   PetscErrorCode ierr;
1515 
1516   PetscFunctionBegin;
1517   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1518   if (!mesh->subdomainSection) {
1519     PetscSection section;
1520     PetscSF      sf;
1521 
1522     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1523     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1524     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1525     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1526   }
1527   *subsection = mesh->subdomainSection;
1528   PetscFunctionReturn(0);
1529 }
1530 
1531 /*@
1532   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1533 
1534   Not collective
1535 
1536   Input Parameter:
1537 . mesh - The DMPlex
1538 
1539   Output Parameters:
1540 + pStart - The first mesh point
1541 - pEnd   - The upper bound for mesh points
1542 
1543   Level: beginner
1544 
1545 .seealso: DMPlexCreate(), DMPlexSetChart()
1546 @*/
1547 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1548 {
1549   DM_Plex       *mesh = (DM_Plex*) dm->data;
1550   PetscErrorCode ierr;
1551 
1552   PetscFunctionBegin;
1553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1554   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1555   PetscFunctionReturn(0);
1556 }
1557 
1558 /*@
1559   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1560 
1561   Not collective
1562 
1563   Input Parameters:
1564 + mesh - The DMPlex
1565 . pStart - The first mesh point
1566 - pEnd   - The upper bound for mesh points
1567 
1568   Output Parameters:
1569 
1570   Level: beginner
1571 
1572 .seealso: DMPlexCreate(), DMPlexGetChart()
1573 @*/
1574 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1575 {
1576   DM_Plex       *mesh = (DM_Plex*) dm->data;
1577   PetscErrorCode ierr;
1578 
1579   PetscFunctionBegin;
1580   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1581   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1582   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1583   PetscFunctionReturn(0);
1584 }
1585 
1586 /*@
1587   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1588 
1589   Not collective
1590 
1591   Input Parameters:
1592 + mesh - The DMPlex
1593 - p - The point, which must lie in the chart set with DMPlexSetChart()
1594 
1595   Output Parameter:
1596 . size - The cone size for point p
1597 
1598   Level: beginner
1599 
1600 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1601 @*/
1602 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1603 {
1604   DM_Plex       *mesh = (DM_Plex*) dm->data;
1605   PetscErrorCode ierr;
1606 
1607   PetscFunctionBegin;
1608   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1609   PetscValidPointer(size, 3);
1610   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1611   PetscFunctionReturn(0);
1612 }
1613 
1614 /*@
1615   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1616 
1617   Not collective
1618 
1619   Input Parameters:
1620 + mesh - The DMPlex
1621 . p - The point, which must lie in the chart set with DMPlexSetChart()
1622 - size - The cone size for point p
1623 
1624   Output Parameter:
1625 
1626   Note:
1627   This should be called after DMPlexSetChart().
1628 
1629   Level: beginner
1630 
1631 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1632 @*/
1633 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1634 {
1635   DM_Plex       *mesh = (DM_Plex*) dm->data;
1636   PetscErrorCode ierr;
1637 
1638   PetscFunctionBegin;
1639   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1640   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1641 
1642   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1643   PetscFunctionReturn(0);
1644 }
1645 
1646 /*@
1647   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1648 
1649   Not collective
1650 
1651   Input Parameters:
1652 + mesh - The DMPlex
1653 . p - The point, which must lie in the chart set with DMPlexSetChart()
1654 - size - The additional cone size for point p
1655 
1656   Output Parameter:
1657 
1658   Note:
1659   This should be called after DMPlexSetChart().
1660 
1661   Level: beginner
1662 
1663 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1664 @*/
1665 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1666 {
1667   DM_Plex       *mesh = (DM_Plex*) dm->data;
1668   PetscInt       csize;
1669   PetscErrorCode ierr;
1670 
1671   PetscFunctionBegin;
1672   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1673   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1674   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1675 
1676   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1677   PetscFunctionReturn(0);
1678 }
1679 
1680 /*@C
1681   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1682 
1683   Not collective
1684 
1685   Input Parameters:
1686 + dm - The DMPlex
1687 - p - The point, which must lie in the chart set with DMPlexSetChart()
1688 
1689   Output Parameter:
1690 . cone - An array of points which are on the in-edges for point p
1691 
1692   Level: beginner
1693 
1694   Fortran Notes:
1695   Since it returns an array, this routine is only available in Fortran 90, and you must
1696   include petsc.h90 in your code.
1697   You must also call DMPlexRestoreCone() after you finish using the returned array.
1698   DMPlexRestoreCone() is not needed/available in C.
1699 
1700 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1701 @*/
1702 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1703 {
1704   DM_Plex       *mesh = (DM_Plex*) dm->data;
1705   PetscInt       off;
1706   PetscErrorCode ierr;
1707 
1708   PetscFunctionBegin;
1709   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1710   PetscValidPointer(cone, 3);
1711   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1712   *cone = &mesh->cones[off];
1713   PetscFunctionReturn(0);
1714 }
1715 
1716 /*@C
1717   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1718 
1719   Not collective
1720 
1721   Input Parameters:
1722 + dm - The DMPlex
1723 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1724 
1725   Output Parameter:
1726 + pConesSection - PetscSection describing the layout of pCones
1727 - pCones - An array of points which are on the in-edges for the point set p
1728 
1729   Level: intermediate
1730 
1731 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1732 @*/
1733 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1734 {
1735   PetscSection        cs, newcs;
1736   PetscInt            *cones;
1737   PetscInt            *newarr=NULL;
1738   PetscInt            n;
1739   PetscErrorCode      ierr;
1740 
1741   PetscFunctionBegin;
1742   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1743   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1744   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1745   if (pConesSection) *pConesSection = newcs;
1746   if (pCones) {
1747     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1748     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1749   }
1750   PetscFunctionReturn(0);
1751 }
1752 
1753 /*@
1754   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1755 
1756   Not collective
1757 
1758   Input Parameters:
1759 + dm - The DMPlex
1760 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1761 
1762   Output Parameter:
1763 . expandedPoints - An array of vertices recursively expanded from input points
1764 
1765   Level: advanced
1766 
1767   Notes:
1768   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1769   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1770 
1771 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1772 @*/
1773 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1774 {
1775   IS                  *expandedPointsAll;
1776   PetscInt            depth;
1777   PetscErrorCode      ierr;
1778 
1779   PetscFunctionBegin;
1780   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1781   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1782   PetscValidPointer(expandedPoints, 3);
1783   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1784   *expandedPoints = expandedPointsAll[0];
1785   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1786   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1787   PetscFunctionReturn(0);
1788 }
1789 
1790 /*@
1791   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1792 
1793   Not collective
1794 
1795   Input Parameters:
1796 + dm - The DMPlex
1797 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1798 
1799   Output Parameter:
1800 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1801 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1802 - sections - (optional) An array of sections which describe mappings from points to their cone points
1803 
1804   Level: advanced
1805 
1806   Notes:
1807   Like DMPlexGetConeTuple() but recursive.
1808 
1809   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1810   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1811 
1812   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1813   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1814   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1815 
1816 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1817 @*/
1818 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1819 {
1820   const PetscInt      *arr0=NULL, *cone=NULL;
1821   PetscInt            *arr=NULL, *newarr=NULL;
1822   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1823   IS                  *expandedPoints_;
1824   PetscSection        *sections_;
1825   PetscErrorCode      ierr;
1826 
1827   PetscFunctionBegin;
1828   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1829   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1830   if (depth) PetscValidIntPointer(depth, 3);
1831   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1832   if (sections) PetscValidPointer(sections, 5);
1833   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1834   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1835   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1836   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1837   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1838   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1839   for (d=depth_-1; d>=0; d--) {
1840     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1841     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1842     for (i=0; i<n; i++) {
1843       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1844       if (arr[i] >= start && arr[i] < end) {
1845         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1846         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1847       } else {
1848         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1849       }
1850     }
1851     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1852     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1853     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1854     for (i=0; i<n; i++) {
1855       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1856       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1857       if (cn > 1) {
1858         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1859         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1860       } else {
1861         newarr[co] = arr[i];
1862       }
1863     }
1864     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1865     arr = newarr;
1866     n = newn;
1867   }
1868   *depth = depth_;
1869   if (expandedPoints) *expandedPoints = expandedPoints_;
1870   else {
1871     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1872     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1873   }
1874   if (sections) *sections = sections_;
1875   else {
1876     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1877     ierr = PetscFree(sections_);CHKERRQ(ierr);
1878   }
1879   PetscFunctionReturn(0);
1880 }
1881 
1882 /*@
1883   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1884 
1885   Not collective
1886 
1887   Input Parameters:
1888 + dm - The DMPlex
1889 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1890 
1891   Output Parameter:
1892 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1893 . expandedPoints - (optional) An array of recursively expanded cones
1894 - sections - (optional) An array of sections which describe mappings from points to their cone points
1895 
1896   Level: advanced
1897 
1898   Notes:
1899   See DMPlexGetConeRecursive() for details.
1900 
1901 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1902 @*/
1903 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1904 {
1905   PetscInt            d, depth_;
1906   PetscErrorCode      ierr;
1907 
1908   PetscFunctionBegin;
1909   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1910   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1911   if (depth) *depth = 0;
1912   if (expandedPoints) {
1913     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1914     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1915   }
1916   if (sections)  {
1917     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1918     ierr = PetscFree(*sections);CHKERRQ(ierr);
1919   }
1920   PetscFunctionReturn(0);
1921 }
1922 
1923 /*@
1924   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1925 
1926   Not collective
1927 
1928   Input Parameters:
1929 + mesh - The DMPlex
1930 . p - The point, which must lie in the chart set with DMPlexSetChart()
1931 - cone - An array of points which are on the in-edges for point p
1932 
1933   Output Parameter:
1934 
1935   Note:
1936   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1937 
1938   Developer Note: Why not call this DMPlexSetCover()
1939 
1940   Level: beginner
1941 
1942 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1943 @*/
1944 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1945 {
1946   DM_Plex       *mesh = (DM_Plex*) dm->data;
1947   PetscInt       pStart, pEnd;
1948   PetscInt       dof, off, c;
1949   PetscErrorCode ierr;
1950 
1951   PetscFunctionBegin;
1952   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1953   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1954   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1955   if (dof) PetscValidPointer(cone, 3);
1956   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1957   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1958   for (c = 0; c < dof; ++c) {
1959     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1960     mesh->cones[off+c] = cone[c];
1961   }
1962   PetscFunctionReturn(0);
1963 }
1964 
1965 /*@C
1966   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1967 
1968   Not collective
1969 
1970   Input Parameters:
1971 + mesh - The DMPlex
1972 - p - The point, which must lie in the chart set with DMPlexSetChart()
1973 
1974   Output Parameter:
1975 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1976                     integer giving the prescription for cone traversal. If it is negative, the cone is
1977                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1978                     the index of the cone point on which to start.
1979 
1980   Level: beginner
1981 
1982   Fortran Notes:
1983   Since it returns an array, this routine is only available in Fortran 90, and you must
1984   include petsc.h90 in your code.
1985   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1986   DMPlexRestoreConeOrientation() is not needed/available in C.
1987 
1988 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1989 @*/
1990 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1991 {
1992   DM_Plex       *mesh = (DM_Plex*) dm->data;
1993   PetscInt       off;
1994   PetscErrorCode ierr;
1995 
1996   PetscFunctionBegin;
1997   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1998 #if defined(PETSC_USE_DEBUG)
1999   {
2000     PetscInt dof;
2001     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2002     if (dof) PetscValidPointer(coneOrientation, 3);
2003   }
2004 #endif
2005   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2006 
2007   *coneOrientation = &mesh->coneOrientations[off];
2008   PetscFunctionReturn(0);
2009 }
2010 
2011 /*@
2012   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
2013 
2014   Not collective
2015 
2016   Input Parameters:
2017 + mesh - The DMPlex
2018 . p - The point, which must lie in the chart set with DMPlexSetChart()
2019 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
2020                     integer giving the prescription for cone traversal. If it is negative, the cone is
2021                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
2022                     the index of the cone point on which to start.
2023 
2024   Output Parameter:
2025 
2026   Note:
2027   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
2028 
2029   Level: beginner
2030 
2031 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2032 @*/
2033 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2034 {
2035   DM_Plex       *mesh = (DM_Plex*) dm->data;
2036   PetscInt       pStart, pEnd;
2037   PetscInt       dof, off, c;
2038   PetscErrorCode ierr;
2039 
2040   PetscFunctionBegin;
2041   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2042   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2043   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2044   if (dof) PetscValidPointer(coneOrientation, 3);
2045   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2046   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2047   for (c = 0; c < dof; ++c) {
2048     PetscInt cdof, o = coneOrientation[c];
2049 
2050     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2051     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2052     mesh->coneOrientations[off+c] = o;
2053   }
2054   PetscFunctionReturn(0);
2055 }
2056 
2057 /*@
2058   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2059 
2060   Not collective
2061 
2062   Input Parameters:
2063 + mesh - The DMPlex
2064 . p - The point, which must lie in the chart set with DMPlexSetChart()
2065 . conePos - The local index in the cone where the point should be put
2066 - conePoint - The mesh point to insert
2067 
2068   Level: beginner
2069 
2070 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2071 @*/
2072 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2073 {
2074   DM_Plex       *mesh = (DM_Plex*) dm->data;
2075   PetscInt       pStart, pEnd;
2076   PetscInt       dof, off;
2077   PetscErrorCode ierr;
2078 
2079   PetscFunctionBegin;
2080   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2081   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2082   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2083   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2084   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2085   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2086   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2087   mesh->cones[off+conePos] = conePoint;
2088   PetscFunctionReturn(0);
2089 }
2090 
2091 /*@
2092   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2093 
2094   Not collective
2095 
2096   Input Parameters:
2097 + mesh - The DMPlex
2098 . p - The point, which must lie in the chart set with DMPlexSetChart()
2099 . conePos - The local index in the cone where the point should be put
2100 - coneOrientation - The point orientation to insert
2101 
2102   Level: beginner
2103 
2104 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2105 @*/
2106 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2107 {
2108   DM_Plex       *mesh = (DM_Plex*) dm->data;
2109   PetscInt       pStart, pEnd;
2110   PetscInt       dof, off;
2111   PetscErrorCode ierr;
2112 
2113   PetscFunctionBegin;
2114   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2115   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2116   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2117   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2118   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2119   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2120   mesh->coneOrientations[off+conePos] = coneOrientation;
2121   PetscFunctionReturn(0);
2122 }
2123 
2124 /*@
2125   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2126 
2127   Not collective
2128 
2129   Input Parameters:
2130 + mesh - The DMPlex
2131 - p - The point, which must lie in the chart set with DMPlexSetChart()
2132 
2133   Output Parameter:
2134 . size - The support size for point p
2135 
2136   Level: beginner
2137 
2138 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2139 @*/
2140 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2141 {
2142   DM_Plex       *mesh = (DM_Plex*) dm->data;
2143   PetscErrorCode ierr;
2144 
2145   PetscFunctionBegin;
2146   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2147   PetscValidPointer(size, 3);
2148   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2149   PetscFunctionReturn(0);
2150 }
2151 
2152 /*@
2153   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2154 
2155   Not collective
2156 
2157   Input Parameters:
2158 + mesh - The DMPlex
2159 . p - The point, which must lie in the chart set with DMPlexSetChart()
2160 - size - The support size for point p
2161 
2162   Output Parameter:
2163 
2164   Note:
2165   This should be called after DMPlexSetChart().
2166 
2167   Level: beginner
2168 
2169 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2170 @*/
2171 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2172 {
2173   DM_Plex       *mesh = (DM_Plex*) dm->data;
2174   PetscErrorCode ierr;
2175 
2176   PetscFunctionBegin;
2177   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2178   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2179 
2180   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2181   PetscFunctionReturn(0);
2182 }
2183 
2184 /*@C
2185   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2186 
2187   Not collective
2188 
2189   Input Parameters:
2190 + mesh - The DMPlex
2191 - p - The point, which must lie in the chart set with DMPlexSetChart()
2192 
2193   Output Parameter:
2194 . support - An array of points which are on the out-edges for point p
2195 
2196   Level: beginner
2197 
2198   Fortran Notes:
2199   Since it returns an array, this routine is only available in Fortran 90, and you must
2200   include petsc.h90 in your code.
2201   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2202   DMPlexRestoreSupport() is not needed/available in C.
2203 
2204 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2205 @*/
2206 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2207 {
2208   DM_Plex       *mesh = (DM_Plex*) dm->data;
2209   PetscInt       off;
2210   PetscErrorCode ierr;
2211 
2212   PetscFunctionBegin;
2213   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2214   PetscValidPointer(support, 3);
2215   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2216   *support = &mesh->supports[off];
2217   PetscFunctionReturn(0);
2218 }
2219 
2220 /*@
2221   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2222 
2223   Not collective
2224 
2225   Input Parameters:
2226 + mesh - The DMPlex
2227 . p - The point, which must lie in the chart set with DMPlexSetChart()
2228 - support - An array of points which are on the out-edges for point p
2229 
2230   Output Parameter:
2231 
2232   Note:
2233   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2234 
2235   Level: beginner
2236 
2237 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2238 @*/
2239 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2240 {
2241   DM_Plex       *mesh = (DM_Plex*) dm->data;
2242   PetscInt       pStart, pEnd;
2243   PetscInt       dof, off, c;
2244   PetscErrorCode ierr;
2245 
2246   PetscFunctionBegin;
2247   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2248   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2249   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2250   if (dof) PetscValidPointer(support, 3);
2251   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2252   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2253   for (c = 0; c < dof; ++c) {
2254     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2255     mesh->supports[off+c] = support[c];
2256   }
2257   PetscFunctionReturn(0);
2258 }
2259 
2260 /*@
2261   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2262 
2263   Not collective
2264 
2265   Input Parameters:
2266 + mesh - The DMPlex
2267 . p - The point, which must lie in the chart set with DMPlexSetChart()
2268 . supportPos - The local index in the cone where the point should be put
2269 - supportPoint - The mesh point to insert
2270 
2271   Level: beginner
2272 
2273 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2274 @*/
2275 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2276 {
2277   DM_Plex       *mesh = (DM_Plex*) dm->data;
2278   PetscInt       pStart, pEnd;
2279   PetscInt       dof, off;
2280   PetscErrorCode ierr;
2281 
2282   PetscFunctionBegin;
2283   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2284   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2285   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2286   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2287   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2288   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2289   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2290   mesh->supports[off+supportPos] = supportPoint;
2291   PetscFunctionReturn(0);
2292 }
2293 
2294 /*@C
2295   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2296 
2297   Not collective
2298 
2299   Input Parameters:
2300 + mesh - The DMPlex
2301 . p - The point, which must lie in the chart set with DMPlexSetChart()
2302 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2303 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2304 
2305   Output Parameters:
2306 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2307 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2308 
2309   Note:
2310   If using internal storage (points is NULL on input), each call overwrites the last output.
2311 
2312   Fortran Notes:
2313   Since it returns an array, this routine is only available in Fortran 90, and you must
2314   include petsc.h90 in your code.
2315 
2316   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2317 
2318   Level: beginner
2319 
2320 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2321 @*/
2322 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2323 {
2324   DM_Plex        *mesh = (DM_Plex*) dm->data;
2325   PetscInt       *closure, *fifo;
2326   const PetscInt *tmp = NULL, *tmpO = NULL;
2327   PetscInt        tmpSize, t;
2328   PetscInt        depth       = 0, maxSize;
2329   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2330   PetscErrorCode  ierr;
2331 
2332   PetscFunctionBegin;
2333   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2334   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2335   /* This is only 1-level */
2336   if (useCone) {
2337     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2338     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2339     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2340   } else {
2341     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2342     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2343   }
2344   if (depth == 1) {
2345     if (*points) {
2346       closure = *points;
2347     } else {
2348       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2349       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2350     }
2351     closure[0] = p; closure[1] = 0;
2352     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2353       closure[closureSize]   = tmp[t];
2354       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2355     }
2356     if (numPoints) *numPoints = closureSize/2;
2357     if (points)    *points    = closure;
2358     PetscFunctionReturn(0);
2359   }
2360   {
2361     PetscInt c, coneSeries, s,supportSeries;
2362 
2363     c = mesh->maxConeSize;
2364     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2365     s = mesh->maxSupportSize;
2366     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2367     maxSize = 2*PetscMax(coneSeries,supportSeries);
2368   }
2369   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2370   if (*points) {
2371     closure = *points;
2372   } else {
2373     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2374   }
2375   closure[0] = p; closure[1] = 0;
2376   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2377     const PetscInt cp = tmp[t];
2378     const PetscInt co = tmpO ? tmpO[t] : 0;
2379 
2380     closure[closureSize]   = cp;
2381     closure[closureSize+1] = co;
2382     fifo[fifoSize]         = cp;
2383     fifo[fifoSize+1]       = co;
2384   }
2385   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2386   while (fifoSize - fifoStart) {
2387     const PetscInt q   = fifo[fifoStart];
2388     const PetscInt o   = fifo[fifoStart+1];
2389     const PetscInt rev = o >= 0 ? 0 : 1;
2390     const PetscInt off = rev ? -(o+1) : o;
2391 
2392     if (useCone) {
2393       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2394       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2395       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2396     } else {
2397       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2398       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2399       tmpO = NULL;
2400     }
2401     for (t = 0; t < tmpSize; ++t) {
2402       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2403       const PetscInt cp = tmp[i];
2404       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2405       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2406        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2407       PetscInt       co = tmpO ? tmpO[i] : 0;
2408       PetscInt       c;
2409 
2410       if (rev) {
2411         PetscInt childSize, coff;
2412         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2413         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2414         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2415       }
2416       /* Check for duplicate */
2417       for (c = 0; c < closureSize; c += 2) {
2418         if (closure[c] == cp) break;
2419       }
2420       if (c == closureSize) {
2421         closure[closureSize]   = cp;
2422         closure[closureSize+1] = co;
2423         fifo[fifoSize]         = cp;
2424         fifo[fifoSize+1]       = co;
2425         closureSize           += 2;
2426         fifoSize              += 2;
2427       }
2428     }
2429     fifoStart += 2;
2430   }
2431   if (numPoints) *numPoints = closureSize/2;
2432   if (points)    *points    = closure;
2433   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2434   PetscFunctionReturn(0);
2435 }
2436 
2437 /*@C
2438   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2439 
2440   Not collective
2441 
2442   Input Parameters:
2443 + mesh - The DMPlex
2444 . p - The point, which must lie in the chart set with DMPlexSetChart()
2445 . orientation - The orientation of the point
2446 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2447 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2448 
2449   Output Parameters:
2450 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2451 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2452 
2453   Note:
2454   If using internal storage (points is NULL on input), each call overwrites the last output.
2455 
2456   Fortran Notes:
2457   Since it returns an array, this routine is only available in Fortran 90, and you must
2458   include petsc.h90 in your code.
2459 
2460   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2461 
2462   Level: beginner
2463 
2464 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2465 @*/
2466 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2467 {
2468   DM_Plex        *mesh = (DM_Plex*) dm->data;
2469   PetscInt       *closure, *fifo;
2470   const PetscInt *tmp = NULL, *tmpO = NULL;
2471   PetscInt        tmpSize, t;
2472   PetscInt        depth       = 0, maxSize;
2473   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2474   PetscErrorCode  ierr;
2475 
2476   PetscFunctionBegin;
2477   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2478   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2479   /* This is only 1-level */
2480   if (useCone) {
2481     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2482     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2483     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2484   } else {
2485     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2486     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2487   }
2488   if (depth == 1) {
2489     if (*points) {
2490       closure = *points;
2491     } else {
2492       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2493       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2494     }
2495     closure[0] = p; closure[1] = ornt;
2496     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2497       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2498       closure[closureSize]   = tmp[i];
2499       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2500     }
2501     if (numPoints) *numPoints = closureSize/2;
2502     if (points)    *points    = closure;
2503     PetscFunctionReturn(0);
2504   }
2505   {
2506     PetscInt c, coneSeries, s,supportSeries;
2507 
2508     c = mesh->maxConeSize;
2509     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2510     s = mesh->maxSupportSize;
2511     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2512     maxSize = 2*PetscMax(coneSeries,supportSeries);
2513   }
2514   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2515   if (*points) {
2516     closure = *points;
2517   } else {
2518     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2519   }
2520   closure[0] = p; closure[1] = ornt;
2521   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2522     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2523     const PetscInt cp = tmp[i];
2524     PetscInt       co = tmpO ? tmpO[i] : 0;
2525 
2526     if (ornt < 0) {
2527       PetscInt childSize, coff;
2528       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2529       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2530       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2531     }
2532     closure[closureSize]   = cp;
2533     closure[closureSize+1] = co;
2534     fifo[fifoSize]         = cp;
2535     fifo[fifoSize+1]       = co;
2536   }
2537   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2538   while (fifoSize - fifoStart) {
2539     const PetscInt q   = fifo[fifoStart];
2540     const PetscInt o   = fifo[fifoStart+1];
2541     const PetscInt rev = o >= 0 ? 0 : 1;
2542     const PetscInt off = rev ? -(o+1) : o;
2543 
2544     if (useCone) {
2545       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2546       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2547       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2548     } else {
2549       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2550       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2551       tmpO = NULL;
2552     }
2553     for (t = 0; t < tmpSize; ++t) {
2554       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2555       const PetscInt cp = tmp[i];
2556       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2557       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2558        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2559       PetscInt       co = tmpO ? tmpO[i] : 0;
2560       PetscInt       c;
2561 
2562       if (rev) {
2563         PetscInt childSize, coff;
2564         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2565         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2566         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2567       }
2568       /* Check for duplicate */
2569       for (c = 0; c < closureSize; c += 2) {
2570         if (closure[c] == cp) break;
2571       }
2572       if (c == closureSize) {
2573         closure[closureSize]   = cp;
2574         closure[closureSize+1] = co;
2575         fifo[fifoSize]         = cp;
2576         fifo[fifoSize+1]       = co;
2577         closureSize           += 2;
2578         fifoSize              += 2;
2579       }
2580     }
2581     fifoStart += 2;
2582   }
2583   if (numPoints) *numPoints = closureSize/2;
2584   if (points)    *points    = closure;
2585   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2586   PetscFunctionReturn(0);
2587 }
2588 
2589 /*@C
2590   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2591 
2592   Not collective
2593 
2594   Input Parameters:
2595 + mesh - The DMPlex
2596 . p - The point, which must lie in the chart set with DMPlexSetChart()
2597 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2598 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2599 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2600 
2601   Note:
2602   If not using internal storage (points is not NULL on input), this call is unnecessary
2603 
2604   Fortran Notes:
2605   Since it returns an array, this routine is only available in Fortran 90, and you must
2606   include petsc.h90 in your code.
2607 
2608   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2609 
2610   Level: beginner
2611 
2612 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2613 @*/
2614 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2615 {
2616   PetscErrorCode ierr;
2617 
2618   PetscFunctionBegin;
2619   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2620   if (numPoints) PetscValidIntPointer(numPoints,4);
2621   if (points) PetscValidPointer(points,5);
2622   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2623   if (numPoints) *numPoints = 0;
2624   PetscFunctionReturn(0);
2625 }
2626 
2627 /*@
2628   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2629 
2630   Not collective
2631 
2632   Input Parameter:
2633 . mesh - The DMPlex
2634 
2635   Output Parameters:
2636 + maxConeSize - The maximum number of in-edges
2637 - maxSupportSize - The maximum number of out-edges
2638 
2639   Level: beginner
2640 
2641 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2642 @*/
2643 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2644 {
2645   DM_Plex *mesh = (DM_Plex*) dm->data;
2646 
2647   PetscFunctionBegin;
2648   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2649   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2650   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2651   PetscFunctionReturn(0);
2652 }
2653 
2654 PetscErrorCode DMSetUp_Plex(DM dm)
2655 {
2656   DM_Plex       *mesh = (DM_Plex*) dm->data;
2657   PetscInt       size;
2658   PetscErrorCode ierr;
2659 
2660   PetscFunctionBegin;
2661   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2662   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2663   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2664   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2665   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2666   if (mesh->maxSupportSize) {
2667     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2668     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2669     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2670   }
2671   PetscFunctionReturn(0);
2672 }
2673 
2674 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2675 {
2676   PetscErrorCode ierr;
2677 
2678   PetscFunctionBegin;
2679   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2680   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2681   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2682   if (dm->useNatural && dm->sfMigration) {
2683     PetscSF        sfMigrationInv,sfNatural;
2684     PetscSection   section, sectionSeq;
2685 
2686     (*subdm)->sfMigration = dm->sfMigration;
2687     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2688     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);
2689     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2690     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2691     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2692 
2693     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2694     (*subdm)->sfNatural = sfNatural;
2695     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2696     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2697   }
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2702 {
2703   PetscErrorCode ierr;
2704   PetscInt       i = 0;
2705 
2706   PetscFunctionBegin;
2707   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2708   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2709   (*superdm)->useNatural = PETSC_FALSE;
2710   for (i = 0; i < len; i++){
2711     if (dms[i]->useNatural && dms[i]->sfMigration) {
2712       PetscSF        sfMigrationInv,sfNatural;
2713       PetscSection   section, sectionSeq;
2714 
2715       (*superdm)->sfMigration = dms[i]->sfMigration;
2716       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2717       (*superdm)->useNatural = PETSC_TRUE;
2718       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2719       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2720       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2721       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2722 
2723       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2724       (*superdm)->sfNatural = sfNatural;
2725       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2726       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2727       break;
2728     }
2729   }
2730   PetscFunctionReturn(0);
2731 }
2732 
2733 /*@
2734   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2735 
2736   Not collective
2737 
2738   Input Parameter:
2739 . mesh - The DMPlex
2740 
2741   Output Parameter:
2742 
2743   Note:
2744   This should be called after all calls to DMPlexSetCone()
2745 
2746   Level: beginner
2747 
2748 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2749 @*/
2750 PetscErrorCode DMPlexSymmetrize(DM dm)
2751 {
2752   DM_Plex       *mesh = (DM_Plex*) dm->data;
2753   PetscInt      *offsets;
2754   PetscInt       supportSize;
2755   PetscInt       pStart, pEnd, p;
2756   PetscErrorCode ierr;
2757 
2758   PetscFunctionBegin;
2759   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2760   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2761   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2762   /* Calculate support sizes */
2763   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2764   for (p = pStart; p < pEnd; ++p) {
2765     PetscInt dof, off, c;
2766 
2767     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2768     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2769     for (c = off; c < off+dof; ++c) {
2770       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2771     }
2772   }
2773   for (p = pStart; p < pEnd; ++p) {
2774     PetscInt dof;
2775 
2776     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2777 
2778     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2779   }
2780   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2781   /* Calculate supports */
2782   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2783   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2784   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2785   for (p = pStart; p < pEnd; ++p) {
2786     PetscInt dof, off, c;
2787 
2788     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2789     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2790     for (c = off; c < off+dof; ++c) {
2791       const PetscInt q = mesh->cones[c];
2792       PetscInt       offS;
2793 
2794       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2795 
2796       mesh->supports[offS+offsets[q]] = p;
2797       ++offsets[q];
2798     }
2799   }
2800   ierr = PetscFree(offsets);CHKERRQ(ierr);
2801   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2802   PetscFunctionReturn(0);
2803 }
2804 
2805 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2806 {
2807   IS             stratumIS;
2808   PetscErrorCode ierr;
2809 
2810   PetscFunctionBegin;
2811   if (pStart >= pEnd) PetscFunctionReturn(0);
2812 #if defined(PETSC_USE_DEBUG)
2813   {
2814     PetscInt  qStart, qEnd, numLevels, level;
2815     PetscBool overlap = PETSC_FALSE;
2816     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2817     for (level = 0; level < numLevels; level++) {
2818       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2819       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2820     }
2821     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2822   }
2823 #endif
2824   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2825   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2826   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2827   PetscFunctionReturn(0);
2828 }
2829 
2830 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2831 
2832 /*@
2833   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2834   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2835   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2836   the DAG.
2837 
2838   Collective on dm
2839 
2840   Input Parameter:
2841 . mesh - The DMPlex
2842 
2843   Output Parameter:
2844 
2845   Notes:
2846   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
2847   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
2848   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2849   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2850   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2851 
2852   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
2853   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
2854   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
2855   to interpolate only that one (e0), so that
2856 $  cone(c0) = {e0, v2}
2857 $  cone(e0) = {v0, v1}
2858   If DMPlexStratify() is run on this mesh, it will give depths
2859 $  depth 0 = {v0, v1, v2}
2860 $  depth 1 = {e0, c0}
2861   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
2862 
2863   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2864 
2865   Level: beginner
2866 
2867 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2868 @*/
2869 PetscErrorCode DMPlexStratify(DM dm)
2870 {
2871   DM_Plex       *mesh = (DM_Plex*) dm->data;
2872   DMLabel        label;
2873   PetscInt       pStart, pEnd, p;
2874   PetscInt       numRoots = 0, numLeaves = 0;
2875   PetscInt       cMax, fMax, eMax, vMax;
2876   PetscErrorCode ierr;
2877 
2878   PetscFunctionBegin;
2879   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2880   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2881 
2882   /* Create depth label */
2883   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2884   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2885   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2886 
2887   {
2888     /* Initialize roots and count leaves */
2889     PetscInt sMin = PETSC_MAX_INT;
2890     PetscInt sMax = PETSC_MIN_INT;
2891     PetscInt coneSize, supportSize;
2892 
2893     for (p = pStart; p < pEnd; ++p) {
2894       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2895       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2896       if (!coneSize && supportSize) {
2897         sMin = PetscMin(p, sMin);
2898         sMax = PetscMax(p, sMax);
2899         ++numRoots;
2900       } else if (!supportSize && coneSize) {
2901         ++numLeaves;
2902       } else if (!supportSize && !coneSize) {
2903         /* Isolated points */
2904         sMin = PetscMin(p, sMin);
2905         sMax = PetscMax(p, sMax);
2906       }
2907     }
2908     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2909   }
2910 
2911   if (numRoots + numLeaves == (pEnd - pStart)) {
2912     PetscInt sMin = PETSC_MAX_INT;
2913     PetscInt sMax = PETSC_MIN_INT;
2914     PetscInt coneSize, supportSize;
2915 
2916     for (p = pStart; p < pEnd; ++p) {
2917       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2918       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2919       if (!supportSize && coneSize) {
2920         sMin = PetscMin(p, sMin);
2921         sMax = PetscMax(p, sMax);
2922       }
2923     }
2924     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2925   } else {
2926     PetscInt level = 0;
2927     PetscInt qStart, qEnd, q;
2928 
2929     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2930     while (qEnd > qStart) {
2931       PetscInt sMin = PETSC_MAX_INT;
2932       PetscInt sMax = PETSC_MIN_INT;
2933 
2934       for (q = qStart; q < qEnd; ++q) {
2935         const PetscInt *support;
2936         PetscInt        supportSize, s;
2937 
2938         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2939         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2940         for (s = 0; s < supportSize; ++s) {
2941           sMin = PetscMin(support[s], sMin);
2942           sMax = PetscMax(support[s], sMax);
2943         }
2944       }
2945       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2946       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2947       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2948     }
2949   }
2950   { /* just in case there is an empty process */
2951     PetscInt numValues, maxValues = 0, v;
2952 
2953     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2954     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2955     for (v = numValues; v < maxValues; v++) {
2956       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2957     }
2958   }
2959   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2960 
2961   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2962   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2963     PetscInt dim;
2964     DMLabel  dimLabel;
2965 
2966     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2967     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2968     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2969     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2970     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2971     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2972     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2973   }
2974   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2975   PetscFunctionReturn(0);
2976 }
2977 
2978 /*@C
2979   DMPlexGetJoin - Get an array for the join of the set of points
2980 
2981   Not Collective
2982 
2983   Input Parameters:
2984 + dm - The DMPlex object
2985 . numPoints - The number of input points for the join
2986 - points - The input points
2987 
2988   Output Parameters:
2989 + numCoveredPoints - The number of points in the join
2990 - coveredPoints - The points in the join
2991 
2992   Level: intermediate
2993 
2994   Note: Currently, this is restricted to a single level join
2995 
2996   Fortran Notes:
2997   Since it returns an array, this routine is only available in Fortran 90, and you must
2998   include petsc.h90 in your code.
2999 
3000   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3001 
3002 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
3003 @*/
3004 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3005 {
3006   DM_Plex       *mesh = (DM_Plex*) dm->data;
3007   PetscInt      *join[2];
3008   PetscInt       joinSize, i = 0;
3009   PetscInt       dof, off, p, c, m;
3010   PetscErrorCode ierr;
3011 
3012   PetscFunctionBegin;
3013   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3014   PetscValidIntPointer(points, 3);
3015   PetscValidIntPointer(numCoveredPoints, 4);
3016   PetscValidPointer(coveredPoints, 5);
3017   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3018   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3019   /* Copy in support of first point */
3020   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
3021   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
3022   for (joinSize = 0; joinSize < dof; ++joinSize) {
3023     join[i][joinSize] = mesh->supports[off+joinSize];
3024   }
3025   /* Check each successive support */
3026   for (p = 1; p < numPoints; ++p) {
3027     PetscInt newJoinSize = 0;
3028 
3029     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
3030     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
3031     for (c = 0; c < dof; ++c) {
3032       const PetscInt point = mesh->supports[off+c];
3033 
3034       for (m = 0; m < joinSize; ++m) {
3035         if (point == join[i][m]) {
3036           join[1-i][newJoinSize++] = point;
3037           break;
3038         }
3039       }
3040     }
3041     joinSize = newJoinSize;
3042     i        = 1-i;
3043   }
3044   *numCoveredPoints = joinSize;
3045   *coveredPoints    = join[i];
3046   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3047   PetscFunctionReturn(0);
3048 }
3049 
3050 /*@C
3051   DMPlexRestoreJoin - Restore an array for the join of the set of points
3052 
3053   Not Collective
3054 
3055   Input Parameters:
3056 + dm - The DMPlex object
3057 . numPoints - The number of input points for the join
3058 - points - The input points
3059 
3060   Output Parameters:
3061 + numCoveredPoints - The number of points in the join
3062 - coveredPoints - The points in the join
3063 
3064   Fortran Notes:
3065   Since it returns an array, this routine is only available in Fortran 90, and you must
3066   include petsc.h90 in your code.
3067 
3068   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3069 
3070   Level: intermediate
3071 
3072 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3073 @*/
3074 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3075 {
3076   PetscErrorCode ierr;
3077 
3078   PetscFunctionBegin;
3079   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3080   if (points) PetscValidIntPointer(points,3);
3081   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3082   PetscValidPointer(coveredPoints, 5);
3083   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3084   if (numCoveredPoints) *numCoveredPoints = 0;
3085   PetscFunctionReturn(0);
3086 }
3087 
3088 /*@C
3089   DMPlexGetFullJoin - Get an array for the join of the set of points
3090 
3091   Not Collective
3092 
3093   Input Parameters:
3094 + dm - The DMPlex object
3095 . numPoints - The number of input points for the join
3096 - points - The input points
3097 
3098   Output Parameters:
3099 + numCoveredPoints - The number of points in the join
3100 - coveredPoints - The points in the join
3101 
3102   Fortran Notes:
3103   Since it returns an array, this routine is only available in Fortran 90, and you must
3104   include petsc.h90 in your code.
3105 
3106   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3107 
3108   Level: intermediate
3109 
3110 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3111 @*/
3112 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3113 {
3114   DM_Plex       *mesh = (DM_Plex*) dm->data;
3115   PetscInt      *offsets, **closures;
3116   PetscInt      *join[2];
3117   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3118   PetscInt       p, d, c, m, ms;
3119   PetscErrorCode ierr;
3120 
3121   PetscFunctionBegin;
3122   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3123   PetscValidIntPointer(points, 3);
3124   PetscValidIntPointer(numCoveredPoints, 4);
3125   PetscValidPointer(coveredPoints, 5);
3126 
3127   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3128   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3129   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3130   ms      = mesh->maxSupportSize;
3131   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3132   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3133   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3134 
3135   for (p = 0; p < numPoints; ++p) {
3136     PetscInt closureSize;
3137 
3138     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3139 
3140     offsets[p*(depth+2)+0] = 0;
3141     for (d = 0; d < depth+1; ++d) {
3142       PetscInt pStart, pEnd, i;
3143 
3144       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3145       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3146         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3147           offsets[p*(depth+2)+d+1] = i;
3148           break;
3149         }
3150       }
3151       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3152     }
3153     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3154   }
3155   for (d = 0; d < depth+1; ++d) {
3156     PetscInt dof;
3157 
3158     /* Copy in support of first point */
3159     dof = offsets[d+1] - offsets[d];
3160     for (joinSize = 0; joinSize < dof; ++joinSize) {
3161       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3162     }
3163     /* Check each successive cone */
3164     for (p = 1; p < numPoints && joinSize; ++p) {
3165       PetscInt newJoinSize = 0;
3166 
3167       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3168       for (c = 0; c < dof; ++c) {
3169         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3170 
3171         for (m = 0; m < joinSize; ++m) {
3172           if (point == join[i][m]) {
3173             join[1-i][newJoinSize++] = point;
3174             break;
3175           }
3176         }
3177       }
3178       joinSize = newJoinSize;
3179       i        = 1-i;
3180     }
3181     if (joinSize) break;
3182   }
3183   *numCoveredPoints = joinSize;
3184   *coveredPoints    = join[i];
3185   for (p = 0; p < numPoints; ++p) {
3186     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3187   }
3188   ierr = PetscFree(closures);CHKERRQ(ierr);
3189   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3190   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3191   PetscFunctionReturn(0);
3192 }
3193 
3194 /*@C
3195   DMPlexGetMeet - Get an array for the meet of the set of points
3196 
3197   Not Collective
3198 
3199   Input Parameters:
3200 + dm - The DMPlex object
3201 . numPoints - The number of input points for the meet
3202 - points - The input points
3203 
3204   Output Parameters:
3205 + numCoveredPoints - The number of points in the meet
3206 - coveredPoints - The points in the meet
3207 
3208   Level: intermediate
3209 
3210   Note: Currently, this is restricted to a single level meet
3211 
3212   Fortran Notes:
3213   Since it returns an array, this routine is only available in Fortran 90, and you must
3214   include petsc.h90 in your code.
3215 
3216   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3217 
3218 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3219 @*/
3220 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3221 {
3222   DM_Plex       *mesh = (DM_Plex*) dm->data;
3223   PetscInt      *meet[2];
3224   PetscInt       meetSize, i = 0;
3225   PetscInt       dof, off, p, c, m;
3226   PetscErrorCode ierr;
3227 
3228   PetscFunctionBegin;
3229   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3230   PetscValidPointer(points, 2);
3231   PetscValidPointer(numCoveringPoints, 3);
3232   PetscValidPointer(coveringPoints, 4);
3233   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3234   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3235   /* Copy in cone of first point */
3236   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3237   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3238   for (meetSize = 0; meetSize < dof; ++meetSize) {
3239     meet[i][meetSize] = mesh->cones[off+meetSize];
3240   }
3241   /* Check each successive cone */
3242   for (p = 1; p < numPoints; ++p) {
3243     PetscInt newMeetSize = 0;
3244 
3245     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3246     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3247     for (c = 0; c < dof; ++c) {
3248       const PetscInt point = mesh->cones[off+c];
3249 
3250       for (m = 0; m < meetSize; ++m) {
3251         if (point == meet[i][m]) {
3252           meet[1-i][newMeetSize++] = point;
3253           break;
3254         }
3255       }
3256     }
3257     meetSize = newMeetSize;
3258     i        = 1-i;
3259   }
3260   *numCoveringPoints = meetSize;
3261   *coveringPoints    = meet[i];
3262   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3263   PetscFunctionReturn(0);
3264 }
3265 
3266 /*@C
3267   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3268 
3269   Not Collective
3270 
3271   Input Parameters:
3272 + dm - The DMPlex object
3273 . numPoints - The number of input points for the meet
3274 - points - The input points
3275 
3276   Output Parameters:
3277 + numCoveredPoints - The number of points in the meet
3278 - coveredPoints - The points in the meet
3279 
3280   Level: intermediate
3281 
3282   Fortran Notes:
3283   Since it returns an array, this routine is only available in Fortran 90, and you must
3284   include petsc.h90 in your code.
3285 
3286   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3287 
3288 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3289 @*/
3290 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3291 {
3292   PetscErrorCode ierr;
3293 
3294   PetscFunctionBegin;
3295   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3296   if (points) PetscValidIntPointer(points,3);
3297   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3298   PetscValidPointer(coveredPoints,5);
3299   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3300   if (numCoveredPoints) *numCoveredPoints = 0;
3301   PetscFunctionReturn(0);
3302 }
3303 
3304 /*@C
3305   DMPlexGetFullMeet - Get an array for the meet of the set of points
3306 
3307   Not Collective
3308 
3309   Input Parameters:
3310 + dm - The DMPlex object
3311 . numPoints - The number of input points for the meet
3312 - points - The input points
3313 
3314   Output Parameters:
3315 + numCoveredPoints - The number of points in the meet
3316 - coveredPoints - The points in the meet
3317 
3318   Level: intermediate
3319 
3320   Fortran Notes:
3321   Since it returns an array, this routine is only available in Fortran 90, and you must
3322   include petsc.h90 in your code.
3323 
3324   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3325 
3326 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3327 @*/
3328 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3329 {
3330   DM_Plex       *mesh = (DM_Plex*) dm->data;
3331   PetscInt      *offsets, **closures;
3332   PetscInt      *meet[2];
3333   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3334   PetscInt       p, h, c, m, mc;
3335   PetscErrorCode ierr;
3336 
3337   PetscFunctionBegin;
3338   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3339   PetscValidPointer(points, 2);
3340   PetscValidPointer(numCoveredPoints, 3);
3341   PetscValidPointer(coveredPoints, 4);
3342 
3343   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3344   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3345   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3346   mc      = mesh->maxConeSize;
3347   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3348   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3349   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3350 
3351   for (p = 0; p < numPoints; ++p) {
3352     PetscInt closureSize;
3353 
3354     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3355 
3356     offsets[p*(height+2)+0] = 0;
3357     for (h = 0; h < height+1; ++h) {
3358       PetscInt pStart, pEnd, i;
3359 
3360       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3361       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3362         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3363           offsets[p*(height+2)+h+1] = i;
3364           break;
3365         }
3366       }
3367       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3368     }
3369     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3370   }
3371   for (h = 0; h < height+1; ++h) {
3372     PetscInt dof;
3373 
3374     /* Copy in cone of first point */
3375     dof = offsets[h+1] - offsets[h];
3376     for (meetSize = 0; meetSize < dof; ++meetSize) {
3377       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3378     }
3379     /* Check each successive cone */
3380     for (p = 1; p < numPoints && meetSize; ++p) {
3381       PetscInt newMeetSize = 0;
3382 
3383       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3384       for (c = 0; c < dof; ++c) {
3385         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3386 
3387         for (m = 0; m < meetSize; ++m) {
3388           if (point == meet[i][m]) {
3389             meet[1-i][newMeetSize++] = point;
3390             break;
3391           }
3392         }
3393       }
3394       meetSize = newMeetSize;
3395       i        = 1-i;
3396     }
3397     if (meetSize) break;
3398   }
3399   *numCoveredPoints = meetSize;
3400   *coveredPoints    = meet[i];
3401   for (p = 0; p < numPoints; ++p) {
3402     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3403   }
3404   ierr = PetscFree(closures);CHKERRQ(ierr);
3405   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3406   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3407   PetscFunctionReturn(0);
3408 }
3409 
3410 /*@C
3411   DMPlexEqual - Determine if two DMs have the same topology
3412 
3413   Not Collective
3414 
3415   Input Parameters:
3416 + dmA - A DMPlex object
3417 - dmB - A DMPlex object
3418 
3419   Output Parameters:
3420 . equal - PETSC_TRUE if the topologies are identical
3421 
3422   Level: intermediate
3423 
3424   Notes:
3425   We are not solving graph isomorphism, so we do not permutation.
3426 
3427 .seealso: DMPlexGetCone()
3428 @*/
3429 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3430 {
3431   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3432   PetscErrorCode ierr;
3433 
3434   PetscFunctionBegin;
3435   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3436   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3437   PetscValidPointer(equal, 3);
3438 
3439   *equal = PETSC_FALSE;
3440   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3441   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3442   if (depth != depthB) PetscFunctionReturn(0);
3443   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3444   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3445   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3446   for (p = pStart; p < pEnd; ++p) {
3447     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3448     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3449 
3450     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3451     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3452     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3453     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3454     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3455     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3456     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3457     for (c = 0; c < coneSize; ++c) {
3458       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3459       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3460     }
3461     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3462     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3463     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3464     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3465     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3466     for (s = 0; s < supportSize; ++s) {
3467       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3468     }
3469   }
3470   *equal = PETSC_TRUE;
3471   PetscFunctionReturn(0);
3472 }
3473 
3474 /*@C
3475   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3476 
3477   Not Collective
3478 
3479   Input Parameters:
3480 + dm         - The DMPlex
3481 . cellDim    - The cell dimension
3482 - numCorners - The number of vertices on a cell
3483 
3484   Output Parameters:
3485 . numFaceVertices - The number of vertices on a face
3486 
3487   Level: developer
3488 
3489   Notes:
3490   Of course this can only work for a restricted set of symmetric shapes
3491 
3492 .seealso: DMPlexGetCone()
3493 @*/
3494 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3495 {
3496   MPI_Comm       comm;
3497   PetscErrorCode ierr;
3498 
3499   PetscFunctionBegin;
3500   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3501   PetscValidPointer(numFaceVertices,3);
3502   switch (cellDim) {
3503   case 0:
3504     *numFaceVertices = 0;
3505     break;
3506   case 1:
3507     *numFaceVertices = 1;
3508     break;
3509   case 2:
3510     switch (numCorners) {
3511     case 3: /* triangle */
3512       *numFaceVertices = 2; /* Edge has 2 vertices */
3513       break;
3514     case 4: /* quadrilateral */
3515       *numFaceVertices = 2; /* Edge has 2 vertices */
3516       break;
3517     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3518       *numFaceVertices = 3; /* Edge has 3 vertices */
3519       break;
3520     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3521       *numFaceVertices = 3; /* Edge has 3 vertices */
3522       break;
3523     default:
3524       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3525     }
3526     break;
3527   case 3:
3528     switch (numCorners) {
3529     case 4: /* tetradehdron */
3530       *numFaceVertices = 3; /* Face has 3 vertices */
3531       break;
3532     case 6: /* tet cohesive cells */
3533       *numFaceVertices = 4; /* Face has 4 vertices */
3534       break;
3535     case 8: /* hexahedron */
3536       *numFaceVertices = 4; /* Face has 4 vertices */
3537       break;
3538     case 9: /* tet cohesive Lagrange cells */
3539       *numFaceVertices = 6; /* Face has 6 vertices */
3540       break;
3541     case 10: /* quadratic tetrahedron */
3542       *numFaceVertices = 6; /* Face has 6 vertices */
3543       break;
3544     case 12: /* hex cohesive Lagrange cells */
3545       *numFaceVertices = 6; /* Face has 6 vertices */
3546       break;
3547     case 18: /* quadratic tet cohesive Lagrange cells */
3548       *numFaceVertices = 6; /* Face has 6 vertices */
3549       break;
3550     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3551       *numFaceVertices = 9; /* Face has 9 vertices */
3552       break;
3553     default:
3554       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3555     }
3556     break;
3557   default:
3558     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3559   }
3560   PetscFunctionReturn(0);
3561 }
3562 
3563 /*@
3564   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3565 
3566   Not Collective
3567 
3568   Input Parameter:
3569 . dm    - The DMPlex object
3570 
3571   Output Parameter:
3572 . depthLabel - The DMLabel recording point depth
3573 
3574   Level: developer
3575 
3576 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3577 @*/
3578 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3579 {
3580   PetscFunctionBegin;
3581   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3582   PetscValidPointer(depthLabel, 2);
3583   *depthLabel = dm->depthLabel;
3584   PetscFunctionReturn(0);
3585 }
3586 
3587 /*@
3588   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3589 
3590   Not Collective
3591 
3592   Input Parameter:
3593 . dm    - The DMPlex object
3594 
3595   Output Parameter:
3596 . depth - The number of strata (breadth first levels) in the DAG
3597 
3598   Level: developer
3599 
3600   Notes:
3601   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
3602   The point depth is described more in detail in DMPlexSymmetrize().
3603 
3604 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexSymmetrize()
3605 @*/
3606 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3607 {
3608   DMLabel        label;
3609   PetscInt       d = 0;
3610   PetscErrorCode ierr;
3611 
3612   PetscFunctionBegin;
3613   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3614   PetscValidPointer(depth, 2);
3615   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3616   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3617   *depth = d-1;
3618   PetscFunctionReturn(0);
3619 }
3620 
3621 /*@
3622   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3623 
3624   Not Collective
3625 
3626   Input Parameters:
3627 + dm           - The DMPlex object
3628 - stratumValue - The requested depth
3629 
3630   Output Parameters:
3631 + start - The first point at this depth
3632 - end   - One beyond the last point at this depth
3633 
3634   Notes:
3635   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3636   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3637   higher dimension, e.g., "edges".
3638 
3639   Level: developer
3640 
3641 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3642 @*/
3643 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3644 {
3645   DMLabel        label;
3646   PetscInt       pStart, pEnd;
3647   PetscErrorCode ierr;
3648 
3649   PetscFunctionBegin;
3650   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3651   if (start) {PetscValidPointer(start, 3); *start = 0;}
3652   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3653   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3654   if (pStart == pEnd) PetscFunctionReturn(0);
3655   if (stratumValue < 0) {
3656     if (start) *start = pStart;
3657     if (end)   *end   = pEnd;
3658     PetscFunctionReturn(0);
3659   }
3660   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3661   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3662   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3663   PetscFunctionReturn(0);
3664 }
3665 
3666 /*@
3667   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3668 
3669   Not Collective
3670 
3671   Input Parameters:
3672 + dm           - The DMPlex object
3673 - stratumValue - The requested height
3674 
3675   Output Parameters:
3676 + start - The first point at this height
3677 - end   - One beyond the last point at this height
3678 
3679   Notes:
3680   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3681   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3682   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3683 
3684   Level: developer
3685 
3686 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3687 @*/
3688 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3689 {
3690   DMLabel        label;
3691   PetscInt       depth, pStart, pEnd;
3692   PetscErrorCode ierr;
3693 
3694   PetscFunctionBegin;
3695   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3696   if (start) {PetscValidPointer(start, 3); *start = 0;}
3697   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3698   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3699   if (pStart == pEnd) PetscFunctionReturn(0);
3700   if (stratumValue < 0) {
3701     if (start) *start = pStart;
3702     if (end)   *end   = pEnd;
3703     PetscFunctionReturn(0);
3704   }
3705   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3706   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3707   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3708   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3709   PetscFunctionReturn(0);
3710 }
3711 
3712 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3713 {
3714   PetscSection   section, s;
3715   Mat            m;
3716   PetscInt       maxHeight;
3717   PetscErrorCode ierr;
3718 
3719   PetscFunctionBegin;
3720   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3721   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3722   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3723   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3724   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3725   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3726   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3727   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3728   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3729   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3730   ierr = MatDestroy(&m);CHKERRQ(ierr);
3731 
3732   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3733   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3734   PetscFunctionReturn(0);
3735 }
3736 
3737 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3738 {
3739   Vec            coordsLocal;
3740   DM             coordsDM;
3741   PetscErrorCode ierr;
3742 
3743   PetscFunctionBegin;
3744   *field = NULL;
3745   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3746   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3747   if (coordsLocal && coordsDM) {
3748     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3749   }
3750   PetscFunctionReturn(0);
3751 }
3752 
3753 /*@C
3754   DMPlexGetConeSection - Return a section which describes the layout of cone data
3755 
3756   Not Collective
3757 
3758   Input Parameters:
3759 . dm        - The DMPlex object
3760 
3761   Output Parameter:
3762 . section - The PetscSection object
3763 
3764   Level: developer
3765 
3766 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3767 @*/
3768 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3769 {
3770   DM_Plex *mesh = (DM_Plex*) dm->data;
3771 
3772   PetscFunctionBegin;
3773   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3774   if (section) *section = mesh->coneSection;
3775   PetscFunctionReturn(0);
3776 }
3777 
3778 /*@C
3779   DMPlexGetSupportSection - Return a section which describes the layout of support data
3780 
3781   Not Collective
3782 
3783   Input Parameters:
3784 . dm        - The DMPlex object
3785 
3786   Output Parameter:
3787 . section - The PetscSection object
3788 
3789   Level: developer
3790 
3791 .seealso: DMPlexGetConeSection()
3792 @*/
3793 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3794 {
3795   DM_Plex *mesh = (DM_Plex*) dm->data;
3796 
3797   PetscFunctionBegin;
3798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3799   if (section) *section = mesh->supportSection;
3800   PetscFunctionReturn(0);
3801 }
3802 
3803 /*@C
3804   DMPlexGetCones - Return cone data
3805 
3806   Not Collective
3807 
3808   Input Parameters:
3809 . dm        - The DMPlex object
3810 
3811   Output Parameter:
3812 . cones - The cone for each point
3813 
3814   Level: developer
3815 
3816 .seealso: DMPlexGetConeSection()
3817 @*/
3818 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3819 {
3820   DM_Plex *mesh = (DM_Plex*) dm->data;
3821 
3822   PetscFunctionBegin;
3823   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3824   if (cones) *cones = mesh->cones;
3825   PetscFunctionReturn(0);
3826 }
3827 
3828 /*@C
3829   DMPlexGetConeOrientations - Return cone orientation data
3830 
3831   Not Collective
3832 
3833   Input Parameters:
3834 . dm        - The DMPlex object
3835 
3836   Output Parameter:
3837 . coneOrientations - The cone orientation for each point
3838 
3839   Level: developer
3840 
3841 .seealso: DMPlexGetConeSection()
3842 @*/
3843 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3844 {
3845   DM_Plex *mesh = (DM_Plex*) dm->data;
3846 
3847   PetscFunctionBegin;
3848   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3849   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3850   PetscFunctionReturn(0);
3851 }
3852 
3853 /******************************** FEM Support **********************************/
3854 
3855 /*
3856  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3857  representing a line in the section.
3858 */
3859 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3860 {
3861   PetscErrorCode ierr;
3862 
3863   PetscFunctionBeginHot;
3864   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3865   if (line < 0) {
3866     *k = 0;
3867     *Nc = 0;
3868   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3869     *k = 1;
3870   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3871     /* An order k SEM disc has k-1 dofs on an edge */
3872     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3873     *k = *k / *Nc + 1;
3874   }
3875   PetscFunctionReturn(0);
3876 }
3877 
3878 /*@
3879 
3880   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3881   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3882   section provided (or the section of the DM).
3883 
3884   Input Parameters:
3885 + dm      - The DM
3886 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3887 - section - The PetscSection to reorder, or NULL for the default section
3888 
3889   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3890   degree of the basis.
3891 
3892   Example:
3893   A typical interpolated single-quad mesh might order points as
3894 .vb
3895   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3896 
3897   v4 -- e6 -- v3
3898   |           |
3899   e7    c0    e8
3900   |           |
3901   v1 -- e5 -- v2
3902 .ve
3903 
3904   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3905   dofs in the order of points, e.g.,
3906 .vb
3907     c0 -> [0,1,2,3]
3908     v1 -> [4]
3909     ...
3910     e5 -> [8, 9]
3911 .ve
3912 
3913   which corresponds to the dofs
3914 .vb
3915     6   10  11  7
3916     13  2   3   15
3917     12  0   1   14
3918     4   8   9   5
3919 .ve
3920 
3921   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3922 .vb
3923   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3924 .ve
3925 
3926   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3927 .vb
3928    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3929 .ve
3930 
3931   Level: developer
3932 
3933 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlobalSection()
3934 @*/
3935 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3936 {
3937   DMLabel        label;
3938   PetscInt      *perm;
3939   PetscInt       dim, depth = -1, eStart = -1, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3940   PetscBool      vertexchart;
3941   PetscErrorCode ierr;
3942 
3943   PetscFunctionBegin;
3944   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3945   if (dim < 1) PetscFunctionReturn(0);
3946   if (point < 0) {
3947     PetscInt sStart,sEnd;
3948 
3949     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
3950     point = sEnd-sStart ? sStart : point;
3951   }
3952   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3953   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
3954   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
3955   if (depth == 1) {eStart = point;}
3956   else if  (depth == dim) {
3957     const PetscInt *cone;
3958 
3959     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3960     if (dim == 2) eStart = cone[0];
3961     else if (dim == 3) {
3962       const PetscInt *cone2;
3963       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3964       eStart = cone2[0];
3965     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3966   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3967   {                             /* Determine whether the chart covers all points or just vertices. */
3968     PetscInt pStart,pEnd,cStart,cEnd;
3969     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3970     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3971     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3972     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3973   }
3974   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3975   for (f = 0; f < Nf; ++f) {
3976     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3977     size += PetscPowInt(k+1, dim)*Nc;
3978   }
3979   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3980   for (f = 0; f < Nf; ++f) {
3981     switch (dim) {
3982     case 1:
3983       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3984       /*
3985         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3986         We want              [ vtx0; edge of length k-1; vtx1 ]
3987       */
3988       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3989       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3990       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3991       foffset = offset;
3992       break;
3993     case 2:
3994       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3995       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3996       /* The SEM order is
3997 
3998          v_lb, {e_b}, v_rb,
3999          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
4000          v_lt, reverse {e_t}, v_rt
4001       */
4002       {
4003         const PetscInt of   = 0;
4004         const PetscInt oeb  = of   + PetscSqr(k-1);
4005         const PetscInt oer  = oeb  + (k-1);
4006         const PetscInt oet  = oer  + (k-1);
4007         const PetscInt oel  = oet  + (k-1);
4008         const PetscInt ovlb = oel  + (k-1);
4009         const PetscInt ovrb = ovlb + 1;
4010         const PetscInt ovrt = ovrb + 1;
4011         const PetscInt ovlt = ovrt + 1;
4012         PetscInt       o;
4013 
4014         /* bottom */
4015         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
4016         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4017         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
4018         /* middle */
4019         for (i = 0; i < k-1; ++i) {
4020           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
4021           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4022           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
4023         }
4024         /* top */
4025         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
4026         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4027         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
4028         foffset = offset;
4029       }
4030       break;
4031     case 3:
4032       /* The original hex closure is
4033 
4034          {c,
4035           f_b, f_t, f_f, f_b, f_r, f_l,
4036           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
4037           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
4038       */
4039       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4040       /* The SEM order is
4041          Bottom Slice
4042          v_blf, {e^{(k-1)-n}_bf}, v_brf,
4043          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
4044          v_blb, {e_bb}, v_brb,
4045 
4046          Middle Slice (j)
4047          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
4048          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
4049          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
4050 
4051          Top Slice
4052          v_tlf, {e_tf}, v_trf,
4053          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
4054          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
4055       */
4056       {
4057         const PetscInt oc    = 0;
4058         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4059         const PetscInt oft   = ofb   + PetscSqr(k-1);
4060         const PetscInt off   = oft   + PetscSqr(k-1);
4061         const PetscInt ofk   = off   + PetscSqr(k-1);
4062         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4063         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4064         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4065         const PetscInt oebb  = oebl  + (k-1);
4066         const PetscInt oebr  = oebb  + (k-1);
4067         const PetscInt oebf  = oebr  + (k-1);
4068         const PetscInt oetf  = oebf  + (k-1);
4069         const PetscInt oetr  = oetf  + (k-1);
4070         const PetscInt oetb  = oetr  + (k-1);
4071         const PetscInt oetl  = oetb  + (k-1);
4072         const PetscInt oerf  = oetl  + (k-1);
4073         const PetscInt oelf  = oerf  + (k-1);
4074         const PetscInt oelb  = oelf  + (k-1);
4075         const PetscInt oerb  = oelb  + (k-1);
4076         const PetscInt ovblf = oerb  + (k-1);
4077         const PetscInt ovblb = ovblf + 1;
4078         const PetscInt ovbrb = ovblb + 1;
4079         const PetscInt ovbrf = ovbrb + 1;
4080         const PetscInt ovtlf = ovbrf + 1;
4081         const PetscInt ovtrf = ovtlf + 1;
4082         const PetscInt ovtrb = ovtrf + 1;
4083         const PetscInt ovtlb = ovtrb + 1;
4084         PetscInt       o, n;
4085 
4086         /* Bottom Slice */
4087         /*   bottom */
4088         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4089         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4090         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4091         /*   middle */
4092         for (i = 0; i < k-1; ++i) {
4093           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4094           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4095           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4096         }
4097         /*   top */
4098         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4099         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4100         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4101 
4102         /* Middle Slice */
4103         for (j = 0; j < k-1; ++j) {
4104           /*   bottom */
4105           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4106           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4107           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4108           /*   middle */
4109           for (i = 0; i < k-1; ++i) {
4110             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4111             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4112             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4113           }
4114           /*   top */
4115           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4116           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4117           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4118         }
4119 
4120         /* Top Slice */
4121         /*   bottom */
4122         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4123         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4124         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4125         /*   middle */
4126         for (i = 0; i < k-1; ++i) {
4127           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4128           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4129           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4130         }
4131         /*   top */
4132         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4133         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4134         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4135 
4136         foffset = offset;
4137       }
4138       break;
4139     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4140     }
4141   }
4142   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4143   /* Check permutation */
4144   {
4145     PetscInt *check;
4146 
4147     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4148     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4149     for (i = 0; i < size; ++i) check[perm[i]] = i;
4150     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4151     ierr = PetscFree(check);CHKERRQ(ierr);
4152   }
4153   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4154   PetscFunctionReturn(0);
4155 }
4156 
4157 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4158 {
4159   PetscDS        prob;
4160   PetscInt       depth, Nf, h;
4161   DMLabel        label;
4162   PetscErrorCode ierr;
4163 
4164   PetscFunctionBeginHot;
4165   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4166   Nf      = prob->Nf;
4167   label   = dm->depthLabel;
4168   *dspace = NULL;
4169   if (field < Nf) {
4170     PetscObject disc = prob->disc[field];
4171 
4172     if (disc->classid == PETSCFE_CLASSID) {
4173       PetscDualSpace dsp;
4174 
4175       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4176       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4177       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4178       h    = depth - 1 - h;
4179       if (h) {
4180         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4181       } else {
4182         *dspace = dsp;
4183       }
4184     }
4185   }
4186   PetscFunctionReturn(0);
4187 }
4188 
4189 
4190 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4191 {
4192   PetscScalar    *array, *vArray;
4193   const PetscInt *cone, *coneO;
4194   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4195   PetscErrorCode  ierr;
4196 
4197   PetscFunctionBeginHot;
4198   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4199   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4200   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4201   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4202   if (!values || !*values) {
4203     if ((point >= pStart) && (point < pEnd)) {
4204       PetscInt dof;
4205 
4206       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4207       size += dof;
4208     }
4209     for (p = 0; p < numPoints; ++p) {
4210       const PetscInt cp = cone[p];
4211       PetscInt       dof;
4212 
4213       if ((cp < pStart) || (cp >= pEnd)) continue;
4214       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4215       size += dof;
4216     }
4217     if (!values) {
4218       if (csize) *csize = size;
4219       PetscFunctionReturn(0);
4220     }
4221     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4222   } else {
4223     array = *values;
4224   }
4225   size = 0;
4226   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4227   if ((point >= pStart) && (point < pEnd)) {
4228     PetscInt     dof, off, d;
4229     PetscScalar *varr;
4230 
4231     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4232     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4233     varr = &vArray[off];
4234     for (d = 0; d < dof; ++d, ++offset) {
4235       array[offset] = varr[d];
4236     }
4237     size += dof;
4238   }
4239   for (p = 0; p < numPoints; ++p) {
4240     const PetscInt cp = cone[p];
4241     PetscInt       o  = coneO[p];
4242     PetscInt       dof, off, d;
4243     PetscScalar   *varr;
4244 
4245     if ((cp < pStart) || (cp >= pEnd)) continue;
4246     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4247     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4248     varr = &vArray[off];
4249     if (o >= 0) {
4250       for (d = 0; d < dof; ++d, ++offset) {
4251         array[offset] = varr[d];
4252       }
4253     } else {
4254       for (d = dof-1; d >= 0; --d, ++offset) {
4255         array[offset] = varr[d];
4256       }
4257     }
4258     size += dof;
4259   }
4260   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4261   if (!*values) {
4262     if (csize) *csize = size;
4263     *values = array;
4264   } else {
4265     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4266     *csize = size;
4267   }
4268   PetscFunctionReturn(0);
4269 }
4270 
4271 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4272 {
4273   const PetscInt *cla;
4274   PetscInt       np, *pts = NULL;
4275   PetscErrorCode ierr;
4276 
4277   PetscFunctionBeginHot;
4278   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4279   if (!*clPoints) {
4280     PetscInt pStart, pEnd, p, q;
4281 
4282     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4283     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4284     /* Compress out points not in the section */
4285     for (p = 0, q = 0; p < np; p++) {
4286       PetscInt r = pts[2*p];
4287       if ((r >= pStart) && (r < pEnd)) {
4288         pts[q*2]   = r;
4289         pts[q*2+1] = pts[2*p+1];
4290         ++q;
4291       }
4292     }
4293     np = q;
4294     cla = NULL;
4295   } else {
4296     PetscInt dof, off;
4297 
4298     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4299     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4300     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4301     np   = dof/2;
4302     pts  = (PetscInt *) &cla[off];
4303   }
4304   *numPoints = np;
4305   *points    = pts;
4306   *clp       = cla;
4307 
4308   PetscFunctionReturn(0);
4309 }
4310 
4311 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4312 {
4313   PetscErrorCode ierr;
4314 
4315   PetscFunctionBeginHot;
4316   if (!*clPoints) {
4317     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4318   } else {
4319     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4320   }
4321   *numPoints = 0;
4322   *points    = NULL;
4323   *clSec     = NULL;
4324   *clPoints  = NULL;
4325   *clp       = NULL;
4326   PetscFunctionReturn(0);
4327 }
4328 
4329 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4330 {
4331   PetscInt          offset = 0, p;
4332   const PetscInt    **perms = NULL;
4333   const PetscScalar **flips = NULL;
4334   PetscErrorCode    ierr;
4335 
4336   PetscFunctionBeginHot;
4337   *size = 0;
4338   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4339   for (p = 0; p < numPoints; p++) {
4340     const PetscInt    point = points[2*p];
4341     const PetscInt    *perm = perms ? perms[p] : NULL;
4342     const PetscScalar *flip = flips ? flips[p] : NULL;
4343     PetscInt          dof, off, d;
4344     const PetscScalar *varr;
4345 
4346     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4347     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4348     varr = &vArray[off];
4349     if (clperm) {
4350       if (perm) {
4351         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4352       } else {
4353         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4354       }
4355       if (flip) {
4356         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4357       }
4358     } else {
4359       if (perm) {
4360         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4361       } else {
4362         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4363       }
4364       if (flip) {
4365         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4366       }
4367     }
4368     offset += dof;
4369   }
4370   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4371   *size = offset;
4372   PetscFunctionReturn(0);
4373 }
4374 
4375 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4376 {
4377   PetscInt          offset = 0, f;
4378   PetscErrorCode    ierr;
4379 
4380   PetscFunctionBeginHot;
4381   *size = 0;
4382   for (f = 0; f < numFields; ++f) {
4383     PetscInt          p;
4384     const PetscInt    **perms = NULL;
4385     const PetscScalar **flips = NULL;
4386 
4387     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4388     for (p = 0; p < numPoints; p++) {
4389       const PetscInt    point = points[2*p];
4390       PetscInt          fdof, foff, b;
4391       const PetscScalar *varr;
4392       const PetscInt    *perm = perms ? perms[p] : NULL;
4393       const PetscScalar *flip = flips ? flips[p] : NULL;
4394 
4395       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4396       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4397       varr = &vArray[foff];
4398       if (clperm) {
4399         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4400         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4401         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4402       } else {
4403         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4404         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4405         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4406       }
4407       offset += fdof;
4408     }
4409     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4410   }
4411   *size = offset;
4412   PetscFunctionReturn(0);
4413 }
4414 
4415 /*@C
4416   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4417 
4418   Not collective
4419 
4420   Input Parameters:
4421 + dm - The DM
4422 . section - The section describing the layout in v, or NULL to use the default section
4423 . v - The local vector
4424 . point - The point in the DM
4425 . csize - The size of the input values array, or NULL
4426 - values - An array to use for the values, or NULL to have it allocated automatically
4427 
4428   Output Parameters:
4429 + csize - The number of values in the closure
4430 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4431 
4432 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4433 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4434 $ assembly function, and a user may already have allocated storage for this operation.
4435 $
4436 $ A typical use could be
4437 $
4438 $  values = NULL;
4439 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4440 $  for (cl = 0; cl < clSize; ++cl) {
4441 $    <Compute on closure>
4442 $  }
4443 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4444 $
4445 $ or
4446 $
4447 $  PetscMalloc1(clMaxSize, &values);
4448 $  for (p = pStart; p < pEnd; ++p) {
4449 $    clSize = clMaxSize;
4450 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4451 $    for (cl = 0; cl < clSize; ++cl) {
4452 $      <Compute on closure>
4453 $    }
4454 $  }
4455 $  PetscFree(values);
4456 
4457   Fortran Notes:
4458   Since it returns an array, this routine is only available in Fortran 90, and you must
4459   include petsc.h90 in your code.
4460 
4461   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4462 
4463   Level: intermediate
4464 
4465 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4466 @*/
4467 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4468 {
4469   PetscSection       clSection;
4470   IS                 clPoints;
4471   PetscScalar       *array;
4472   const PetscScalar *vArray;
4473   PetscInt          *points = NULL;
4474   const PetscInt    *clp, *perm;
4475   PetscInt           depth, numFields, numPoints, size;
4476   PetscErrorCode     ierr;
4477 
4478   PetscFunctionBeginHot;
4479   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4480   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4481   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4482   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4483   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4484   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4485   if (depth == 1 && numFields < 2) {
4486     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4487     PetscFunctionReturn(0);
4488   }
4489   /* Get points */
4490   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4491   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4492   /* Get array */
4493   if (!values || !*values) {
4494     PetscInt asize = 0, dof, p;
4495 
4496     for (p = 0; p < numPoints*2; p += 2) {
4497       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4498       asize += dof;
4499     }
4500     if (!values) {
4501       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4502       if (csize) *csize = asize;
4503       PetscFunctionReturn(0);
4504     }
4505     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4506   } else {
4507     array = *values;
4508   }
4509   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4510   /* Get values */
4511   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4512   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4513   /* Cleanup points */
4514   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4515   /* Cleanup array */
4516   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4517   if (!*values) {
4518     if (csize) *csize = size;
4519     *values = array;
4520   } else {
4521     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4522     *csize = size;
4523   }
4524   PetscFunctionReturn(0);
4525 }
4526 
4527 /*@C
4528   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4529 
4530   Not collective
4531 
4532   Input Parameters:
4533 + dm - The DM
4534 . section - The section describing the layout in v, or NULL to use the default section
4535 . v - The local vector
4536 . point - The point in the DM
4537 . csize - The number of values in the closure, or NULL
4538 - values - The array of values, which is a borrowed array and should not be freed
4539 
4540   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4541 
4542   Fortran Notes:
4543   Since it returns an array, this routine is only available in Fortran 90, and you must
4544   include petsc.h90 in your code.
4545 
4546   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4547 
4548   Level: intermediate
4549 
4550 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4551 @*/
4552 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4553 {
4554   PetscInt       size = 0;
4555   PetscErrorCode ierr;
4556 
4557   PetscFunctionBegin;
4558   /* Should work without recalculating size */
4559   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4560   *values = NULL;
4561   PetscFunctionReturn(0);
4562 }
4563 
4564 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4565 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4566 
4567 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4568 {
4569   PetscInt        cdof;   /* The number of constraints on this point */
4570   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4571   PetscScalar    *a;
4572   PetscInt        off, cind = 0, k;
4573   PetscErrorCode  ierr;
4574 
4575   PetscFunctionBegin;
4576   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4577   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4578   a    = &array[off];
4579   if (!cdof || setBC) {
4580     if (clperm) {
4581       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4582       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4583     } else {
4584       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4585       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4586     }
4587   } else {
4588     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4589     if (clperm) {
4590       if (perm) {for (k = 0; k < dof; ++k) {
4591           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4592           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4593         }
4594       } else {
4595         for (k = 0; k < dof; ++k) {
4596           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4597           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4598         }
4599       }
4600     } else {
4601       if (perm) {
4602         for (k = 0; k < dof; ++k) {
4603           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4604           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4605         }
4606       } else {
4607         for (k = 0; k < dof; ++k) {
4608           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4609           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4610         }
4611       }
4612     }
4613   }
4614   PetscFunctionReturn(0);
4615 }
4616 
4617 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4618 {
4619   PetscInt        cdof;   /* The number of constraints on this point */
4620   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4621   PetscScalar    *a;
4622   PetscInt        off, cind = 0, k;
4623   PetscErrorCode  ierr;
4624 
4625   PetscFunctionBegin;
4626   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4627   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4628   a    = &array[off];
4629   if (cdof) {
4630     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4631     if (clperm) {
4632       if (perm) {
4633         for (k = 0; k < dof; ++k) {
4634           if ((cind < cdof) && (k == cdofs[cind])) {
4635             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4636             cind++;
4637           }
4638         }
4639       } else {
4640         for (k = 0; k < dof; ++k) {
4641           if ((cind < cdof) && (k == cdofs[cind])) {
4642             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4643             cind++;
4644           }
4645         }
4646       }
4647     } else {
4648       if (perm) {
4649         for (k = 0; k < dof; ++k) {
4650           if ((cind < cdof) && (k == cdofs[cind])) {
4651             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4652             cind++;
4653           }
4654         }
4655       } else {
4656         for (k = 0; k < dof; ++k) {
4657           if ((cind < cdof) && (k == cdofs[cind])) {
4658             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4659             cind++;
4660           }
4661         }
4662       }
4663     }
4664   }
4665   PetscFunctionReturn(0);
4666 }
4667 
4668 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4669 {
4670   PetscScalar    *a;
4671   PetscInt        fdof, foff, fcdof, foffset = *offset;
4672   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4673   PetscInt        cind = 0, b;
4674   PetscErrorCode  ierr;
4675 
4676   PetscFunctionBegin;
4677   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4678   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4679   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4680   a    = &array[foff];
4681   if (!fcdof || setBC) {
4682     if (clperm) {
4683       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4684       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4685     } else {
4686       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4687       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4688     }
4689   } else {
4690     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4691     if (clperm) {
4692       if (perm) {
4693         for (b = 0; b < fdof; b++) {
4694           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4695           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4696         }
4697       } else {
4698         for (b = 0; b < fdof; b++) {
4699           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4700           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4701         }
4702       }
4703     } else {
4704       if (perm) {
4705         for (b = 0; b < fdof; b++) {
4706           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4707           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4708         }
4709       } else {
4710         for (b = 0; b < fdof; b++) {
4711           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4712           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4713         }
4714       }
4715     }
4716   }
4717   *offset += fdof;
4718   PetscFunctionReturn(0);
4719 }
4720 
4721 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4722 {
4723   PetscScalar    *a;
4724   PetscInt        fdof, foff, fcdof, foffset = *offset;
4725   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4726   PetscInt        cind = 0, ncind = 0, b;
4727   PetscBool       ncSet, fcSet;
4728   PetscErrorCode  ierr;
4729 
4730   PetscFunctionBegin;
4731   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4732   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4733   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4734   a    = &array[foff];
4735   if (fcdof) {
4736     /* We just override fcdof and fcdofs with Ncc and comps */
4737     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4738     if (clperm) {
4739       if (perm) {
4740         if (comps) {
4741           for (b = 0; b < fdof; b++) {
4742             ncSet = fcSet = PETSC_FALSE;
4743             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4744             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4745             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4746           }
4747         } else {
4748           for (b = 0; b < fdof; b++) {
4749             if ((cind < fcdof) && (b == fcdofs[cind])) {
4750               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4751               ++cind;
4752             }
4753           }
4754         }
4755       } else {
4756         if (comps) {
4757           for (b = 0; b < fdof; b++) {
4758             ncSet = fcSet = PETSC_FALSE;
4759             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4760             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4761             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4762           }
4763         } else {
4764           for (b = 0; b < fdof; b++) {
4765             if ((cind < fcdof) && (b == fcdofs[cind])) {
4766               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4767               ++cind;
4768             }
4769           }
4770         }
4771       }
4772     } else {
4773       if (perm) {
4774         if (comps) {
4775           for (b = 0; b < fdof; b++) {
4776             ncSet = fcSet = PETSC_FALSE;
4777             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4778             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4779             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4780           }
4781         } else {
4782           for (b = 0; b < fdof; b++) {
4783             if ((cind < fcdof) && (b == fcdofs[cind])) {
4784               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4785               ++cind;
4786             }
4787           }
4788         }
4789       } else {
4790         if (comps) {
4791           for (b = 0; b < fdof; b++) {
4792             ncSet = fcSet = PETSC_FALSE;
4793             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4794             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4795             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4796           }
4797         } else {
4798           for (b = 0; b < fdof; b++) {
4799             if ((cind < fcdof) && (b == fcdofs[cind])) {
4800               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4801               ++cind;
4802             }
4803           }
4804         }
4805       }
4806     }
4807   }
4808   *offset += fdof;
4809   PetscFunctionReturn(0);
4810 }
4811 
4812 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4813 {
4814   PetscScalar    *array;
4815   const PetscInt *cone, *coneO;
4816   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4817   PetscErrorCode  ierr;
4818 
4819   PetscFunctionBeginHot;
4820   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4821   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4822   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4823   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4824   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4825   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4826     const PetscInt cp = !p ? point : cone[p-1];
4827     const PetscInt o  = !p ? 0     : coneO[p-1];
4828 
4829     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4830     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4831     /* ADD_VALUES */
4832     {
4833       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4834       PetscScalar    *a;
4835       PetscInt        cdof, coff, cind = 0, k;
4836 
4837       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4838       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4839       a    = &array[coff];
4840       if (!cdof) {
4841         if (o >= 0) {
4842           for (k = 0; k < dof; ++k) {
4843             a[k] += values[off+k];
4844           }
4845         } else {
4846           for (k = 0; k < dof; ++k) {
4847             a[k] += values[off+dof-k-1];
4848           }
4849         }
4850       } else {
4851         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4852         if (o >= 0) {
4853           for (k = 0; k < dof; ++k) {
4854             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4855             a[k] += values[off+k];
4856           }
4857         } else {
4858           for (k = 0; k < dof; ++k) {
4859             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4860             a[k] += values[off+dof-k-1];
4861           }
4862         }
4863       }
4864     }
4865   }
4866   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4867   PetscFunctionReturn(0);
4868 }
4869 
4870 /*@C
4871   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4872 
4873   Not collective
4874 
4875   Input Parameters:
4876 + dm - The DM
4877 . section - The section describing the layout in v, or NULL to use the default section
4878 . v - The local vector
4879 . point - The point in the DM
4880 . values - The array of values
4881 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4882          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4883 
4884   Fortran Notes:
4885   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4886 
4887   Level: intermediate
4888 
4889 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4890 @*/
4891 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4892 {
4893   PetscSection    clSection;
4894   IS              clPoints;
4895   PetscScalar    *array;
4896   PetscInt       *points = NULL;
4897   const PetscInt *clp, *clperm;
4898   PetscInt        depth, numFields, numPoints, p;
4899   PetscErrorCode  ierr;
4900 
4901   PetscFunctionBeginHot;
4902   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4903   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4904   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4905   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4906   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4907   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4908   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4909     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4910     PetscFunctionReturn(0);
4911   }
4912   /* Get points */
4913   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4914   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4915   /* Get array */
4916   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4917   /* Get values */
4918   if (numFields > 0) {
4919     PetscInt offset = 0, f;
4920     for (f = 0; f < numFields; ++f) {
4921       const PetscInt    **perms = NULL;
4922       const PetscScalar **flips = NULL;
4923 
4924       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4925       switch (mode) {
4926       case INSERT_VALUES:
4927         for (p = 0; p < numPoints; p++) {
4928           const PetscInt    point = points[2*p];
4929           const PetscInt    *perm = perms ? perms[p] : NULL;
4930           const PetscScalar *flip = flips ? flips[p] : NULL;
4931           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4932         } break;
4933       case INSERT_ALL_VALUES:
4934         for (p = 0; p < numPoints; p++) {
4935           const PetscInt    point = points[2*p];
4936           const PetscInt    *perm = perms ? perms[p] : NULL;
4937           const PetscScalar *flip = flips ? flips[p] : NULL;
4938           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4939         } break;
4940       case INSERT_BC_VALUES:
4941         for (p = 0; p < numPoints; p++) {
4942           const PetscInt    point = points[2*p];
4943           const PetscInt    *perm = perms ? perms[p] : NULL;
4944           const PetscScalar *flip = flips ? flips[p] : NULL;
4945           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4946         } break;
4947       case ADD_VALUES:
4948         for (p = 0; p < numPoints; p++) {
4949           const PetscInt    point = points[2*p];
4950           const PetscInt    *perm = perms ? perms[p] : NULL;
4951           const PetscScalar *flip = flips ? flips[p] : NULL;
4952           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4953         } break;
4954       case ADD_ALL_VALUES:
4955         for (p = 0; p < numPoints; p++) {
4956           const PetscInt    point = points[2*p];
4957           const PetscInt    *perm = perms ? perms[p] : NULL;
4958           const PetscScalar *flip = flips ? flips[p] : NULL;
4959           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4960         } break;
4961       case ADD_BC_VALUES:
4962         for (p = 0; p < numPoints; p++) {
4963           const PetscInt    point = points[2*p];
4964           const PetscInt    *perm = perms ? perms[p] : NULL;
4965           const PetscScalar *flip = flips ? flips[p] : NULL;
4966           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4967         } break;
4968       default:
4969         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4970       }
4971       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4972     }
4973   } else {
4974     PetscInt dof, off;
4975     const PetscInt    **perms = NULL;
4976     const PetscScalar **flips = NULL;
4977 
4978     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4979     switch (mode) {
4980     case INSERT_VALUES:
4981       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4982         const PetscInt    point = points[2*p];
4983         const PetscInt    *perm = perms ? perms[p] : NULL;
4984         const PetscScalar *flip = flips ? flips[p] : NULL;
4985         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4986         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4987       } break;
4988     case INSERT_ALL_VALUES:
4989       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4990         const PetscInt    point = points[2*p];
4991         const PetscInt    *perm = perms ? perms[p] : NULL;
4992         const PetscScalar *flip = flips ? flips[p] : NULL;
4993         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4994         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4995       } break;
4996     case INSERT_BC_VALUES:
4997       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4998         const PetscInt    point = points[2*p];
4999         const PetscInt    *perm = perms ? perms[p] : NULL;
5000         const PetscScalar *flip = flips ? flips[p] : NULL;
5001         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5002         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
5003       } break;
5004     case ADD_VALUES:
5005       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5006         const PetscInt    point = points[2*p];
5007         const PetscInt    *perm = perms ? perms[p] : NULL;
5008         const PetscScalar *flip = flips ? flips[p] : NULL;
5009         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5010         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
5011       } break;
5012     case ADD_ALL_VALUES:
5013       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5014         const PetscInt    point = points[2*p];
5015         const PetscInt    *perm = perms ? perms[p] : NULL;
5016         const PetscScalar *flip = flips ? flips[p] : NULL;
5017         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5018         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
5019       } break;
5020     case ADD_BC_VALUES:
5021       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
5022         const PetscInt    point = points[2*p];
5023         const PetscInt    *perm = perms ? perms[p] : NULL;
5024         const PetscScalar *flip = flips ? flips[p] : NULL;
5025         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5026         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
5027       } break;
5028     default:
5029       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5030     }
5031     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5032   }
5033   /* Cleanup points */
5034   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5035   /* Cleanup array */
5036   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5037   PetscFunctionReturn(0);
5038 }
5039 
5040 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
5041 {
5042   PetscSection      clSection;
5043   IS                clPoints;
5044   PetscScalar       *array;
5045   PetscInt          *points = NULL;
5046   const PetscInt    *clp, *clperm;
5047   PetscInt          numFields, numPoints, p;
5048   PetscInt          offset = 0, f;
5049   PetscErrorCode    ierr;
5050 
5051   PetscFunctionBeginHot;
5052   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5053   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5054   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5055   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5056   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5057   /* Get points */
5058   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5059   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5060   /* Get array */
5061   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5062   /* Get values */
5063   for (f = 0; f < numFields; ++f) {
5064     const PetscInt    **perms = NULL;
5065     const PetscScalar **flips = NULL;
5066 
5067     if (!fieldActive[f]) {
5068       for (p = 0; p < numPoints*2; p += 2) {
5069         PetscInt fdof;
5070         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5071         offset += fdof;
5072       }
5073       continue;
5074     }
5075     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5076     switch (mode) {
5077     case INSERT_VALUES:
5078       for (p = 0; p < numPoints; p++) {
5079         const PetscInt    point = points[2*p];
5080         const PetscInt    *perm = perms ? perms[p] : NULL;
5081         const PetscScalar *flip = flips ? flips[p] : NULL;
5082         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5083       } break;
5084     case INSERT_ALL_VALUES:
5085       for (p = 0; p < numPoints; p++) {
5086         const PetscInt    point = points[2*p];
5087         const PetscInt    *perm = perms ? perms[p] : NULL;
5088         const PetscScalar *flip = flips ? flips[p] : NULL;
5089         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5090         } break;
5091     case INSERT_BC_VALUES:
5092       for (p = 0; p < numPoints; p++) {
5093         const PetscInt    point = points[2*p];
5094         const PetscInt    *perm = perms ? perms[p] : NULL;
5095         const PetscScalar *flip = flips ? flips[p] : NULL;
5096         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
5097       } break;
5098     case ADD_VALUES:
5099       for (p = 0; p < numPoints; p++) {
5100         const PetscInt    point = points[2*p];
5101         const PetscInt    *perm = perms ? perms[p] : NULL;
5102         const PetscScalar *flip = flips ? flips[p] : NULL;
5103         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5104       } break;
5105     case ADD_ALL_VALUES:
5106       for (p = 0; p < numPoints; p++) {
5107         const PetscInt    point = points[2*p];
5108         const PetscInt    *perm = perms ? perms[p] : NULL;
5109         const PetscScalar *flip = flips ? flips[p] : NULL;
5110         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5111       } break;
5112     default:
5113       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5114     }
5115     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5116   }
5117   /* Cleanup points */
5118   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5119   /* Cleanup array */
5120   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5121   PetscFunctionReturn(0);
5122 }
5123 
5124 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5125 {
5126   PetscMPIInt    rank;
5127   PetscInt       i, j;
5128   PetscErrorCode ierr;
5129 
5130   PetscFunctionBegin;
5131   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5132   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5133   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5134   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5135   numCIndices = numCIndices ? numCIndices : numRIndices;
5136   for (i = 0; i < numRIndices; i++) {
5137     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5138     for (j = 0; j < numCIndices; j++) {
5139 #if defined(PETSC_USE_COMPLEX)
5140       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5141 #else
5142       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5143 #endif
5144     }
5145     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5146   }
5147   PetscFunctionReturn(0);
5148 }
5149 
5150 /*
5151   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5152 
5153   Input Parameters:
5154 + section - The section for this data layout
5155 . point   - The point contributing dofs with these indices
5156 . off     - The global offset of this point
5157 . loff    - The local offset of each field
5158 . setBC   - The flag determining whether to include indices of bounsary values
5159 . perm    - A permutation of the dofs on this point, or NULL
5160 - indperm - A permutation of the entire indices array, or NULL
5161 
5162   Output Parameter:
5163 . indices - Indices for dofs on this point
5164 
5165   Level: developer
5166 
5167   Note: The indices could be local or global, depending on the value of 'off'.
5168 */
5169 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5170 {
5171   PetscInt        dof;   /* The number of unknowns on this point */
5172   PetscInt        cdof;  /* The number of constraints on this point */
5173   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5174   PetscInt        cind = 0, k;
5175   PetscErrorCode  ierr;
5176 
5177   PetscFunctionBegin;
5178   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5179   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5180   if (!cdof || setBC) {
5181     for (k = 0; k < dof; ++k) {
5182       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5183       const PetscInt ind    = indperm ? indperm[preind] : preind;
5184 
5185       indices[ind] = off + k;
5186     }
5187   } else {
5188     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5189     for (k = 0; k < dof; ++k) {
5190       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5191       const PetscInt ind    = indperm ? indperm[preind] : preind;
5192 
5193       if ((cind < cdof) && (k == cdofs[cind])) {
5194         /* Insert check for returning constrained indices */
5195         indices[ind] = -(off+k+1);
5196         ++cind;
5197       } else {
5198         indices[ind] = off+k-cind;
5199       }
5200     }
5201   }
5202   *loff += dof;
5203   PetscFunctionReturn(0);
5204 }
5205 
5206 /*
5207   This version only believes the point offset from the globalSection
5208 
5209  . off - The global offset of this point
5210 */
5211 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5212 {
5213   PetscInt       numFields, foff, f;
5214   PetscErrorCode ierr;
5215 
5216   PetscFunctionBegin;
5217   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5218   for (f = 0, foff = 0; f < numFields; ++f) {
5219     PetscInt        fdof, cfdof;
5220     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5221     PetscInt        cind = 0, b;
5222     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5223 
5224     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5225     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5226     if (!cfdof || setBC) {
5227       for (b = 0; b < fdof; ++b) {
5228         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5229         const PetscInt ind    = indperm ? indperm[preind] : preind;
5230 
5231         indices[ind] = off+foff+b;
5232       }
5233     } else {
5234       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5235       for (b = 0; b < fdof; ++b) {
5236         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5237         const PetscInt ind    = indperm ? indperm[preind] : preind;
5238 
5239         if ((cind < cfdof) && (b == fcdofs[cind])) {
5240           indices[ind] = -(off+foff+b+1);
5241           ++cind;
5242         } else {
5243           indices[ind] = off+foff+b-cind;
5244         }
5245       }
5246     }
5247     foff     += (setBC ? fdof : (fdof - cfdof));
5248     foffs[f] += fdof;
5249   }
5250   PetscFunctionReturn(0);
5251 }
5252 
5253 /*
5254   This version believes the globalSection offsets for each field, rather than just the point offset
5255 
5256  . foffs - The offset into 'indices' for each field, since it is segregated by field
5257 */
5258 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5259 {
5260   PetscInt       numFields, foff, f;
5261   PetscErrorCode ierr;
5262 
5263   PetscFunctionBegin;
5264   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5265   for (f = 0; f < numFields; ++f) {
5266     PetscInt        fdof, cfdof;
5267     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5268     PetscInt        cind = 0, b;
5269     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5270 
5271     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5272     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5273     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5274     if (!cfdof || setBC) {
5275       for (b = 0; b < fdof; ++b) {
5276         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5277         const PetscInt ind    = indperm ? indperm[preind] : preind;
5278 
5279         indices[ind] = foff+b;
5280       }
5281     } else {
5282       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5283       for (b = 0; b < fdof; ++b) {
5284         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5285         const PetscInt ind    = indperm ? indperm[preind] : preind;
5286 
5287         if ((cind < cfdof) && (b == fcdofs[cind])) {
5288           indices[ind] = -(foff+b+1);
5289           ++cind;
5290         } else {
5291           indices[ind] = foff+b-cind;
5292         }
5293       }
5294     }
5295     foffs[f] += fdof;
5296   }
5297   PetscFunctionReturn(0);
5298 }
5299 
5300 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5301 {
5302   Mat             cMat;
5303   PetscSection    aSec, cSec;
5304   IS              aIS;
5305   PetscInt        aStart = -1, aEnd = -1;
5306   const PetscInt  *anchors;
5307   PetscInt        numFields, f, p, q, newP = 0;
5308   PetscInt        newNumPoints = 0, newNumIndices = 0;
5309   PetscInt        *newPoints, *indices, *newIndices;
5310   PetscInt        maxAnchor, maxDof;
5311   PetscInt        newOffsets[32];
5312   PetscInt        *pointMatOffsets[32];
5313   PetscInt        *newPointOffsets[32];
5314   PetscScalar     *pointMat[32];
5315   PetscScalar     *newValues=NULL,*tmpValues;
5316   PetscBool       anyConstrained = PETSC_FALSE;
5317   PetscErrorCode  ierr;
5318 
5319   PetscFunctionBegin;
5320   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5321   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5322   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5323 
5324   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5325   /* if there are point-to-point constraints */
5326   if (aSec) {
5327     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5328     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5329     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5330     /* figure out how many points are going to be in the new element matrix
5331      * (we allow double counting, because it's all just going to be summed
5332      * into the global matrix anyway) */
5333     for (p = 0; p < 2*numPoints; p+=2) {
5334       PetscInt b    = points[p];
5335       PetscInt bDof = 0, bSecDof;
5336 
5337       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5338       if (!bSecDof) {
5339         continue;
5340       }
5341       if (b >= aStart && b < aEnd) {
5342         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5343       }
5344       if (bDof) {
5345         /* this point is constrained */
5346         /* it is going to be replaced by its anchors */
5347         PetscInt bOff, q;
5348 
5349         anyConstrained = PETSC_TRUE;
5350         newNumPoints  += bDof;
5351         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5352         for (q = 0; q < bDof; q++) {
5353           PetscInt a = anchors[bOff + q];
5354           PetscInt aDof;
5355 
5356           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5357           newNumIndices += aDof;
5358           for (f = 0; f < numFields; ++f) {
5359             PetscInt fDof;
5360 
5361             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5362             newOffsets[f+1] += fDof;
5363           }
5364         }
5365       }
5366       else {
5367         /* this point is not constrained */
5368         newNumPoints++;
5369         newNumIndices += bSecDof;
5370         for (f = 0; f < numFields; ++f) {
5371           PetscInt fDof;
5372 
5373           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5374           newOffsets[f+1] += fDof;
5375         }
5376       }
5377     }
5378   }
5379   if (!anyConstrained) {
5380     if (outNumPoints)  *outNumPoints  = 0;
5381     if (outNumIndices) *outNumIndices = 0;
5382     if (outPoints)     *outPoints     = NULL;
5383     if (outValues)     *outValues     = NULL;
5384     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5385     PetscFunctionReturn(0);
5386   }
5387 
5388   if (outNumPoints)  *outNumPoints  = newNumPoints;
5389   if (outNumIndices) *outNumIndices = newNumIndices;
5390 
5391   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5392 
5393   if (!outPoints && !outValues) {
5394     if (offsets) {
5395       for (f = 0; f <= numFields; f++) {
5396         offsets[f] = newOffsets[f];
5397       }
5398     }
5399     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5400     PetscFunctionReturn(0);
5401   }
5402 
5403   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5404 
5405   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5406 
5407   /* workspaces */
5408   if (numFields) {
5409     for (f = 0; f < numFields; f++) {
5410       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5411       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5412     }
5413   }
5414   else {
5415     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5416     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5417   }
5418 
5419   /* get workspaces for the point-to-point matrices */
5420   if (numFields) {
5421     PetscInt totalOffset, totalMatOffset;
5422 
5423     for (p = 0; p < numPoints; p++) {
5424       PetscInt b    = points[2*p];
5425       PetscInt bDof = 0, bSecDof;
5426 
5427       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5428       if (!bSecDof) {
5429         for (f = 0; f < numFields; f++) {
5430           newPointOffsets[f][p + 1] = 0;
5431           pointMatOffsets[f][p + 1] = 0;
5432         }
5433         continue;
5434       }
5435       if (b >= aStart && b < aEnd) {
5436         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5437       }
5438       if (bDof) {
5439         for (f = 0; f < numFields; f++) {
5440           PetscInt fDof, q, bOff, allFDof = 0;
5441 
5442           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5443           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5444           for (q = 0; q < bDof; q++) {
5445             PetscInt a = anchors[bOff + q];
5446             PetscInt aFDof;
5447 
5448             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5449             allFDof += aFDof;
5450           }
5451           newPointOffsets[f][p+1] = allFDof;
5452           pointMatOffsets[f][p+1] = fDof * allFDof;
5453         }
5454       }
5455       else {
5456         for (f = 0; f < numFields; f++) {
5457           PetscInt fDof;
5458 
5459           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5460           newPointOffsets[f][p+1] = fDof;
5461           pointMatOffsets[f][p+1] = 0;
5462         }
5463       }
5464     }
5465     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5466       newPointOffsets[f][0] = totalOffset;
5467       pointMatOffsets[f][0] = totalMatOffset;
5468       for (p = 0; p < numPoints; p++) {
5469         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5470         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5471       }
5472       totalOffset    = newPointOffsets[f][numPoints];
5473       totalMatOffset = pointMatOffsets[f][numPoints];
5474       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5475     }
5476   }
5477   else {
5478     for (p = 0; p < numPoints; p++) {
5479       PetscInt b    = points[2*p];
5480       PetscInt bDof = 0, bSecDof;
5481 
5482       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5483       if (!bSecDof) {
5484         newPointOffsets[0][p + 1] = 0;
5485         pointMatOffsets[0][p + 1] = 0;
5486         continue;
5487       }
5488       if (b >= aStart && b < aEnd) {
5489         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5490       }
5491       if (bDof) {
5492         PetscInt bOff, q, allDof = 0;
5493 
5494         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5495         for (q = 0; q < bDof; q++) {
5496           PetscInt a = anchors[bOff + q], aDof;
5497 
5498           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5499           allDof += aDof;
5500         }
5501         newPointOffsets[0][p+1] = allDof;
5502         pointMatOffsets[0][p+1] = bSecDof * allDof;
5503       }
5504       else {
5505         newPointOffsets[0][p+1] = bSecDof;
5506         pointMatOffsets[0][p+1] = 0;
5507       }
5508     }
5509     newPointOffsets[0][0] = 0;
5510     pointMatOffsets[0][0] = 0;
5511     for (p = 0; p < numPoints; p++) {
5512       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5513       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5514     }
5515     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5516   }
5517 
5518   /* output arrays */
5519   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5520 
5521   /* get the point-to-point matrices; construct newPoints */
5522   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5523   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5524   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5525   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5526   if (numFields) {
5527     for (p = 0, newP = 0; p < numPoints; p++) {
5528       PetscInt b    = points[2*p];
5529       PetscInt o    = points[2*p+1];
5530       PetscInt bDof = 0, bSecDof;
5531 
5532       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5533       if (!bSecDof) {
5534         continue;
5535       }
5536       if (b >= aStart && b < aEnd) {
5537         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5538       }
5539       if (bDof) {
5540         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5541 
5542         fStart[0] = 0;
5543         fEnd[0]   = 0;
5544         for (f = 0; f < numFields; f++) {
5545           PetscInt fDof;
5546 
5547           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5548           fStart[f+1] = fStart[f] + fDof;
5549           fEnd[f+1]   = fStart[f+1];
5550         }
5551         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5552         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5553 
5554         fAnchorStart[0] = 0;
5555         fAnchorEnd[0]   = 0;
5556         for (f = 0; f < numFields; f++) {
5557           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5558 
5559           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5560           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5561         }
5562         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5563         for (q = 0; q < bDof; q++) {
5564           PetscInt a = anchors[bOff + q], aOff;
5565 
5566           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5567           newPoints[2*(newP + q)]     = a;
5568           newPoints[2*(newP + q) + 1] = 0;
5569           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5570           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5571         }
5572         newP += bDof;
5573 
5574         if (outValues) {
5575           /* get the point-to-point submatrix */
5576           for (f = 0; f < numFields; f++) {
5577             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5578           }
5579         }
5580       }
5581       else {
5582         newPoints[2 * newP]     = b;
5583         newPoints[2 * newP + 1] = o;
5584         newP++;
5585       }
5586     }
5587   } else {
5588     for (p = 0; p < numPoints; p++) {
5589       PetscInt b    = points[2*p];
5590       PetscInt o    = points[2*p+1];
5591       PetscInt bDof = 0, bSecDof;
5592 
5593       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5594       if (!bSecDof) {
5595         continue;
5596       }
5597       if (b >= aStart && b < aEnd) {
5598         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5599       }
5600       if (bDof) {
5601         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5602 
5603         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5604         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5605 
5606         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5607         for (q = 0; q < bDof; q++) {
5608           PetscInt a = anchors[bOff + q], aOff;
5609 
5610           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5611 
5612           newPoints[2*(newP + q)]     = a;
5613           newPoints[2*(newP + q) + 1] = 0;
5614           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5615           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5616         }
5617         newP += bDof;
5618 
5619         /* get the point-to-point submatrix */
5620         if (outValues) {
5621           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5622         }
5623       }
5624       else {
5625         newPoints[2 * newP]     = b;
5626         newPoints[2 * newP + 1] = o;
5627         newP++;
5628       }
5629     }
5630   }
5631 
5632   if (outValues) {
5633     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5634     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5635     /* multiply constraints on the right */
5636     if (numFields) {
5637       for (f = 0; f < numFields; f++) {
5638         PetscInt oldOff = offsets[f];
5639 
5640         for (p = 0; p < numPoints; p++) {
5641           PetscInt cStart = newPointOffsets[f][p];
5642           PetscInt b      = points[2 * p];
5643           PetscInt c, r, k;
5644           PetscInt dof;
5645 
5646           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5647           if (!dof) {
5648             continue;
5649           }
5650           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5651             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5652             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5653 
5654             for (r = 0; r < numIndices; r++) {
5655               for (c = 0; c < nCols; c++) {
5656                 for (k = 0; k < dof; k++) {
5657                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5658                 }
5659               }
5660             }
5661           }
5662           else {
5663             /* copy this column as is */
5664             for (r = 0; r < numIndices; r++) {
5665               for (c = 0; c < dof; c++) {
5666                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5667               }
5668             }
5669           }
5670           oldOff += dof;
5671         }
5672       }
5673     }
5674     else {
5675       PetscInt oldOff = 0;
5676       for (p = 0; p < numPoints; p++) {
5677         PetscInt cStart = newPointOffsets[0][p];
5678         PetscInt b      = points[2 * p];
5679         PetscInt c, r, k;
5680         PetscInt dof;
5681 
5682         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5683         if (!dof) {
5684           continue;
5685         }
5686         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5687           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5688           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5689 
5690           for (r = 0; r < numIndices; r++) {
5691             for (c = 0; c < nCols; c++) {
5692               for (k = 0; k < dof; k++) {
5693                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5694               }
5695             }
5696           }
5697         }
5698         else {
5699           /* copy this column as is */
5700           for (r = 0; r < numIndices; r++) {
5701             for (c = 0; c < dof; c++) {
5702               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5703             }
5704           }
5705         }
5706         oldOff += dof;
5707       }
5708     }
5709 
5710     if (multiplyLeft) {
5711       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5712       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5713       /* multiply constraints transpose on the left */
5714       if (numFields) {
5715         for (f = 0; f < numFields; f++) {
5716           PetscInt oldOff = offsets[f];
5717 
5718           for (p = 0; p < numPoints; p++) {
5719             PetscInt rStart = newPointOffsets[f][p];
5720             PetscInt b      = points[2 * p];
5721             PetscInt c, r, k;
5722             PetscInt dof;
5723 
5724             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5725             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5726               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5727               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5728 
5729               for (r = 0; r < nRows; r++) {
5730                 for (c = 0; c < newNumIndices; c++) {
5731                   for (k = 0; k < dof; k++) {
5732                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5733                   }
5734                 }
5735               }
5736             }
5737             else {
5738               /* copy this row as is */
5739               for (r = 0; r < dof; r++) {
5740                 for (c = 0; c < newNumIndices; c++) {
5741                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5742                 }
5743               }
5744             }
5745             oldOff += dof;
5746           }
5747         }
5748       }
5749       else {
5750         PetscInt oldOff = 0;
5751 
5752         for (p = 0; p < numPoints; p++) {
5753           PetscInt rStart = newPointOffsets[0][p];
5754           PetscInt b      = points[2 * p];
5755           PetscInt c, r, k;
5756           PetscInt dof;
5757 
5758           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5759           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5760             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5761             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5762 
5763             for (r = 0; r < nRows; r++) {
5764               for (c = 0; c < newNumIndices; c++) {
5765                 for (k = 0; k < dof; k++) {
5766                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5767                 }
5768               }
5769             }
5770           }
5771           else {
5772             /* copy this row as is */
5773             for (r = 0; r < dof; r++) {
5774               for (c = 0; c < newNumIndices; c++) {
5775                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5776               }
5777             }
5778           }
5779           oldOff += dof;
5780         }
5781       }
5782 
5783       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5784     }
5785     else {
5786       newValues = tmpValues;
5787     }
5788   }
5789 
5790   /* clean up */
5791   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5792   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5793 
5794   if (numFields) {
5795     for (f = 0; f < numFields; f++) {
5796       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5797       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5798       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5799     }
5800   }
5801   else {
5802     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5803     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5804     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5805   }
5806   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5807 
5808   /* output */
5809   if (outPoints) {
5810     *outPoints = newPoints;
5811   }
5812   else {
5813     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5814   }
5815   if (outValues) {
5816     *outValues = newValues;
5817   }
5818   for (f = 0; f <= numFields; f++) {
5819     offsets[f] = newOffsets[f];
5820   }
5821   PetscFunctionReturn(0);
5822 }
5823 
5824 /*@C
5825   DMPlexGetClosureIndices - Get the global indices for all local points in the closure of the given point
5826 
5827   Not collective
5828 
5829   Input Parameters:
5830 + dm - The DM
5831 . section - The section describing the local layout
5832 . globalSection - The section describing the parallel layout
5833 - point - The mesh point
5834 
5835   Output parameters:
5836 + numIndices - The number of indices
5837 . indices - The indices
5838 - outOffsets - Field offset if not NULL
5839 
5840   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5841 
5842   Level: advanced
5843 
5844 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5845 @*/
5846 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5847 {
5848   PetscSection    clSection;
5849   IS              clPoints;
5850   const PetscInt *clp, *clperm;
5851   const PetscInt  **perms[32] = {NULL};
5852   PetscInt       *points = NULL, *pointsNew;
5853   PetscInt        numPoints, numPointsNew;
5854   PetscInt        offsets[32];
5855   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5856   PetscErrorCode  ierr;
5857 
5858   PetscFunctionBegin;
5859   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5860   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5861   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5862   if (numIndices) PetscValidPointer(numIndices, 4);
5863   PetscValidPointer(indices, 5);
5864   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5865   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5866   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5867   /* Get points in closure */
5868   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5869   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5870   /* Get number of indices and indices per field */
5871   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5872     PetscInt dof, fdof;
5873 
5874     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5875     for (f = 0; f < Nf; ++f) {
5876       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5877       offsets[f+1] += fdof;
5878     }
5879     Nind += dof;
5880   }
5881   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5882   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5883   if (!Nf) offsets[1] = Nind;
5884   /* Get dual space symmetries */
5885   for (f = 0; f < PetscMax(1,Nf); f++) {
5886     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5887     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5888   }
5889   /* Correct for hanging node constraints */
5890   {
5891     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5892     if (numPointsNew) {
5893       for (f = 0; f < PetscMax(1,Nf); f++) {
5894         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5895         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5896       }
5897       for (f = 0; f < PetscMax(1,Nf); f++) {
5898         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5899         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5900       }
5901       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5902       numPoints = numPointsNew;
5903       Nind      = NindNew;
5904       points    = pointsNew;
5905     }
5906   }
5907   /* Calculate indices */
5908   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5909   if (Nf) {
5910     if (outOffsets) {
5911       PetscInt f;
5912 
5913       for (f = 0; f <= Nf; f++) {
5914         outOffsets[f] = offsets[f];
5915       }
5916     }
5917     for (p = 0; p < numPoints; p++) {
5918       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5919       ierr = DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);CHKERRQ(ierr);
5920     }
5921   } else {
5922     for (p = 0, off = 0; p < numPoints; p++) {
5923       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5924 
5925       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5926       ierr = DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);CHKERRQ(ierr);
5927     }
5928   }
5929   /* Cleanup points */
5930   for (f = 0; f < PetscMax(1,Nf); f++) {
5931     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5932     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5933   }
5934   if (numPointsNew) {
5935     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5936   } else {
5937     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5938   }
5939   if (numIndices) *numIndices = Nind;
5940   PetscFunctionReturn(0);
5941 }
5942 
5943 /*@C
5944   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5945 
5946   Not collective
5947 
5948   Input Parameters:
5949 + dm - The DM
5950 . section - The section describing the layout in v, or NULL to use the default section
5951 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5952 . point - The mesh point
5953 . numIndices - The number of indices
5954 . indices - The indices
5955 - outOffsets - Field offset if not NULL
5956 
5957   Level: advanced
5958 
5959 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5960 @*/
5961 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5962 {
5963   PetscErrorCode ierr;
5964 
5965   PetscFunctionBegin;
5966   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5967   PetscValidPointer(indices, 5);
5968   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5969   PetscFunctionReturn(0);
5970 }
5971 
5972 /*@C
5973   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5974 
5975   Not collective
5976 
5977   Input Parameters:
5978 + dm - The DM
5979 . section - The section describing the layout in v, or NULL to use the default section
5980 . globalSection - The section describing the layout in v, or NULL to use the default global section
5981 . A - The matrix
5982 . point - The point in the DM
5983 . values - The array of values
5984 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5985 
5986   Fortran Notes:
5987   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5988 
5989   Level: intermediate
5990 
5991 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5992 @*/
5993 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5994 {
5995   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5996   PetscSection        clSection;
5997   IS                  clPoints;
5998   PetscInt           *points = NULL, *newPoints;
5999   const PetscInt     *clp, *clperm;
6000   PetscInt           *indices;
6001   PetscInt            offsets[32];
6002   const PetscInt    **perms[32] = {NULL};
6003   const PetscScalar **flips[32] = {NULL};
6004   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
6005   PetscScalar        *valCopy = NULL;
6006   PetscScalar        *newValues;
6007   PetscErrorCode      ierr;
6008 
6009   PetscFunctionBegin;
6010   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6011   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
6012   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6013   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
6014   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
6015   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
6016   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6017   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6018   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
6019   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
6020   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6021   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
6022     PetscInt fdof;
6023 
6024     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6025     for (f = 0; f < numFields; ++f) {
6026       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6027       offsets[f+1] += fdof;
6028     }
6029     numIndices += dof;
6030   }
6031   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6032 
6033   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
6034   /* Get symmetries */
6035   for (f = 0; f < PetscMax(1,numFields); f++) {
6036     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6037     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6038     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
6039       PetscInt foffset = offsets[f];
6040 
6041       for (p = 0; p < numPoints; p++) {
6042         PetscInt point          = points[2*p], fdof;
6043         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
6044 
6045         if (!numFields) {
6046           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
6047         } else {
6048           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
6049         }
6050         if (flip) {
6051           PetscInt i, j, k;
6052 
6053           if (!valCopy) {
6054             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6055             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6056             values = valCopy;
6057           }
6058           for (i = 0; i < fdof; i++) {
6059             PetscScalar fval = flip[i];
6060 
6061             for (k = 0; k < numIndices; k++) {
6062               valCopy[numIndices * (foffset + i) + k] *= fval;
6063               valCopy[numIndices * k + (foffset + i)] *= fval;
6064             }
6065           }
6066         }
6067         foffset += fdof;
6068       }
6069     }
6070   }
6071   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6072   if (newNumPoints) {
6073     if (valCopy) {
6074       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6075     }
6076     for (f = 0; f < PetscMax(1,numFields); f++) {
6077       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6078       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6079     }
6080     for (f = 0; f < PetscMax(1,numFields); f++) {
6081       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6082       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6083     }
6084     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6085     numPoints  = newNumPoints;
6086     numIndices = newNumIndices;
6087     points     = newPoints;
6088     values     = newValues;
6089   }
6090   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6091   if (numFields) {
6092     PetscBool useFieldOffsets;
6093 
6094     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6095     if (useFieldOffsets) {
6096       for (p = 0; p < numPoints; p++) {
6097         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
6098       }
6099     } else {
6100       for (p = 0; p < numPoints; p++) {
6101         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6102         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
6103       }
6104     }
6105   } else {
6106     for (p = 0, off = 0; p < numPoints; p++) {
6107       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6108       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6109       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
6110     }
6111   }
6112   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6113   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
6114   if (mesh->printFEM > 1) {
6115     PetscInt i;
6116     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6117     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6118     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6119   }
6120   if (ierr) {
6121     PetscMPIInt    rank;
6122     PetscErrorCode ierr2;
6123 
6124     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6125     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6126     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6127     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6128     CHKERRQ(ierr);
6129   }
6130   for (f = 0; f < PetscMax(1,numFields); f++) {
6131     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6132     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6133   }
6134   if (newNumPoints) {
6135     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6136     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6137   }
6138   else {
6139     if (valCopy) {
6140       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6141     }
6142     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6143   }
6144   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6145   PetscFunctionReturn(0);
6146 }
6147 
6148 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6149 {
6150   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6151   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6152   PetscInt       *cpoints = NULL;
6153   PetscInt       *findices, *cindices;
6154   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
6155   PetscInt        foffsets[32], coffsets[32];
6156   CellRefiner     cellRefiner;
6157   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6158   PetscErrorCode  ierr;
6159 
6160   PetscFunctionBegin;
6161   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6162   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6163   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6164   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6165   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6166   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6167   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6168   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6169   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6170   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6171   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6172   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6173   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6174   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6175   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6176   /* Column indices */
6177   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6178   maxFPoints = numCPoints;
6179   /* Compress out points not in the section */
6180   /*   TODO: Squeeze out points with 0 dof as well */
6181   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6182   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6183     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6184       cpoints[q*2]   = cpoints[p];
6185       cpoints[q*2+1] = cpoints[p+1];
6186       ++q;
6187     }
6188   }
6189   numCPoints = q;
6190   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6191     PetscInt fdof;
6192 
6193     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6194     if (!dof) continue;
6195     for (f = 0; f < numFields; ++f) {
6196       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6197       coffsets[f+1] += fdof;
6198     }
6199     numCIndices += dof;
6200   }
6201   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6202   /* Row indices */
6203   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6204   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6205   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6206   for (r = 0, q = 0; r < numSubcells; ++r) {
6207     /* TODO Map from coarse to fine cells */
6208     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6209     /* Compress out points not in the section */
6210     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6211     for (p = 0; p < numFPoints*2; p += 2) {
6212       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6213         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6214         if (!dof) continue;
6215         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6216         if (s < q) continue;
6217         ftotpoints[q*2]   = fpoints[p];
6218         ftotpoints[q*2+1] = fpoints[p+1];
6219         ++q;
6220       }
6221     }
6222     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6223   }
6224   numFPoints = q;
6225   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6226     PetscInt fdof;
6227 
6228     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6229     if (!dof) continue;
6230     for (f = 0; f < numFields; ++f) {
6231       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6232       foffsets[f+1] += fdof;
6233     }
6234     numFIndices += dof;
6235   }
6236   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6237 
6238   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6239   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6240   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6241   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6242   if (numFields) {
6243     const PetscInt **permsF[32] = {NULL};
6244     const PetscInt **permsC[32] = {NULL};
6245 
6246     for (f = 0; f < numFields; f++) {
6247       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6248       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6249     }
6250     for (p = 0; p < numFPoints; p++) {
6251       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6252       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6253     }
6254     for (p = 0; p < numCPoints; p++) {
6255       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6256       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6257     }
6258     for (f = 0; f < numFields; f++) {
6259       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6260       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6261     }
6262   } else {
6263     const PetscInt **permsF = NULL;
6264     const PetscInt **permsC = NULL;
6265 
6266     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6267     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6268     for (p = 0, off = 0; p < numFPoints; p++) {
6269       const PetscInt *perm = permsF ? permsF[p] : NULL;
6270 
6271       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6272       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6273     }
6274     for (p = 0, off = 0; p < numCPoints; p++) {
6275       const PetscInt *perm = permsC ? permsC[p] : NULL;
6276 
6277       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6278       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6279     }
6280     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6281     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6282   }
6283   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6284   /* TODO: flips */
6285   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6286   if (ierr) {
6287     PetscMPIInt    rank;
6288     PetscErrorCode ierr2;
6289 
6290     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6291     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6292     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6293     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6294     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6295     CHKERRQ(ierr);
6296   }
6297   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6298   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6299   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6300   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6301   PetscFunctionReturn(0);
6302 }
6303 
6304 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6305 {
6306   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6307   PetscInt      *cpoints = NULL;
6308   PetscInt       foffsets[32], coffsets[32];
6309   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
6310   CellRefiner    cellRefiner;
6311   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6312   PetscErrorCode ierr;
6313 
6314   PetscFunctionBegin;
6315   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6316   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6317   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6318   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6319   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6320   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6321   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6322   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6323   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6324   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6325   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6326   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6327   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6328   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6329   /* Column indices */
6330   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6331   maxFPoints = numCPoints;
6332   /* Compress out points not in the section */
6333   /*   TODO: Squeeze out points with 0 dof as well */
6334   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6335   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6336     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6337       cpoints[q*2]   = cpoints[p];
6338       cpoints[q*2+1] = cpoints[p+1];
6339       ++q;
6340     }
6341   }
6342   numCPoints = q;
6343   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6344     PetscInt fdof;
6345 
6346     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6347     if (!dof) continue;
6348     for (f = 0; f < numFields; ++f) {
6349       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6350       coffsets[f+1] += fdof;
6351     }
6352     numCIndices += dof;
6353   }
6354   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6355   /* Row indices */
6356   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6357   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6358   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6359   for (r = 0, q = 0; r < numSubcells; ++r) {
6360     /* TODO Map from coarse to fine cells */
6361     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6362     /* Compress out points not in the section */
6363     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6364     for (p = 0; p < numFPoints*2; p += 2) {
6365       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6366         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6367         if (!dof) continue;
6368         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6369         if (s < q) continue;
6370         ftotpoints[q*2]   = fpoints[p];
6371         ftotpoints[q*2+1] = fpoints[p+1];
6372         ++q;
6373       }
6374     }
6375     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6376   }
6377   numFPoints = q;
6378   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6379     PetscInt fdof;
6380 
6381     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6382     if (!dof) continue;
6383     for (f = 0; f < numFields; ++f) {
6384       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6385       foffsets[f+1] += fdof;
6386     }
6387     numFIndices += dof;
6388   }
6389   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6390 
6391   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6392   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6393   if (numFields) {
6394     const PetscInt **permsF[32] = {NULL};
6395     const PetscInt **permsC[32] = {NULL};
6396 
6397     for (f = 0; f < numFields; f++) {
6398       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6399       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6400     }
6401     for (p = 0; p < numFPoints; p++) {
6402       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6403       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6404     }
6405     for (p = 0; p < numCPoints; p++) {
6406       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6407       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6408     }
6409     for (f = 0; f < numFields; f++) {
6410       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6411       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6412     }
6413   } else {
6414     const PetscInt **permsF = NULL;
6415     const PetscInt **permsC = NULL;
6416 
6417     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6418     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6419     for (p = 0, off = 0; p < numFPoints; p++) {
6420       const PetscInt *perm = permsF ? permsF[p] : NULL;
6421 
6422       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6423       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6424     }
6425     for (p = 0, off = 0; p < numCPoints; p++) {
6426       const PetscInt *perm = permsC ? permsC[p] : NULL;
6427 
6428       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6429       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6430     }
6431     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6432     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6433   }
6434   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6435   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6436   PetscFunctionReturn(0);
6437 }
6438 
6439 /*@
6440   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6441 
6442   Input Parameter:
6443 . dm - The DMPlex object
6444 
6445   Output Parameters:
6446 + cMax - The first hybrid cell
6447 . fMax - The first hybrid face
6448 . eMax - The first hybrid edge
6449 - vMax - The first hybrid vertex
6450 
6451   Level: developer
6452 
6453 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6454 @*/
6455 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6456 {
6457   DM_Plex       *mesh = (DM_Plex*) dm->data;
6458   PetscInt       dim;
6459   PetscErrorCode ierr;
6460 
6461   PetscFunctionBegin;
6462   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6463   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6464   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6465   if (cMax) *cMax = mesh->hybridPointMax[dim];
6466   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6467   if (eMax) *eMax = mesh->hybridPointMax[1];
6468   if (vMax) *vMax = mesh->hybridPointMax[0];
6469   PetscFunctionReturn(0);
6470 }
6471 
6472 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6473 {
6474   IS             is, his;
6475   PetscInt       first = 0, stride;
6476   PetscBool      isStride;
6477   PetscErrorCode ierr;
6478 
6479   PetscFunctionBegin;
6480   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6481   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6482   if (isStride) {
6483     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6484   }
6485   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6486   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6487   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6488   ierr = ISDestroy(&his);CHKERRQ(ierr);
6489   ierr = ISDestroy(&is);CHKERRQ(ierr);
6490   PetscFunctionReturn(0);
6491 }
6492 
6493 /*@
6494   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6495 
6496   Input Parameters:
6497 + dm   - The DMPlex object
6498 . cMax - The first hybrid cell
6499 . fMax - The first hybrid face
6500 . eMax - The first hybrid edge
6501 - vMax - The first hybrid vertex
6502 
6503   Level: developer
6504 
6505 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6506 @*/
6507 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6508 {
6509   DM_Plex       *mesh = (DM_Plex*) dm->data;
6510   PetscInt       dim;
6511   PetscErrorCode ierr;
6512 
6513   PetscFunctionBegin;
6514   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6515   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6516   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6517   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6518   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6519   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6520   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6521   PetscFunctionReturn(0);
6522 }
6523 
6524 /*@C
6525   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6526 
6527   Input Parameter:
6528 . dm   - The DMPlex object
6529 
6530   Output Parameter:
6531 . cellHeight - The height of a cell
6532 
6533   Level: developer
6534 
6535 .seealso DMPlexSetVTKCellHeight()
6536 @*/
6537 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6538 {
6539   DM_Plex *mesh = (DM_Plex*) dm->data;
6540 
6541   PetscFunctionBegin;
6542   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6543   PetscValidPointer(cellHeight, 2);
6544   *cellHeight = mesh->vtkCellHeight;
6545   PetscFunctionReturn(0);
6546 }
6547 
6548 /*@C
6549   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6550 
6551   Input Parameters:
6552 + dm   - The DMPlex object
6553 - cellHeight - The height of a cell
6554 
6555   Level: developer
6556 
6557 .seealso DMPlexGetVTKCellHeight()
6558 @*/
6559 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6560 {
6561   DM_Plex *mesh = (DM_Plex*) dm->data;
6562 
6563   PetscFunctionBegin;
6564   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6565   mesh->vtkCellHeight = cellHeight;
6566   PetscFunctionReturn(0);
6567 }
6568 
6569 /*@
6570   DMPlexGetGhostCellStratum - Get the range of cells which are used to enforce FV boundary conditions
6571 
6572   Input Parameter:
6573 . dm - The DMPlex object
6574 
6575   Output Parameters:
6576 + gcStart - The first ghost cell, or NULL
6577 - gcEnd   - The upper bound on ghost cells, or NULL
6578 
6579   Level: advanced
6580 
6581 .seealso DMPlexConstructGhostCells(), DMPlexSetGhostCellStratum(), DMPlexGetHybridBounds()
6582 @*/
6583 PetscErrorCode DMPlexGetGhostCellStratum(DM dm, PetscInt *gcStart, PetscInt *gcEnd)
6584 {
6585   DM_Plex       *mesh = (DM_Plex*) dm->data;
6586   PetscInt       dim;
6587   PetscErrorCode ierr;
6588 
6589   PetscFunctionBegin;
6590   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6591   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6592   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6593   if (gcStart) {PetscValidIntPointer(gcStart, 2); *gcStart = mesh->ghostCellStart;}
6594   if (gcEnd)   {
6595     PetscValidIntPointer(gcEnd, 3);
6596     if (mesh->ghostCellStart >= 0) {ierr = DMPlexGetHeightStratum(dm, 0, NULL, gcEnd);CHKERRQ(ierr);}
6597     else                           {*gcEnd = -1;}
6598   }
6599   PetscFunctionReturn(0);
6600 }
6601 
6602 /*@
6603   DMPlexSetGhostCellStratum - Set the range of cells which are used to enforce FV boundary conditions
6604 
6605   Input Parameters:
6606 + dm      - The DMPlex object
6607 . gcStart - The first ghost cell, or PETSC_DETERMINE
6608 - gcEnd   - The upper bound on ghost cells, or PETSC_DETERMINE
6609 
6610   Level: advanced
6611 
6612   Note: This is not usually called directly by a user.
6613 
6614 .seealso DMPlexConstructGhostCells(), DMPlexGetGhostCellStratum(), DMPlexSetHybridBounds()
6615 @*/
6616 PetscErrorCode DMPlexSetGhostCellStratum(DM dm, PetscInt gcStart, PetscInt gcEnd)
6617 {
6618   DM_Plex       *mesh = (DM_Plex*) dm->data;
6619   PetscInt       dim;
6620   PetscErrorCode ierr;
6621 
6622   PetscFunctionBegin;
6623   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6624   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6625   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6626   mesh->ghostCellStart = gcStart;
6627   if (gcEnd >= 0) {
6628     PetscInt cEnd;
6629     ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
6630     if (gcEnd != cEnd) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Ghost cells must appear at the end of the cell range, but gcEnd %D is not equal to cEnd %D", gcEnd, cEnd);
6631   }
6632   PetscFunctionReturn(0);
6633 }
6634 
6635 /*@
6636   DMPlexGetInteriorCellStratum - Get the range of cells which are neither hybrid nor ghost FV cells
6637 
6638   Input Parameter:
6639 . dm - The DMPlex object
6640 
6641   Output Parameters:
6642 + cStartInterior - The first ghost cell
6643 - cEndInterior   - The upper bound on ghost cells
6644 
6645   Level: developer
6646 
6647 .seealso DMPlexConstructGhostCells(), DMPlexSetGhostCellStratum(), DMPlexGetHybridBounds()
6648 @*/
6649 PetscErrorCode DMPlexGetInteriorCellStratum(DM dm, PetscInt *cStartInterior, PetscInt *cEndInterior)
6650 {
6651   PetscInt       gcEnd, cMax;
6652   PetscErrorCode ierr;
6653 
6654   PetscFunctionBegin;
6655   ierr = DMPlexGetHeightStratum(dm, 0, cStartInterior, cEndInterior);CHKERRQ(ierr);
6656   ierr = DMPlexGetGhostCellStratum(dm, &gcEnd, NULL);CHKERRQ(ierr);
6657   *cEndInterior = gcEnd < 0 ? *cEndInterior : gcEnd;
6658   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6659   *cEndInterior = cMax  < 0 ? *cEndInterior : cMax;
6660   PetscFunctionReturn(0);
6661 }
6662 
6663 /* We can easily have a form that takes an IS instead */
6664 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6665 {
6666   PetscSection   section, globalSection;
6667   PetscInt      *numbers, p;
6668   PetscErrorCode ierr;
6669 
6670   PetscFunctionBegin;
6671   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6672   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6673   for (p = pStart; p < pEnd; ++p) {
6674     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6675   }
6676   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6677   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6678   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6679   for (p = pStart; p < pEnd; ++p) {
6680     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6681     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6682     else                       numbers[p-pStart] += shift;
6683   }
6684   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6685   if (globalSize) {
6686     PetscLayout layout;
6687     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6688     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6689     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6690   }
6691   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6692   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6693   PetscFunctionReturn(0);
6694 }
6695 
6696 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6697 {
6698   PetscInt       cellHeight, cStart, cEnd, cMax;
6699   PetscErrorCode ierr;
6700 
6701   PetscFunctionBegin;
6702   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6703   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6704   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6705   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6706   ierr = DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6707   PetscFunctionReturn(0);
6708 }
6709 
6710 /*@
6711   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6712 
6713   Input Parameter:
6714 . dm   - The DMPlex object
6715 
6716   Output Parameter:
6717 . globalCellNumbers - Global cell numbers for all cells on this process
6718 
6719   Level: developer
6720 
6721 .seealso DMPlexGetVertexNumbering()
6722 @*/
6723 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6724 {
6725   DM_Plex       *mesh = (DM_Plex*) dm->data;
6726   PetscErrorCode ierr;
6727 
6728   PetscFunctionBegin;
6729   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6730   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6731   *globalCellNumbers = mesh->globalCellNumbers;
6732   PetscFunctionReturn(0);
6733 }
6734 
6735 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6736 {
6737   PetscInt       vStart, vEnd, vMax;
6738   PetscErrorCode ierr;
6739 
6740   PetscFunctionBegin;
6741   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6742   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6743   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6744   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6745   ierr = DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6746   PetscFunctionReturn(0);
6747 }
6748 
6749 /*@
6750   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
6751 
6752   Input Parameter:
6753 . dm   - The DMPlex object
6754 
6755   Output Parameter:
6756 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6757 
6758   Level: developer
6759 
6760 .seealso DMPlexGetCellNumbering()
6761 @*/
6762 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6763 {
6764   DM_Plex       *mesh = (DM_Plex*) dm->data;
6765   PetscErrorCode ierr;
6766 
6767   PetscFunctionBegin;
6768   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6769   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6770   *globalVertexNumbers = mesh->globalVertexNumbers;
6771   PetscFunctionReturn(0);
6772 }
6773 
6774 /*@
6775   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6776 
6777   Input Parameter:
6778 . dm   - The DMPlex object
6779 
6780   Output Parameter:
6781 . globalPointNumbers - Global numbers for all points on this process
6782 
6783   Level: developer
6784 
6785 .seealso DMPlexGetCellNumbering()
6786 @*/
6787 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6788 {
6789   IS             nums[4];
6790   PetscInt       depths[4], gdepths[4], starts[4];
6791   PetscInt       depth, d, shift = 0;
6792   PetscErrorCode ierr;
6793 
6794   PetscFunctionBegin;
6795   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6796   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6797   /* For unstratified meshes use dim instead of depth */
6798   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6799   for (d = 0; d <= depth; ++d) {
6800     PetscInt end;
6801 
6802     depths[d] = depth-d;
6803     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6804     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6805   }
6806   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6807   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6808   for (d = 0; d <= depth; ++d) {
6809     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6810   }
6811   for (d = 0; d <= depth; ++d) {
6812     PetscInt pStart, pEnd, gsize;
6813 
6814     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6815     ierr = DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6816     shift += gsize;
6817   }
6818   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6819   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6820   PetscFunctionReturn(0);
6821 }
6822 
6823 
6824 /*@
6825   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6826 
6827   Input Parameter:
6828 . dm - The DMPlex object
6829 
6830   Output Parameter:
6831 . ranks - The rank field
6832 
6833   Options Database Keys:
6834 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6835 
6836   Level: intermediate
6837 
6838 .seealso: DMView()
6839 @*/
6840 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6841 {
6842   DM             rdm;
6843   PetscFE        fe;
6844   PetscScalar   *r;
6845   PetscMPIInt    rank;
6846   PetscInt       dim, cStart, cEnd, c;
6847   PetscErrorCode ierr;
6848 
6849   PetscFunctionBeginUser;
6850   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6851   PetscValidPointer(ranks, 2);
6852   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6853   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6854   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6855   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6856   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6857   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6858   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6859   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6860   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6861   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6862   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6863   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6864   for (c = cStart; c < cEnd; ++c) {
6865     PetscScalar *lr;
6866 
6867     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6868     *lr = rank;
6869   }
6870   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6871   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6872   PetscFunctionReturn(0);
6873 }
6874 
6875 /*@
6876   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6877 
6878   Input Parameters:
6879 + dm    - The DMPlex
6880 - label - The DMLabel
6881 
6882   Output Parameter:
6883 . val - The label value field
6884 
6885   Options Database Keys:
6886 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6887 
6888   Level: intermediate
6889 
6890 .seealso: DMView()
6891 @*/
6892 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6893 {
6894   DM             rdm;
6895   PetscFE        fe;
6896   PetscScalar   *v;
6897   PetscInt       dim, cStart, cEnd, c;
6898   PetscErrorCode ierr;
6899 
6900   PetscFunctionBeginUser;
6901   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6902   PetscValidPointer(label, 2);
6903   PetscValidPointer(val, 3);
6904   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6905   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6906   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6907   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6908   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6909   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6910   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6911   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6912   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6913   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6914   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6915   for (c = cStart; c < cEnd; ++c) {
6916     PetscScalar *lv;
6917     PetscInt     cval;
6918 
6919     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6920     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6921     *lv = cval;
6922   }
6923   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6924   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6925   PetscFunctionReturn(0);
6926 }
6927 
6928 /*@
6929   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6930 
6931   Input Parameter:
6932 . dm - The DMPlex object
6933 
6934   Notes:
6935   This is a useful diagnostic when creating meshes programmatically.
6936 
6937   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
6938 
6939   Level: developer
6940 
6941 .seealso: DMCreate(), DMSetFromOptions()
6942 @*/
6943 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6944 {
6945   PetscSection    coneSection, supportSection;
6946   const PetscInt *cone, *support;
6947   PetscInt        coneSize, c, supportSize, s;
6948   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6949   PetscBool       storagecheck = PETSC_TRUE;
6950   PetscErrorCode  ierr;
6951 
6952   PetscFunctionBegin;
6953   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6954   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6955   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6956   /* Check that point p is found in the support of its cone points, and vice versa */
6957   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6958   for (p = pStart; p < pEnd; ++p) {
6959     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6960     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6961     for (c = 0; c < coneSize; ++c) {
6962       PetscBool dup = PETSC_FALSE;
6963       PetscInt  d;
6964       for (d = c-1; d >= 0; --d) {
6965         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6966       }
6967       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6968       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6969       for (s = 0; s < supportSize; ++s) {
6970         if (support[s] == p) break;
6971       }
6972       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6973         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6974         for (s = 0; s < coneSize; ++s) {
6975           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6976         }
6977         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6978         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6979         for (s = 0; s < supportSize; ++s) {
6980           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6981         }
6982         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6983         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6984         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6985       }
6986     }
6987     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6988     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6989     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6990     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6991     for (s = 0; s < supportSize; ++s) {
6992       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6993       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6994       for (c = 0; c < coneSize; ++c) {
6995         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6996         if (cone[c] != pp) { c = 0; break; }
6997         if (cone[c] == p) break;
6998       }
6999       if (c >= coneSize) {
7000         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
7001         for (c = 0; c < supportSize; ++c) {
7002           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
7003         }
7004         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7005         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
7006         for (c = 0; c < coneSize; ++c) {
7007           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
7008         }
7009         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
7010         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
7011       }
7012     }
7013   }
7014   if (storagecheck) {
7015     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
7016     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
7017     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
7018   }
7019   PetscFunctionReturn(0);
7020 }
7021 
7022 /*@
7023   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
7024 
7025   Input Parameters:
7026 + dm - The DMPlex object
7027 - cellHeight - Normally 0
7028 
7029   Notes:
7030   This is a useful diagnostic when creating meshes programmatically.
7031   Currently applicable only to homogeneous simplex or tensor meshes.
7032 
7033   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7034 
7035   Level: developer
7036 
7037 .seealso: DMCreate(), DMSetFromOptions()
7038 @*/
7039 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
7040 {
7041   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
7042   PetscBool      isSimplex = PETSC_FALSE;
7043   PetscErrorCode ierr;
7044 
7045   PetscFunctionBegin;
7046   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7047   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7048   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
7049   if (cStart < cEnd) {
7050     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
7051     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
7052   }
7053   switch (dim) {
7054   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
7055   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
7056   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
7057   default:
7058     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
7059   }
7060   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7061   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7062   cMax = cMax >= 0 ? cMax : cEnd;
7063   for (c = cStart; c < cMax; ++c) {
7064     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
7065 
7066     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7067     for (cl = 0; cl < closureSize*2; cl += 2) {
7068       const PetscInt p = closure[cl];
7069       if ((p >= vStart) && (p < vEnd)) ++coneSize;
7070     }
7071     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7072     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
7073   }
7074   for (c = cMax; c < cEnd; ++c) {
7075     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
7076 
7077     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7078     for (cl = 0; cl < closureSize*2; cl += 2) {
7079       const PetscInt p = closure[cl];
7080       if ((p >= vStart) && (p < vEnd)) ++coneSize;
7081     }
7082     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7083     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
7084   }
7085   PetscFunctionReturn(0);
7086 }
7087 
7088 /*@
7089   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
7090 
7091   Not Collective
7092 
7093   Input Parameters:
7094 + dm - The DMPlex object
7095 - cellHeight - Normally 0
7096 
7097   Notes:
7098   This is a useful diagnostic when creating meshes programmatically.
7099   This routine is only relevant for meshes that are fully interpolated across all ranks.
7100   It will error out if a partially interpolated mesh is given on some rank.
7101   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
7102 
7103   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7104 
7105   Level: developer
7106 
7107 .seealso: DMCreate(), DMPlexGetVTKCellHeight(), DMSetFromOptions()
7108 @*/
7109 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
7110 {
7111   PetscInt       pMax[4];
7112   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
7113   PetscErrorCode ierr;
7114   DMPlexInterpolatedFlag interpEnum;
7115 
7116   PetscFunctionBegin;
7117   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7118   ierr = DMPlexIsInterpolated(dm, &interpEnum);CHKERRQ(ierr);
7119   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(0);
7120   if (interpEnum == DMPLEX_INTERPOLATED_PARTIAL) {
7121     PetscMPIInt	rank;
7122     MPI_Comm	comm;
7123 
7124     ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
7125     ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7126     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Mesh is only partially interpolated on rank %d, this is currently not supported", rank);
7127   }
7128 
7129   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7130   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7131   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7132   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
7133   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
7134     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
7135     for (c = cStart; c < cEnd; ++c) {
7136       const PetscInt *cone, *ornt, *faces;
7137       PetscInt        numFaces, faceSize, coneSize,f;
7138       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
7139 
7140       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
7141       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
7142       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
7143       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
7144       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7145       for (cl = 0; cl < closureSize*2; cl += 2) {
7146         const PetscInt p = closure[cl];
7147         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
7148       }
7149       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
7150       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
7151       for (f = 0; f < numFaces; ++f) {
7152         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
7153 
7154         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7155         for (cl = 0; cl < fclosureSize*2; cl += 2) {
7156           const PetscInt p = fclosure[cl];
7157           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
7158         }
7159         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
7160         for (v = 0; v < fnumCorners; ++v) {
7161           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
7162         }
7163         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7164       }
7165       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
7166       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7167     }
7168   }
7169   PetscFunctionReturn(0);
7170 }
7171 
7172 /*@
7173   DMPlexCheckGeometry - Check the geometry of mesh cells
7174 
7175   Input Parameter:
7176 . dm - The DMPlex object
7177 
7178   Notes:
7179   This is a useful diagnostic when creating meshes programmatically.
7180 
7181   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7182 
7183   Level: developer
7184 
7185 .seealso: DMCreate(), DMSetFromOptions()
7186 @*/
7187 PetscErrorCode DMPlexCheckGeometry(DM dm)
7188 {
7189   PetscReal      detJ, J[9], refVol = 1.0;
7190   PetscReal      vol;
7191   PetscInt       dim, depth, d, cStart, cEnd, c, cMax;
7192   PetscErrorCode ierr;
7193 
7194   PetscFunctionBegin;
7195   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7196   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7197   for (d = 0; d < dim; ++d) refVol *= 2.0;
7198   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7199   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7200   cMax = cMax < 0 ? cEnd : cMax;
7201   for (c = cStart; c < cMax; ++c) {
7202     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7203     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7204     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7205     if (depth > 1) {
7206       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7207       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7208       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7209     }
7210   }
7211   PetscFunctionReturn(0);
7212 }
7213 
7214 /*@
7215   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
7216 
7217   Input Parameters:
7218 . dm - The DMPlex object
7219 
7220   Notes:
7221   This is mainly intended for debugging/testing purposes.
7222   It currently checks only meshes with no partition overlapping.
7223 
7224   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7225 
7226   Level: developer
7227 
7228 .seealso: DMGetPointSF(), DMSetFromOptions()
7229 @*/
7230 PetscErrorCode DMPlexCheckPointSF(DM dm)
7231 {
7232   PetscSF         pointSF;
7233   PetscInt        cellHeight, cStart, cEnd, l, nleaves, nroots, overlap;
7234   const PetscInt *locals, *rootdegree;
7235   PetscBool       distributed;
7236   PetscErrorCode  ierr;
7237 
7238   PetscFunctionBegin;
7239   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7240   ierr = DMGetPointSF(dm, &pointSF);CHKERRQ(ierr);
7241   ierr = DMPlexIsDistributed(dm, &distributed);CHKERRQ(ierr);
7242   if (!distributed) PetscFunctionReturn(0);
7243   ierr = DMPlexGetOverlap(dm, &overlap);CHKERRQ(ierr);
7244   if (overlap) {
7245     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");
7246     PetscFunctionReturn(0);
7247   }
7248   if (!pointSF) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but does not have PointSF attached");
7249   ierr = PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, NULL);CHKERRQ(ierr);
7250   if (nroots < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set");
7251   ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
7252   ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
7253 
7254   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7255   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7256   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
7257   for (l = 0; l < nleaves; ++l) {
7258     const PetscInt point = locals[l];
7259 
7260     if (point >= cStart && point < cEnd) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D which is a cell", point);
7261   }
7262 
7263   /* 2) if some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
7264   for (l = 0; l < nleaves; ++l) {
7265     const PetscInt  point = locals[l];
7266     const PetscInt *cone;
7267     PetscInt        coneSize, c, idx;
7268 
7269     ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
7270     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
7271     for (c = 0; c < coneSize; ++c) {
7272       if (!rootdegree[cone[c]]) {
7273         ierr = PetscFindInt(cone[c], nleaves, locals, &idx);CHKERRQ(ierr);
7274         if (idx < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %D but not %D from its cone", point, cone[c]);
7275       }
7276     }
7277   }
7278   PetscFunctionReturn(0);
7279 }
7280 
7281 typedef struct cell_stats
7282 {
7283   PetscReal min, max, sum, squaresum;
7284   PetscInt  count;
7285 } cell_stats_t;
7286 
7287 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7288 {
7289   PetscInt i, N = *len;
7290 
7291   for (i = 0; i < N; i++) {
7292     cell_stats_t *A = (cell_stats_t *) a;
7293     cell_stats_t *B = (cell_stats_t *) b;
7294 
7295     B->min = PetscMin(A->min,B->min);
7296     B->max = PetscMax(A->max,B->max);
7297     B->sum += A->sum;
7298     B->squaresum += A->squaresum;
7299     B->count += A->count;
7300   }
7301 }
7302 
7303 /*@
7304   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
7305 
7306   Collective on dm
7307 
7308   Input Parameters:
7309 + dm        - The DMPlex object
7310 . output    - If true, statistics will be displayed on stdout
7311 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
7312 
7313   Notes:
7314   This is mainly intended for debugging/testing purposes.
7315 
7316   For the complete list of DMPlexCheck* functions, see DMSetFromOptions().
7317 
7318   Level: developer
7319 
7320 .seealso: DMSetFromOptions()
7321 @*/
7322 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
7323 {
7324   DM             dmCoarse;
7325   cell_stats_t   stats, globalStats;
7326   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7327   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7328   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
7329   PetscInt       cdim, cStart, cEnd, cMax, c, eStart, eEnd, count = 0;
7330   PetscMPIInt    rank,size;
7331   PetscErrorCode ierr;
7332 
7333   PetscFunctionBegin;
7334   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7335   stats.min   = PETSC_MAX_REAL;
7336   stats.max   = PETSC_MIN_REAL;
7337   stats.sum   = stats.squaresum = 0.;
7338   stats.count = 0;
7339 
7340   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
7341   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7342   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
7343   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
7344   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7345   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
7346   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7347   cMax = cMax < 0 ? cEnd : cMax;
7348   for (c = cStart; c < cMax; c++) {
7349     PetscInt  i;
7350     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7351 
7352     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7353     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7354     for (i = 0; i < PetscSqr(cdim); ++i) {
7355       frobJ    += J[i] * J[i];
7356       frobInvJ += invJ[i] * invJ[i];
7357     }
7358     cond2 = frobJ * frobInvJ;
7359     cond  = PetscSqrtReal(cond2);
7360 
7361     stats.min        = PetscMin(stats.min,cond);
7362     stats.max        = PetscMax(stats.max,cond);
7363     stats.sum       += cond;
7364     stats.squaresum += cond2;
7365     stats.count++;
7366     if (output && cond > limit) {
7367       PetscSection coordSection;
7368       Vec          coordsLocal;
7369       PetscScalar *coords = NULL;
7370       PetscInt     Nv, d, clSize, cl, *closure = NULL;
7371 
7372       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
7373       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
7374       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7375       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
7376       for (i = 0; i < Nv/cdim; ++i) {
7377         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
7378         for (d = 0; d < cdim; ++d) {
7379           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
7380           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
7381         }
7382         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
7383       }
7384       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7385       for (cl = 0; cl < clSize*2; cl += 2) {
7386         const PetscInt edge = closure[cl];
7387 
7388         if ((edge >= eStart) && (edge < eEnd)) {
7389           PetscReal len;
7390 
7391           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
7392           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
7393         }
7394       }
7395       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7396       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7397     }
7398   }
7399   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
7400 
7401   if (size > 1) {
7402     PetscMPIInt   blockLengths[2] = {4,1};
7403     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7404     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7405     MPI_Op        statReduce;
7406 
7407     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7408     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7409     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7410     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7411     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7412     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7413   } else {
7414     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7415   }
7416   if (!rank) {
7417     count = globalStats.count;
7418     min   = globalStats.min;
7419     max   = globalStats.max;
7420     mean  = globalStats.sum / globalStats.count;
7421     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7422   }
7423 
7424   if (output) {
7425     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7426   }
7427   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7428 
7429   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7430   if (dmCoarse) {
7431     PetscBool isplex;
7432 
7433     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7434     if (isplex) {
7435       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
7436     }
7437   }
7438   PetscFunctionReturn(0);
7439 }
7440 
7441 /* Pointwise interpolation
7442      Just code FEM for now
7443      u^f = I u^c
7444      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7445      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7446      I_{ij} = psi^f_i phi^c_j
7447 */
7448 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7449 {
7450   PetscSection   gsc, gsf;
7451   PetscInt       m, n;
7452   void          *ctx;
7453   DM             cdm;
7454   PetscBool      regular, ismatis;
7455   PetscErrorCode ierr;
7456 
7457   PetscFunctionBegin;
7458   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7459   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7460   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7461   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7462 
7463   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7464   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7465   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7466   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7467   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7468 
7469   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7470   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7471   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7472   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7473   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7474   if (scaling) {
7475     /* Use naive scaling */
7476     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7477   }
7478   PetscFunctionReturn(0);
7479 }
7480 
7481 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7482 {
7483   PetscErrorCode ierr;
7484   VecScatter     ctx;
7485 
7486   PetscFunctionBegin;
7487   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7488   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7489   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7490   PetscFunctionReturn(0);
7491 }
7492 
7493 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7494 {
7495   PetscSection   gsc, gsf;
7496   PetscInt       m, n;
7497   void          *ctx;
7498   DM             cdm;
7499   PetscBool      regular;
7500   PetscErrorCode ierr;
7501 
7502   PetscFunctionBegin;
7503   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7504   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7505   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7506   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7507 
7508   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7509   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7510   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7511   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7512 
7513   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7514   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7515   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7516   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7517   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7518   PetscFunctionReturn(0);
7519 }
7520 
7521 /*@
7522   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7523 
7524   Input Parameter:
7525 . dm - The DMPlex object
7526 
7527   Output Parameter:
7528 . regular - The flag
7529 
7530   Level: intermediate
7531 
7532 .seealso: DMPlexSetRegularRefinement()
7533 @*/
7534 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7535 {
7536   PetscFunctionBegin;
7537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7538   PetscValidPointer(regular, 2);
7539   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7540   PetscFunctionReturn(0);
7541 }
7542 
7543 /*@
7544   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7545 
7546   Input Parameters:
7547 + dm - The DMPlex object
7548 - regular - The flag
7549 
7550   Level: intermediate
7551 
7552 .seealso: DMPlexGetRegularRefinement()
7553 @*/
7554 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7555 {
7556   PetscFunctionBegin;
7557   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7558   ((DM_Plex *) dm->data)->regularRefinement = regular;
7559   PetscFunctionReturn(0);
7560 }
7561 
7562 /* anchors */
7563 /*@
7564   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7565   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7566 
7567   not collective
7568 
7569   Input Parameters:
7570 . dm - The DMPlex object
7571 
7572   Output Parameters:
7573 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7574 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7575 
7576 
7577   Level: intermediate
7578 
7579 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7580 @*/
7581 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7582 {
7583   DM_Plex *plex = (DM_Plex *)dm->data;
7584   PetscErrorCode ierr;
7585 
7586   PetscFunctionBegin;
7587   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7588   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7589   if (anchorSection) *anchorSection = plex->anchorSection;
7590   if (anchorIS) *anchorIS = plex->anchorIS;
7591   PetscFunctionReturn(0);
7592 }
7593 
7594 /*@
7595   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7596   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7597   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7598 
7599   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7600   DMGetConstraints() and filling in the entries in the constraint matrix.
7601 
7602   collective on dm
7603 
7604   Input Parameters:
7605 + dm - The DMPlex object
7606 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7607 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7608 
7609   The reference counts of anchorSection and anchorIS are incremented.
7610 
7611   Level: intermediate
7612 
7613 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7614 @*/
7615 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7616 {
7617   DM_Plex        *plex = (DM_Plex *)dm->data;
7618   PetscMPIInt    result;
7619   PetscErrorCode ierr;
7620 
7621   PetscFunctionBegin;
7622   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7623   if (anchorSection) {
7624     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7625     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7626     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7627   }
7628   if (anchorIS) {
7629     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7630     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7631     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7632   }
7633 
7634   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7635   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7636   plex->anchorSection = anchorSection;
7637 
7638   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7639   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7640   plex->anchorIS = anchorIS;
7641 
7642 #if defined(PETSC_USE_DEBUG)
7643   if (anchorIS && anchorSection) {
7644     PetscInt size, a, pStart, pEnd;
7645     const PetscInt *anchors;
7646 
7647     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7648     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7649     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7650     for (a = 0; a < size; a++) {
7651       PetscInt p;
7652 
7653       p = anchors[a];
7654       if (p >= pStart && p < pEnd) {
7655         PetscInt dof;
7656 
7657         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7658         if (dof) {
7659           PetscErrorCode ierr2;
7660 
7661           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7662           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7663         }
7664       }
7665     }
7666     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7667   }
7668 #endif
7669   /* reset the generic constraints */
7670   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7671   PetscFunctionReturn(0);
7672 }
7673 
7674 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7675 {
7676   PetscSection anchorSection;
7677   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7678   PetscErrorCode ierr;
7679 
7680   PetscFunctionBegin;
7681   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7682   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7683   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7684   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7685   if (numFields) {
7686     PetscInt f;
7687     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7688 
7689     for (f = 0; f < numFields; f++) {
7690       PetscInt numComp;
7691 
7692       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7693       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7694     }
7695   }
7696   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7697   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7698   pStart = PetscMax(pStart,sStart);
7699   pEnd   = PetscMin(pEnd,sEnd);
7700   pEnd   = PetscMax(pStart,pEnd);
7701   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7702   for (p = pStart; p < pEnd; p++) {
7703     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7704     if (dof) {
7705       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7706       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7707       for (f = 0; f < numFields; f++) {
7708         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7709         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7710       }
7711     }
7712   }
7713   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7714   PetscFunctionReturn(0);
7715 }
7716 
7717 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7718 {
7719   PetscSection aSec;
7720   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7721   const PetscInt *anchors;
7722   PetscInt numFields, f;
7723   IS aIS;
7724   PetscErrorCode ierr;
7725 
7726   PetscFunctionBegin;
7727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7728   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7729   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7730   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7731   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7732   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7733   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7734   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7735   /* cSec will be a subset of aSec and section */
7736   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7737   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7738   i[0] = 0;
7739   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7740   for (p = pStart; p < pEnd; p++) {
7741     PetscInt rDof, rOff, r;
7742 
7743     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7744     if (!rDof) continue;
7745     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7746     if (numFields) {
7747       for (f = 0; f < numFields; f++) {
7748         annz = 0;
7749         for (r = 0; r < rDof; r++) {
7750           a = anchors[rOff + r];
7751           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7752           annz += aDof;
7753         }
7754         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7755         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7756         for (q = 0; q < dof; q++) {
7757           i[off + q + 1] = i[off + q] + annz;
7758         }
7759       }
7760     }
7761     else {
7762       annz = 0;
7763       for (q = 0; q < dof; q++) {
7764         a = anchors[off + q];
7765         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7766         annz += aDof;
7767       }
7768       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7769       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7770       for (q = 0; q < dof; q++) {
7771         i[off + q + 1] = i[off + q] + annz;
7772       }
7773     }
7774   }
7775   nnz = i[m];
7776   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7777   offset = 0;
7778   for (p = pStart; p < pEnd; p++) {
7779     if (numFields) {
7780       for (f = 0; f < numFields; f++) {
7781         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7782         for (q = 0; q < dof; q++) {
7783           PetscInt rDof, rOff, r;
7784           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7785           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7786           for (r = 0; r < rDof; r++) {
7787             PetscInt s;
7788 
7789             a = anchors[rOff + r];
7790             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7791             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7792             for (s = 0; s < aDof; s++) {
7793               j[offset++] = aOff + s;
7794             }
7795           }
7796         }
7797       }
7798     }
7799     else {
7800       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7801       for (q = 0; q < dof; q++) {
7802         PetscInt rDof, rOff, r;
7803         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7804         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7805         for (r = 0; r < rDof; r++) {
7806           PetscInt s;
7807 
7808           a = anchors[rOff + r];
7809           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7810           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7811           for (s = 0; s < aDof; s++) {
7812             j[offset++] = aOff + s;
7813           }
7814         }
7815       }
7816     }
7817   }
7818   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7819   ierr = PetscFree(i);CHKERRQ(ierr);
7820   ierr = PetscFree(j);CHKERRQ(ierr);
7821   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7822   PetscFunctionReturn(0);
7823 }
7824 
7825 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7826 {
7827   DM_Plex        *plex = (DM_Plex *)dm->data;
7828   PetscSection   anchorSection, section, cSec;
7829   Mat            cMat;
7830   PetscErrorCode ierr;
7831 
7832   PetscFunctionBegin;
7833   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7834   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7835   if (anchorSection) {
7836     PetscInt Nf;
7837 
7838     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
7839     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7840     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7841     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7842     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7843     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7844     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7845     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7846   }
7847   PetscFunctionReturn(0);
7848 }
7849 
7850 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7851 {
7852   IS             subis;
7853   PetscSection   section, subsection;
7854   PetscErrorCode ierr;
7855 
7856   PetscFunctionBegin;
7857   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
7858   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7859   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7860   /* Create subdomain */
7861   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7862   /* Create submodel */
7863   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7864   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7865   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7866   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
7867   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7868   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7869   /* Create map from submodel to global model */
7870   if (is) {
7871     PetscSection    sectionGlobal, subsectionGlobal;
7872     IS              spIS;
7873     const PetscInt *spmap;
7874     PetscInt       *subIndices;
7875     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7876     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7877 
7878     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7879     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7880     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7881     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7882     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7883     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7884     for (p = pStart; p < pEnd; ++p) {
7885       PetscInt gdof, pSubSize  = 0;
7886 
7887       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7888       if (gdof > 0) {
7889         for (f = 0; f < Nf; ++f) {
7890           PetscInt fdof, fcdof;
7891 
7892           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7893           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7894           pSubSize += fdof-fcdof;
7895         }
7896         subSize += pSubSize;
7897         if (pSubSize) {
7898           if (bs < 0) {
7899             bs = pSubSize;
7900           } else if (bs != pSubSize) {
7901             /* Layout does not admit a pointwise block size */
7902             bs = 1;
7903           }
7904         }
7905       }
7906     }
7907     /* Must have same blocksize on all procs (some might have no points) */
7908     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7909     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7910     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7911     else                            {bs = bsMinMax[0];}
7912     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7913     for (p = pStart; p < pEnd; ++p) {
7914       PetscInt gdof, goff;
7915 
7916       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7917       if (gdof > 0) {
7918         const PetscInt point = spmap[p];
7919 
7920         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7921         for (f = 0; f < Nf; ++f) {
7922           PetscInt fdof, fcdof, fc, f2, poff = 0;
7923 
7924           /* Can get rid of this loop by storing field information in the global section */
7925           for (f2 = 0; f2 < f; ++f2) {
7926             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7927             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7928             poff += fdof-fcdof;
7929           }
7930           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7931           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7932           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7933             subIndices[subOff] = goff+poff+fc;
7934           }
7935         }
7936       }
7937     }
7938     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7939     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7940     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7941     if (bs > 1) {
7942       /* We need to check that the block size does not come from non-contiguous fields */
7943       PetscInt i, j, set = 1;
7944       for (i = 0; i < subSize; i += bs) {
7945         for (j = 0; j < bs; ++j) {
7946           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7947         }
7948       }
7949       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7950     }
7951     /* Attach nullspace */
7952     for (f = 0; f < Nf; ++f) {
7953       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7954       if ((*subdm)->nullspaceConstructors[f]) break;
7955     }
7956     if (f < Nf) {
7957       MatNullSpace nullSpace;
7958 
7959       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7960       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7961       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7962     }
7963   }
7964   PetscFunctionReturn(0);
7965 }
7966 
7967 /*@
7968   DMPlexMonitorThroughput - Report the cell throughput of FE integration
7969 
7970   Input Parameter:
7971 - dm - The DM
7972 
7973   Level: developer
7974 
7975   Options Database Keys:
7976 . -dm_plex_monitor_throughput - Activate the monitor
7977 
7978 .seealso: DMSetFromOptions(), DMPlexCreate()
7979 @*/
7980 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
7981 {
7982   PetscStageLog      stageLog;
7983   PetscLogEvent      event;
7984   PetscLogStage      stage;
7985   PetscEventPerfInfo eventInfo;
7986   PetscReal          cellRate, flopRate;
7987   PetscInt           cStart, cEnd, Nf, N;
7988   const char        *name;
7989   PetscErrorCode     ierr;
7990 
7991   PetscFunctionBegin;
7992   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7993 #if defined(PETSC_USE_LOG)
7994   ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
7995   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7996   ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
7997   ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
7998   ierr = PetscStageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
7999   ierr = PetscLogEventGetId("DMPlexResidualFE", &event);CHKERRQ(ierr);
8000   ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr);
8001   N        = (cEnd - cStart)*Nf*eventInfo.count;
8002   flopRate = eventInfo.flops/eventInfo.time;
8003   cellRate = N/eventInfo.time;
8004   ierr = PetscPrintf(PetscObjectComm((PetscObject) dm), "DM (%s) FE Residual Integration: %D integrals %D reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double) cellRate, (double) flopRate/1.e6);CHKERRQ(ierr);
8005 #else
8006   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off. Reconfigure using --with-log.");
8007 #endif
8008   PetscFunctionReturn(0);
8009 }
8010