xref: /petsc/src/dm/impls/plex/plex.c (revision 28d58a3775d150be3540512488262419d353712e)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     char         lname[PETSC_MAX_PATH_LEN];
675     PetscReal    scale         = 2.0;
676     PetscReal    tikzscale     = 1.0;
677     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
678     double       tcoords[3];
679     PetscScalar *coords;
680     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
681     PetscMPIInt  rank, size;
682     char         **names, **colors, **lcolors;
683     PetscBool    plotEdges, flg, lflg;
684     PetscBT      wp = NULL;
685     PetscInt     pEnd, pStart;
686 
687     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
688     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
689     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
690     numLabels  = PetscMax(numLabels, 10);
691     numColors  = 10;
692     numLColors = 10;
693     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
694     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
695     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
696     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
697     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
698     if (!useLabels) numLabels = 0;
699     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
700     if (!useColors) {
701       numColors = 3;
702       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
703     }
704     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
705     if (!useColors) {
706       numLColors = 4;
707       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
708     }
709     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
710     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
711     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
712     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
713     if (depth < dim) plotEdges = PETSC_FALSE;
714 
715     /* filter points with labelvalue != labeldefaultvalue */
716     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
717     if (lflg) {
718       DMLabel lbl;
719 
720       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
721       if (lbl) {
722         PetscInt val, defval;
723 
724         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
725         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
726         for (c = pStart;  c < pEnd; c++) {
727           PetscInt *closure = NULL;
728           PetscInt  closureSize;
729 
730           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
731           if (val == defval) continue;
732 
733           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
734           for (p = 0; p < closureSize*2; p += 2) {
735             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
736           }
737           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
738         }
739       }
740     }
741 
742     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
743     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
744     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
745     ierr = PetscViewerASCIIPrintf(viewer, "\
746 \\documentclass[tikz]{standalone}\n\n\
747 \\usepackage{pgflibraryshapes}\n\
748 \\usetikzlibrary{backgrounds}\n\
749 \\usetikzlibrary{arrows}\n\
750 \\begin{document}\n");CHKERRQ(ierr);
751     if (size > 1) {
752       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
753       for (p = 0; p < size; ++p) {
754         if (p > 0 && p == size-1) {
755           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
756         } else if (p > 0) {
757           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
758         }
759         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
760       }
761       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
762     }
763     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", tikzscale);CHKERRQ(ierr);
764 
765     /* Plot vertices */
766     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
767     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
768     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
769     for (v = vStart; v < vEnd; ++v) {
770       PetscInt  off, dof, d;
771       PetscBool isLabeled = PETSC_FALSE;
772 
773       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
774       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
775       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
776       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
777       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
778       for (d = 0; d < dof; ++d) {
779         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
780         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
781       }
782       /* Rotate coordinates since PGF makes z point out of the page instead of up */
783       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
784       for (d = 0; d < dof; ++d) {
785         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
786         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", tcoords[d]);CHKERRQ(ierr);
787       }
788       color = colors[rank%numColors];
789       for (l = 0; l < numLabels; ++l) {
790         PetscInt val;
791         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
792         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
793       }
794       if (useNumbers) {
795         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
796       } else {
797         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
798       }
799     }
800     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
801     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
802     /* Plot cells */
803     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
804     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
805     if (dim == 3 || !useNumbers) {
806       for (e = eStart; e < eEnd; ++e) {
807         const PetscInt *cone;
808 
809         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
810         color = colors[rank%numColors];
811         for (l = 0; l < numLabels; ++l) {
812           PetscInt val;
813           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
814           if (val >= 0) {color = lcolors[l%numLColors]; break;}
815         }
816         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
817         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
818       }
819     } else {
820       for (c = cStart; c < cEnd; ++c) {
821         PetscInt *closure = NULL;
822         PetscInt  closureSize, firstPoint = -1;
823 
824         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
825         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
826         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
827         for (p = 0; p < closureSize*2; p += 2) {
828           const PetscInt point = closure[p];
829 
830           if ((point < vStart) || (point >= vEnd)) continue;
831           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
832           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
833           if (firstPoint < 0) firstPoint = point;
834         }
835         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
836         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
837         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
838       }
839     }
840     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
841     for (c = cStart; c < cEnd; ++c) {
842       double    ccoords[3] = {0.0, 0.0, 0.0};
843       PetscBool isLabeled  = PETSC_FALSE;
844       PetscInt *closure    = NULL;
845       PetscInt  closureSize, dof, d, n = 0;
846 
847       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
848       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
849       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
850       for (p = 0; p < closureSize*2; p += 2) {
851         const PetscInt point = closure[p];
852         PetscInt       off;
853 
854         if ((point < vStart) || (point >= vEnd)) continue;
855         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
856         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
857         for (d = 0; d < dof; ++d) {
858           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
859           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
860         }
861         /* Rotate coordinates since PGF makes z point out of the page instead of up */
862         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
863         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
864         ++n;
865       }
866       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
867       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
868       for (d = 0; d < dof; ++d) {
869         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
870         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", ccoords[d]);CHKERRQ(ierr);
871       }
872       color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (useNumbers) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
880       } else {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       }
883     }
884     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
885     /* Plot edges */
886     if (plotEdges) {
887       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
888       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
889       for (e = eStart; e < eEnd; ++e) {
890         const PetscInt *cone;
891         PetscInt        coneSize, offA, offB, dof, d;
892 
893         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
894         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
895         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
896         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
897         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
898         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
899         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
900         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
901         for (d = 0; d < dof; ++d) {
902           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
903           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
904         }
905         /* Rotate coordinates since PGF makes z point out of the page instead of up */
906         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
907         for (d = 0; d < dof; ++d) {
908           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
909           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
910         }
911         color = colors[rank%numColors];
912         for (l = 0; l < numLabels; ++l) {
913           PetscInt val;
914           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
915           if (val >= 0) {color = lcolors[l%numLColors]; break;}
916         }
917         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
918       }
919       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
920       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
921       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
922     }
923     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
924     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
925     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
926     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
927     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
928     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
929     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
930     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
931     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
932   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
933     Vec                    cown,acown;
934     VecScatter             sct;
935     ISLocalToGlobalMapping g2l;
936     IS                     gid,acis;
937     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
938     MPI_Group              ggroup,ngroup;
939     PetscScalar            *array,nid;
940     const PetscInt         *idxs;
941     PetscInt               *idxs2,*start,*adjacency,*work;
942     PetscInt64             lm[3],gm[3];
943     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
944     PetscMPIInt            d1,d2,rank;
945 
946     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
947     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
948 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
949     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
950 #endif
951     if (ncomm != MPI_COMM_NULL) {
952       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
953       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
954       d1   = 0;
955       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
956       nid  = d2;
957       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
958       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
959       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
960     } else nid = 0.0;
961 
962     /* Get connectivity */
963     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
964     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
965 
966     /* filter overlapped local cells */
967     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
968     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
969     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
970     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
971     for (c = cStart, cum = 0; c < cEnd; c++) {
972       if (idxs[c-cStart] < 0) continue;
973       idxs2[cum++] = idxs[c-cStart];
974     }
975     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
976     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
977     ierr = ISDestroy(&gid);CHKERRQ(ierr);
978     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
979 
980     /* support for node-aware cell locality */
981     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
982     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
983     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
984     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
985     for (c = 0; c < numVertices; c++) array[c] = nid;
986     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
987     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
988     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
989     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
990     ierr = ISDestroy(&acis);CHKERRQ(ierr);
991     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
992     ierr = VecDestroy(&cown);CHKERRQ(ierr);
993 
994     /* compute edgeCut */
995     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
996     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
997     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
998     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
999     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1000     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1001     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1002       PetscInt totl;
1003 
1004       totl = start[c+1]-start[c];
1005       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1006       for (i = 0; i < totl; i++) {
1007         if (work[i] < 0) {
1008           ect  += 1;
1009           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1010         }
1011       }
1012     }
1013     ierr  = PetscFree(work);CHKERRQ(ierr);
1014     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1015     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1016     lm[1] = -numVertices;
1017     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1018     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1019     lm[0] = ect; /* edgeCut */
1020     lm[1] = ectn; /* node-aware edgeCut */
1021     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1022     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1023     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1024 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
1025     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1026 #else
1027     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1028 #endif
1029     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1030     ierr  = PetscFree(start);CHKERRQ(ierr);
1031     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1032     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1033   } else {
1034     MPI_Comm    comm;
1035     PetscInt   *sizes, *hybsizes;
1036     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
1037     PetscInt    pStart, pEnd, p;
1038     PetscInt    numLabels, l;
1039     const char *name;
1040     PetscMPIInt size;
1041 
1042     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1043     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1044     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1045     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1046     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1047     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1048     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1049     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1050     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1051     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1052     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1053     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1054     if (depth == 1) {
1055       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1056       pEnd = pEnd - pStart;
1057       pMax[0] -= pStart;
1058       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1059       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1060       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1061       for (p = 0; p < size; ++p) {
1062         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1063         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1064       }
1065       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1066       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1067       pEnd = pEnd - pStart;
1068       pMax[depth] -= pStart;
1069       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1070       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1071       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1072       for (p = 0; p < size; ++p) {
1073         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1074         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1075       }
1076       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1077     } else {
1078       PetscMPIInt rank;
1079       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1080       for (d = 0; d <= dim; d++) {
1081         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1082         pEnd    -= pStart;
1083         pMax[d] -= pStart;
1084         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1085         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1086         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1087         for (p = 0; p < size; ++p) {
1088           if (!rank) {
1089             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1090             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1091           }
1092         }
1093         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1094       }
1095     }
1096     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1097     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1098     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1099     for (l = 0; l < numLabels; ++l) {
1100       DMLabel         label;
1101       const char     *name;
1102       IS              valueIS;
1103       const PetscInt *values;
1104       PetscInt        numValues, v;
1105 
1106       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1107       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1108       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1109       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1110       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1111       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1112       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1113       for (v = 0; v < numValues; ++v) {
1114         PetscInt size;
1115 
1116         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1117         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1118         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1119       }
1120       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1122       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1123       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1124     }
1125     /* If no fields are specified, people do not want to see adjacency */
1126     if (dm->Nf) {
1127       PetscInt f;
1128 
1129       for (f = 0; f < dm->Nf; ++f) {
1130         const char *name;
1131 
1132         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1133         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1134         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1135         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1136         if (dm->fields[f].adjacency[0]) {
1137           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1138           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1139         } else {
1140           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1141           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1142         }
1143         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1144       }
1145     }
1146     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1147     if (cdm) {
1148       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1149       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1150       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1151     }
1152   }
1153   PetscFunctionReturn(0);
1154 }
1155 
1156 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1157 {
1158   PetscDraw          draw;
1159   DM                 cdm;
1160   PetscSection       coordSection;
1161   Vec                coordinates;
1162   const PetscScalar *coords;
1163   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1164   PetscBool          isnull;
1165   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1166   PetscMPIInt        rank;
1167   PetscErrorCode     ierr;
1168 
1169   PetscFunctionBegin;
1170   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1171   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1172   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1173   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1174   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1175   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1176   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1177 
1178   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1179   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1180   if (isnull) PetscFunctionReturn(0);
1181   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1182 
1183   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1184   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1185   for (c = 0; c < N; c += dim) {
1186     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1187     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1188   }
1189   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1190   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1191   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1192   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1193   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1194 
1195   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1196   for (c = cStart; c < cEnd; ++c) {
1197     PetscScalar *coords = NULL;
1198     PetscInt     numCoords,coneSize;
1199 
1200     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1201     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202     switch (coneSize) {
1203     case 3:
1204       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1205                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1206                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1207                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1208       break;
1209     case 4:
1210       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1211                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1212                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1213                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1214                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1215       break;
1216     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1217     }
1218     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1219   }
1220   for (c = cStart; c < cEnd; ++c) {
1221     PetscScalar *coords = NULL;
1222     PetscInt     numCoords,coneSize;
1223 
1224     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1225     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1226     switch (coneSize) {
1227     case 3:
1228       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1229       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1230       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1231       break;
1232     case 4:
1233       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1234       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1235       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1236       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237       break;
1238     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1239     }
1240     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1241   }
1242   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1243   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1244   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1249 {
1250   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1251   char           name[PETSC_MAX_PATH_LEN];
1252   PetscErrorCode ierr;
1253 
1254   PetscFunctionBegin;
1255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1256   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1257   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1258   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1259   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1260   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1261   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1262   if (iascii) {
1263     PetscViewerFormat format;
1264     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1265     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1266       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1267     } else {
1268       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1269     }
1270   } else if (ishdf5) {
1271 #if defined(PETSC_HAVE_HDF5)
1272     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1273 #else
1274     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1275 #endif
1276   } else if (isvtk) {
1277     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1278   } else if (isdraw) {
1279     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1280   } else if (isglvis) {
1281     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1282   } else {
1283     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1284   }
1285   /* Optionally view the partition */
1286   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1287   if (flg) {
1288     Vec ranks;
1289     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1290     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1291     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1292   }
1293   /* Optionally view a label */
1294   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1295   if (flg) {
1296     DMLabel label;
1297     Vec     val;
1298 
1299     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1300     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1301     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1302     ierr = VecView(val, viewer);CHKERRQ(ierr);
1303     ierr = VecDestroy(&val);CHKERRQ(ierr);
1304   }
1305   PetscFunctionReturn(0);
1306 }
1307 
1308 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1309 {
1310   PetscBool      ishdf5;
1311   PetscErrorCode ierr;
1312 
1313   PetscFunctionBegin;
1314   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1315   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1316   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1317   if (ishdf5) {
1318 #if defined(PETSC_HAVE_HDF5)
1319     PetscViewerFormat format;
1320     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1321     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1322       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1323     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1324       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1325     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1326 #else
1327     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1328 #endif
1329   } else {
1330     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1331   }
1332   PetscFunctionReturn(0);
1333 }
1334 
1335 PetscErrorCode DMDestroy_Plex(DM dm)
1336 {
1337   DM_Plex       *mesh = (DM_Plex*) dm->data;
1338   PetscErrorCode ierr;
1339 
1340   PetscFunctionBegin;
1341   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1342   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1343   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
1344   if (--mesh->refct > 0) PetscFunctionReturn(0);
1345   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1346   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1347   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1348   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1349   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1350   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1351   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1352   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1353   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1354   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1355   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1356   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1357   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1358   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1359   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1360   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1361   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1362   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1363   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1364   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1365   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1366   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1367   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1368   ierr = PetscFree(mesh);CHKERRQ(ierr);
1369   PetscFunctionReturn(0);
1370 }
1371 
1372 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1373 {
1374   PetscSection           sectionGlobal;
1375   PetscInt               bs = -1, mbs;
1376   PetscInt               localSize;
1377   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1378   PetscErrorCode         ierr;
1379   MatType                mtype;
1380   ISLocalToGlobalMapping ltog;
1381 
1382   PetscFunctionBegin;
1383   ierr = MatInitializePackage();CHKERRQ(ierr);
1384   mtype = dm->mattype;
1385   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1386   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1387   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1388   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1389   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1390   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1391   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1392   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1393   if (mbs > 1) bs = mbs;
1394   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1395   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1396   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1397   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1398   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1399   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1400   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1401   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1402   if (!isShell) {
1403     PetscSection subSection;
1404     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1405     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1406     PetscInt     pStart, pEnd, p, dof, cdof;
1407 
1408     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1409     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1410       PetscSection section;
1411       PetscInt     size;
1412 
1413       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1414       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1415       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1416       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1417     } else {
1418       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1419     }
1420     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1421     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1422       PetscInt bdof;
1423 
1424       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1425       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1426       dof  = dof < 0 ? -(dof+1) : dof;
1427       bdof = cdof && (dof-cdof) ? 1 : dof;
1428       if (dof) {
1429         if (bs < 0)          {bs = bdof;}
1430         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1431       }
1432       if (isMatIS) {
1433         PetscInt loff,c,off;
1434         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1435         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1436         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1437       }
1438     }
1439     /* Must have same blocksize on all procs (some might have no points) */
1440     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1441     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1442     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1443     else                            {bs = bsMinMax[0];}
1444     bs = PetscMax(1,bs);
1445     if (isMatIS) { /* Must reduce indices by blocksize */
1446       PetscInt l;
1447 
1448       lsize = lsize/bs;
1449       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1450       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1451     }
1452     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1453     if (isMatIS) {
1454       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1455     }
1456     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1457     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1458     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1459   }
1460   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1461   PetscFunctionReturn(0);
1462 }
1463 
1464 /*@
1465   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1466 
1467   Not collective
1468 
1469   Input Parameter:
1470 . mesh - The DMPlex
1471 
1472   Output Parameters:
1473 . subsection - The subdomain section
1474 
1475   Level: developer
1476 
1477 .seealso:
1478 @*/
1479 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1480 {
1481   DM_Plex       *mesh = (DM_Plex*) dm->data;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   if (!mesh->subdomainSection) {
1487     PetscSection section;
1488     PetscSF      sf;
1489 
1490     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1491     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1492     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1493     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1494   }
1495   *subsection = mesh->subdomainSection;
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 /*@
1500   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1501 
1502   Not collective
1503 
1504   Input Parameter:
1505 . mesh - The DMPlex
1506 
1507   Output Parameters:
1508 + pStart - The first mesh point
1509 - pEnd   - The upper bound for mesh points
1510 
1511   Level: beginner
1512 
1513 .seealso: DMPlexCreate(), DMPlexSetChart()
1514 @*/
1515 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1516 {
1517   DM_Plex       *mesh = (DM_Plex*) dm->data;
1518   PetscErrorCode ierr;
1519 
1520   PetscFunctionBegin;
1521   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1522   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1523   PetscFunctionReturn(0);
1524 }
1525 
1526 /*@
1527   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1528 
1529   Not collective
1530 
1531   Input Parameters:
1532 + mesh - The DMPlex
1533 . pStart - The first mesh point
1534 - pEnd   - The upper bound for mesh points
1535 
1536   Output Parameters:
1537 
1538   Level: beginner
1539 
1540 .seealso: DMPlexCreate(), DMPlexGetChart()
1541 @*/
1542 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1543 {
1544   DM_Plex       *mesh = (DM_Plex*) dm->data;
1545   PetscErrorCode ierr;
1546 
1547   PetscFunctionBegin;
1548   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1549   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1550   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1551   PetscFunctionReturn(0);
1552 }
1553 
1554 /*@
1555   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1556 
1557   Not collective
1558 
1559   Input Parameters:
1560 + mesh - The DMPlex
1561 - p - The point, which must lie in the chart set with DMPlexSetChart()
1562 
1563   Output Parameter:
1564 . size - The cone size for point p
1565 
1566   Level: beginner
1567 
1568 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1569 @*/
1570 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1571 {
1572   DM_Plex       *mesh = (DM_Plex*) dm->data;
1573   PetscErrorCode ierr;
1574 
1575   PetscFunctionBegin;
1576   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1577   PetscValidPointer(size, 3);
1578   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1579   PetscFunctionReturn(0);
1580 }
1581 
1582 /*@
1583   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1584 
1585   Not collective
1586 
1587   Input Parameters:
1588 + mesh - The DMPlex
1589 . p - The point, which must lie in the chart set with DMPlexSetChart()
1590 - size - The cone size for point p
1591 
1592   Output Parameter:
1593 
1594   Note:
1595   This should be called after DMPlexSetChart().
1596 
1597   Level: beginner
1598 
1599 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1600 @*/
1601 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1602 {
1603   DM_Plex       *mesh = (DM_Plex*) dm->data;
1604   PetscErrorCode ierr;
1605 
1606   PetscFunctionBegin;
1607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1608   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1609 
1610   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1611   PetscFunctionReturn(0);
1612 }
1613 
1614 /*@
1615   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1616 
1617   Not collective
1618 
1619   Input Parameters:
1620 + mesh - The DMPlex
1621 . p - The point, which must lie in the chart set with DMPlexSetChart()
1622 - size - The additional cone size for point p
1623 
1624   Output Parameter:
1625 
1626   Note:
1627   This should be called after DMPlexSetChart().
1628 
1629   Level: beginner
1630 
1631 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1632 @*/
1633 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1634 {
1635   DM_Plex       *mesh = (DM_Plex*) dm->data;
1636   PetscInt       csize;
1637   PetscErrorCode ierr;
1638 
1639   PetscFunctionBegin;
1640   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1641   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1642   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1643 
1644   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1645   PetscFunctionReturn(0);
1646 }
1647 
1648 /*@C
1649   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1650 
1651   Not collective
1652 
1653   Input Parameters:
1654 + dm - The DMPlex
1655 - p - The point, which must lie in the chart set with DMPlexSetChart()
1656 
1657   Output Parameter:
1658 . cone - An array of points which are on the in-edges for point p
1659 
1660   Level: beginner
1661 
1662   Fortran Notes:
1663   Since it returns an array, this routine is only available in Fortran 90, and you must
1664   include petsc.h90 in your code.
1665   You must also call DMPlexRestoreCone() after you finish using the returned array.
1666   DMPlexRestoreCone() is not needed/available in C.
1667 
1668 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1669 @*/
1670 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1671 {
1672   DM_Plex       *mesh = (DM_Plex*) dm->data;
1673   PetscInt       off;
1674   PetscErrorCode ierr;
1675 
1676   PetscFunctionBegin;
1677   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1678   PetscValidPointer(cone, 3);
1679   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1680   *cone = &mesh->cones[off];
1681   PetscFunctionReturn(0);
1682 }
1683 
1684 /*@C
1685   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1686 
1687   Not collective
1688 
1689   Input Parameters:
1690 + dm - The DMPlex
1691 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1692 
1693   Output Parameter:
1694 + pConesSection - PetscSection describing the layout of pCones
1695 - pCones - An array of points which are on the in-edges for the point set p
1696 
1697   Level: intermediate
1698 
1699 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1700 @*/
1701 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1702 {
1703   PetscSection        cs, newcs;
1704   PetscInt            *cones;
1705   PetscInt            *newarr=NULL;
1706   PetscInt            n;
1707   PetscErrorCode      ierr;
1708 
1709   PetscFunctionBegin;
1710   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1711   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1712   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1713   if (pConesSection) *pConesSection = newcs;
1714   if (pCones) {
1715     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1716     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1717   }
1718   PetscFunctionReturn(0);
1719 }
1720 
1721 /*@
1722   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1723 
1724   Not collective
1725 
1726   Input Parameters:
1727 + dm - The DMPlex
1728 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1729 
1730   Output Parameter:
1731 . expandedPoints - An array of vertices recursively expanded from input points
1732 
1733   Level: advanced
1734 
1735   Notes:
1736   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1737   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1738 
1739 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1740 @*/
1741 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1742 {
1743   IS                  *expandedPointsAll;
1744   PetscInt            depth;
1745   PetscErrorCode      ierr;
1746 
1747   PetscFunctionBegin;
1748   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1749   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1750   PetscValidPointer(expandedPoints, 3);
1751   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1752   *expandedPoints = expandedPointsAll[0];
1753   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1754   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1755   PetscFunctionReturn(0);
1756 }
1757 
1758 /*@
1759   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1760 
1761   Not collective
1762 
1763   Input Parameters:
1764 + dm - The DMPlex
1765 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1766 
1767   Output Parameter:
1768 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1769 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1770 - sections - (optional) An array of sections which describe mappings from points to their cone points
1771 
1772   Level: advanced
1773 
1774   Notes:
1775   Like DMPlexGetConeTuple() but recursive.
1776 
1777   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1778   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1779 
1780   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1781   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1782   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1783 
1784 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1785 @*/
1786 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1787 {
1788   const PetscInt      *arr0=NULL, *cone=NULL;
1789   PetscInt            *arr=NULL, *newarr=NULL;
1790   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1791   IS                  *expandedPoints_;
1792   PetscSection        *sections_;
1793   PetscErrorCode      ierr;
1794 
1795   PetscFunctionBegin;
1796   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1797   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1798   if (depth) PetscValidIntPointer(depth, 3);
1799   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1800   if (sections) PetscValidPointer(sections, 5);
1801   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1802   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1803   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1804   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1805   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1806   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1807   for (d=depth_-1; d>=0; d--) {
1808     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1809     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1810     for (i=0; i<n; i++) {
1811       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1812       if (arr[i] >= start && arr[i] < end) {
1813         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1814         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1815       } else {
1816         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1817       }
1818     }
1819     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1820     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1821     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1822     for (i=0; i<n; i++) {
1823       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1824       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1825       if (cn > 1) {
1826         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1827         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1828       } else {
1829         newarr[co] = arr[i];
1830       }
1831     }
1832     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1833     arr = newarr;
1834     n = newn;
1835   }
1836   *depth = depth_;
1837   if (expandedPoints) *expandedPoints = expandedPoints_;
1838   else {
1839     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1840     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1841   }
1842   if (sections) *sections = sections_;
1843   else {
1844     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1845     ierr = PetscFree(sections_);CHKERRQ(ierr);
1846   }
1847   PetscFunctionReturn(0);
1848 }
1849 
1850 /*@
1851   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1852 
1853   Not collective
1854 
1855   Input Parameters:
1856 + dm - The DMPlex
1857 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1858 
1859   Output Parameter:
1860 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1861 . expandedPoints - (optional) An array of recursively expanded cones
1862 - sections - (optional) An array of sections which describe mappings from points to their cone points
1863 
1864   Level: advanced
1865 
1866   Notes:
1867   See DMPlexGetConeRecursive() for details.
1868 
1869 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1870 @*/
1871 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1872 {
1873   PetscInt            d, depth_;
1874   PetscErrorCode      ierr;
1875 
1876   PetscFunctionBegin;
1877   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1878   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1879   if (depth) *depth = 0;
1880   if (expandedPoints) {
1881     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1882     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1883   }
1884   if (sections)  {
1885     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1886     ierr = PetscFree(*sections);CHKERRQ(ierr);
1887   }
1888   PetscFunctionReturn(0);
1889 }
1890 
1891 /*@
1892   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1893 
1894   Not collective
1895 
1896   Input Parameters:
1897 + mesh - The DMPlex
1898 . p - The point, which must lie in the chart set with DMPlexSetChart()
1899 - cone - An array of points which are on the in-edges for point p
1900 
1901   Output Parameter:
1902 
1903   Note:
1904   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1905 
1906   Developer Note: Why not call this DMPlexSetCover()
1907 
1908   Level: beginner
1909 
1910 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1911 @*/
1912 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1913 {
1914   DM_Plex       *mesh = (DM_Plex*) dm->data;
1915   PetscInt       pStart, pEnd;
1916   PetscInt       dof, off, c;
1917   PetscErrorCode ierr;
1918 
1919   PetscFunctionBegin;
1920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1921   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1922   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1923   if (dof) PetscValidPointer(cone, 3);
1924   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1925   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1926   for (c = 0; c < dof; ++c) {
1927     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1928     mesh->cones[off+c] = cone[c];
1929   }
1930   PetscFunctionReturn(0);
1931 }
1932 
1933 /*@C
1934   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1935 
1936   Not collective
1937 
1938   Input Parameters:
1939 + mesh - The DMPlex
1940 - p - The point, which must lie in the chart set with DMPlexSetChart()
1941 
1942   Output Parameter:
1943 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1944                     integer giving the prescription for cone traversal. If it is negative, the cone is
1945                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1946                     the index of the cone point on which to start.
1947 
1948   Level: beginner
1949 
1950   Fortran Notes:
1951   Since it returns an array, this routine is only available in Fortran 90, and you must
1952   include petsc.h90 in your code.
1953   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1954   DMPlexRestoreConeOrientation() is not needed/available in C.
1955 
1956 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1957 @*/
1958 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1959 {
1960   DM_Plex       *mesh = (DM_Plex*) dm->data;
1961   PetscInt       off;
1962   PetscErrorCode ierr;
1963 
1964   PetscFunctionBegin;
1965   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1966 #if defined(PETSC_USE_DEBUG)
1967   {
1968     PetscInt dof;
1969     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1970     if (dof) PetscValidPointer(coneOrientation, 3);
1971   }
1972 #endif
1973   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1974 
1975   *coneOrientation = &mesh->coneOrientations[off];
1976   PetscFunctionReturn(0);
1977 }
1978 
1979 /*@
1980   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1981 
1982   Not collective
1983 
1984   Input Parameters:
1985 + mesh - The DMPlex
1986 . p - The point, which must lie in the chart set with DMPlexSetChart()
1987 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1988                     integer giving the prescription for cone traversal. If it is negative, the cone is
1989                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1990                     the index of the cone point on which to start.
1991 
1992   Output Parameter:
1993 
1994   Note:
1995   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1996 
1997   Level: beginner
1998 
1999 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2000 @*/
2001 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2002 {
2003   DM_Plex       *mesh = (DM_Plex*) dm->data;
2004   PetscInt       pStart, pEnd;
2005   PetscInt       dof, off, c;
2006   PetscErrorCode ierr;
2007 
2008   PetscFunctionBegin;
2009   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2010   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2011   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2012   if (dof) PetscValidPointer(coneOrientation, 3);
2013   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2014   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2015   for (c = 0; c < dof; ++c) {
2016     PetscInt cdof, o = coneOrientation[c];
2017 
2018     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2019     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2020     mesh->coneOrientations[off+c] = o;
2021   }
2022   PetscFunctionReturn(0);
2023 }
2024 
2025 /*@
2026   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2027 
2028   Not collective
2029 
2030   Input Parameters:
2031 + mesh - The DMPlex
2032 . p - The point, which must lie in the chart set with DMPlexSetChart()
2033 . conePos - The local index in the cone where the point should be put
2034 - conePoint - The mesh point to insert
2035 
2036   Level: beginner
2037 
2038 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2039 @*/
2040 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2041 {
2042   DM_Plex       *mesh = (DM_Plex*) dm->data;
2043   PetscInt       pStart, pEnd;
2044   PetscInt       dof, off;
2045   PetscErrorCode ierr;
2046 
2047   PetscFunctionBegin;
2048   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2049   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2050   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2051   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2052   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2053   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2054   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2055   mesh->cones[off+conePos] = conePoint;
2056   PetscFunctionReturn(0);
2057 }
2058 
2059 /*@
2060   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2061 
2062   Not collective
2063 
2064   Input Parameters:
2065 + mesh - The DMPlex
2066 . p - The point, which must lie in the chart set with DMPlexSetChart()
2067 . conePos - The local index in the cone where the point should be put
2068 - coneOrientation - The point orientation to insert
2069 
2070   Level: beginner
2071 
2072 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2073 @*/
2074 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2075 {
2076   DM_Plex       *mesh = (DM_Plex*) dm->data;
2077   PetscInt       pStart, pEnd;
2078   PetscInt       dof, off;
2079   PetscErrorCode ierr;
2080 
2081   PetscFunctionBegin;
2082   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2083   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2084   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2085   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2086   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2087   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2088   mesh->coneOrientations[off+conePos] = coneOrientation;
2089   PetscFunctionReturn(0);
2090 }
2091 
2092 /*@
2093   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2094 
2095   Not collective
2096 
2097   Input Parameters:
2098 + mesh - The DMPlex
2099 - p - The point, which must lie in the chart set with DMPlexSetChart()
2100 
2101   Output Parameter:
2102 . size - The support size for point p
2103 
2104   Level: beginner
2105 
2106 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2107 @*/
2108 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2109 {
2110   DM_Plex       *mesh = (DM_Plex*) dm->data;
2111   PetscErrorCode ierr;
2112 
2113   PetscFunctionBegin;
2114   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2115   PetscValidPointer(size, 3);
2116   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2117   PetscFunctionReturn(0);
2118 }
2119 
2120 /*@
2121   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2122 
2123   Not collective
2124 
2125   Input Parameters:
2126 + mesh - The DMPlex
2127 . p - The point, which must lie in the chart set with DMPlexSetChart()
2128 - size - The support size for point p
2129 
2130   Output Parameter:
2131 
2132   Note:
2133   This should be called after DMPlexSetChart().
2134 
2135   Level: beginner
2136 
2137 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2138 @*/
2139 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2140 {
2141   DM_Plex       *mesh = (DM_Plex*) dm->data;
2142   PetscErrorCode ierr;
2143 
2144   PetscFunctionBegin;
2145   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2146   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2147 
2148   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2149   PetscFunctionReturn(0);
2150 }
2151 
2152 /*@C
2153   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2154 
2155   Not collective
2156 
2157   Input Parameters:
2158 + mesh - The DMPlex
2159 - p - The point, which must lie in the chart set with DMPlexSetChart()
2160 
2161   Output Parameter:
2162 . support - An array of points which are on the out-edges for point p
2163 
2164   Level: beginner
2165 
2166   Fortran Notes:
2167   Since it returns an array, this routine is only available in Fortran 90, and you must
2168   include petsc.h90 in your code.
2169   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2170   DMPlexRestoreSupport() is not needed/available in C.
2171 
2172 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2173 @*/
2174 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2175 {
2176   DM_Plex       *mesh = (DM_Plex*) dm->data;
2177   PetscInt       off;
2178   PetscErrorCode ierr;
2179 
2180   PetscFunctionBegin;
2181   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2182   PetscValidPointer(support, 3);
2183   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2184   *support = &mesh->supports[off];
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 /*@
2189   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2190 
2191   Not collective
2192 
2193   Input Parameters:
2194 + mesh - The DMPlex
2195 . p - The point, which must lie in the chart set with DMPlexSetChart()
2196 - support - An array of points which are on the out-edges for point p
2197 
2198   Output Parameter:
2199 
2200   Note:
2201   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2202 
2203   Level: beginner
2204 
2205 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2206 @*/
2207 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2208 {
2209   DM_Plex       *mesh = (DM_Plex*) dm->data;
2210   PetscInt       pStart, pEnd;
2211   PetscInt       dof, off, c;
2212   PetscErrorCode ierr;
2213 
2214   PetscFunctionBegin;
2215   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2216   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2217   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2218   if (dof) PetscValidPointer(support, 3);
2219   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2220   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2221   for (c = 0; c < dof; ++c) {
2222     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2223     mesh->supports[off+c] = support[c];
2224   }
2225   PetscFunctionReturn(0);
2226 }
2227 
2228 /*@
2229   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2230 
2231   Not collective
2232 
2233   Input Parameters:
2234 + mesh - The DMPlex
2235 . p - The point, which must lie in the chart set with DMPlexSetChart()
2236 . supportPos - The local index in the cone where the point should be put
2237 - supportPoint - The mesh point to insert
2238 
2239   Level: beginner
2240 
2241 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2242 @*/
2243 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2244 {
2245   DM_Plex       *mesh = (DM_Plex*) dm->data;
2246   PetscInt       pStart, pEnd;
2247   PetscInt       dof, off;
2248   PetscErrorCode ierr;
2249 
2250   PetscFunctionBegin;
2251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2252   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2253   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2254   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2255   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2256   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2257   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2258   mesh->supports[off+supportPos] = supportPoint;
2259   PetscFunctionReturn(0);
2260 }
2261 
2262 /*@C
2263   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2264 
2265   Not collective
2266 
2267   Input Parameters:
2268 + mesh - The DMPlex
2269 . p - The point, which must lie in the chart set with DMPlexSetChart()
2270 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2271 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2272 
2273   Output Parameters:
2274 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2275 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2276 
2277   Note:
2278   If using internal storage (points is NULL on input), each call overwrites the last output.
2279 
2280   Fortran Notes:
2281   Since it returns an array, this routine is only available in Fortran 90, and you must
2282   include petsc.h90 in your code.
2283 
2284   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2285 
2286   Level: beginner
2287 
2288 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2289 @*/
2290 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2291 {
2292   DM_Plex        *mesh = (DM_Plex*) dm->data;
2293   PetscInt       *closure, *fifo;
2294   const PetscInt *tmp = NULL, *tmpO = NULL;
2295   PetscInt        tmpSize, t;
2296   PetscInt        depth       = 0, maxSize;
2297   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2298   PetscErrorCode  ierr;
2299 
2300   PetscFunctionBegin;
2301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2302   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2303   /* This is only 1-level */
2304   if (useCone) {
2305     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2306     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2307     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2308   } else {
2309     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2310     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2311   }
2312   if (depth == 1) {
2313     if (*points) {
2314       closure = *points;
2315     } else {
2316       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2317       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2318     }
2319     closure[0] = p; closure[1] = 0;
2320     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2321       closure[closureSize]   = tmp[t];
2322       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2323     }
2324     if (numPoints) *numPoints = closureSize/2;
2325     if (points)    *points    = closure;
2326     PetscFunctionReturn(0);
2327   }
2328   {
2329     PetscInt c, coneSeries, s,supportSeries;
2330 
2331     c = mesh->maxConeSize;
2332     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2333     s = mesh->maxSupportSize;
2334     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2335     maxSize = 2*PetscMax(coneSeries,supportSeries);
2336   }
2337   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2338   if (*points) {
2339     closure = *points;
2340   } else {
2341     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2342   }
2343   closure[0] = p; closure[1] = 0;
2344   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2345     const PetscInt cp = tmp[t];
2346     const PetscInt co = tmpO ? tmpO[t] : 0;
2347 
2348     closure[closureSize]   = cp;
2349     closure[closureSize+1] = co;
2350     fifo[fifoSize]         = cp;
2351     fifo[fifoSize+1]       = co;
2352   }
2353   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2354   while (fifoSize - fifoStart) {
2355     const PetscInt q   = fifo[fifoStart];
2356     const PetscInt o   = fifo[fifoStart+1];
2357     const PetscInt rev = o >= 0 ? 0 : 1;
2358     const PetscInt off = rev ? -(o+1) : o;
2359 
2360     if (useCone) {
2361       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2362       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2363       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2364     } else {
2365       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2366       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2367       tmpO = NULL;
2368     }
2369     for (t = 0; t < tmpSize; ++t) {
2370       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2371       const PetscInt cp = tmp[i];
2372       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2373       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2374        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2375       PetscInt       co = tmpO ? tmpO[i] : 0;
2376       PetscInt       c;
2377 
2378       if (rev) {
2379         PetscInt childSize, coff;
2380         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2381         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2382         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2383       }
2384       /* Check for duplicate */
2385       for (c = 0; c < closureSize; c += 2) {
2386         if (closure[c] == cp) break;
2387       }
2388       if (c == closureSize) {
2389         closure[closureSize]   = cp;
2390         closure[closureSize+1] = co;
2391         fifo[fifoSize]         = cp;
2392         fifo[fifoSize+1]       = co;
2393         closureSize           += 2;
2394         fifoSize              += 2;
2395       }
2396     }
2397     fifoStart += 2;
2398   }
2399   if (numPoints) *numPoints = closureSize/2;
2400   if (points)    *points    = closure;
2401   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2402   PetscFunctionReturn(0);
2403 }
2404 
2405 /*@C
2406   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2407 
2408   Not collective
2409 
2410   Input Parameters:
2411 + mesh - The DMPlex
2412 . p - The point, which must lie in the chart set with DMPlexSetChart()
2413 . orientation - The orientation of the point
2414 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2415 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2416 
2417   Output Parameters:
2418 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2419 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2420 
2421   Note:
2422   If using internal storage (points is NULL on input), each call overwrites the last output.
2423 
2424   Fortran Notes:
2425   Since it returns an array, this routine is only available in Fortran 90, and you must
2426   include petsc.h90 in your code.
2427 
2428   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2429 
2430   Level: beginner
2431 
2432 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2433 @*/
2434 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2435 {
2436   DM_Plex        *mesh = (DM_Plex*) dm->data;
2437   PetscInt       *closure, *fifo;
2438   const PetscInt *tmp = NULL, *tmpO = NULL;
2439   PetscInt        tmpSize, t;
2440   PetscInt        depth       = 0, maxSize;
2441   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2442   PetscErrorCode  ierr;
2443 
2444   PetscFunctionBegin;
2445   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2446   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2447   /* This is only 1-level */
2448   if (useCone) {
2449     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2450     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2451     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2452   } else {
2453     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2454     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2455   }
2456   if (depth == 1) {
2457     if (*points) {
2458       closure = *points;
2459     } else {
2460       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2461       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2462     }
2463     closure[0] = p; closure[1] = ornt;
2464     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2465       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2466       closure[closureSize]   = tmp[i];
2467       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2468     }
2469     if (numPoints) *numPoints = closureSize/2;
2470     if (points)    *points    = closure;
2471     PetscFunctionReturn(0);
2472   }
2473   {
2474     PetscInt c, coneSeries, s,supportSeries;
2475 
2476     c = mesh->maxConeSize;
2477     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2478     s = mesh->maxSupportSize;
2479     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2480     maxSize = 2*PetscMax(coneSeries,supportSeries);
2481   }
2482   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2483   if (*points) {
2484     closure = *points;
2485   } else {
2486     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2487   }
2488   closure[0] = p; closure[1] = ornt;
2489   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2490     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2491     const PetscInt cp = tmp[i];
2492     PetscInt       co = tmpO ? tmpO[i] : 0;
2493 
2494     if (ornt < 0) {
2495       PetscInt childSize, coff;
2496       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2497       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2498       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2499     }
2500     closure[closureSize]   = cp;
2501     closure[closureSize+1] = co;
2502     fifo[fifoSize]         = cp;
2503     fifo[fifoSize+1]       = co;
2504   }
2505   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2506   while (fifoSize - fifoStart) {
2507     const PetscInt q   = fifo[fifoStart];
2508     const PetscInt o   = fifo[fifoStart+1];
2509     const PetscInt rev = o >= 0 ? 0 : 1;
2510     const PetscInt off = rev ? -(o+1) : o;
2511 
2512     if (useCone) {
2513       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2514       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2515       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2516     } else {
2517       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2518       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2519       tmpO = NULL;
2520     }
2521     for (t = 0; t < tmpSize; ++t) {
2522       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2523       const PetscInt cp = tmp[i];
2524       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2525       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2526        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2527       PetscInt       co = tmpO ? tmpO[i] : 0;
2528       PetscInt       c;
2529 
2530       if (rev) {
2531         PetscInt childSize, coff;
2532         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2533         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2534         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2535       }
2536       /* Check for duplicate */
2537       for (c = 0; c < closureSize; c += 2) {
2538         if (closure[c] == cp) break;
2539       }
2540       if (c == closureSize) {
2541         closure[closureSize]   = cp;
2542         closure[closureSize+1] = co;
2543         fifo[fifoSize]         = cp;
2544         fifo[fifoSize+1]       = co;
2545         closureSize           += 2;
2546         fifoSize              += 2;
2547       }
2548     }
2549     fifoStart += 2;
2550   }
2551   if (numPoints) *numPoints = closureSize/2;
2552   if (points)    *points    = closure;
2553   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2554   PetscFunctionReturn(0);
2555 }
2556 
2557 /*@C
2558   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2559 
2560   Not collective
2561 
2562   Input Parameters:
2563 + mesh - The DMPlex
2564 . p - The point, which must lie in the chart set with DMPlexSetChart()
2565 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2566 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2567 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2568 
2569   Note:
2570   If not using internal storage (points is not NULL on input), this call is unnecessary
2571 
2572   Fortran Notes:
2573   Since it returns an array, this routine is only available in Fortran 90, and you must
2574   include petsc.h90 in your code.
2575 
2576   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2577 
2578   Level: beginner
2579 
2580 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2581 @*/
2582 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2583 {
2584   PetscErrorCode ierr;
2585 
2586   PetscFunctionBegin;
2587   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2588   if (numPoints) PetscValidIntPointer(numPoints,4);
2589   if (points) PetscValidPointer(points,5);
2590   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2591   if (numPoints) *numPoints = 0;
2592   PetscFunctionReturn(0);
2593 }
2594 
2595 /*@
2596   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2597 
2598   Not collective
2599 
2600   Input Parameter:
2601 . mesh - The DMPlex
2602 
2603   Output Parameters:
2604 + maxConeSize - The maximum number of in-edges
2605 - maxSupportSize - The maximum number of out-edges
2606 
2607   Level: beginner
2608 
2609 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2610 @*/
2611 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2612 {
2613   DM_Plex *mesh = (DM_Plex*) dm->data;
2614 
2615   PetscFunctionBegin;
2616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2617   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2618   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2619   PetscFunctionReturn(0);
2620 }
2621 
2622 PetscErrorCode DMSetUp_Plex(DM dm)
2623 {
2624   DM_Plex       *mesh = (DM_Plex*) dm->data;
2625   PetscInt       size;
2626   PetscErrorCode ierr;
2627 
2628   PetscFunctionBegin;
2629   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2630   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2631   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2632   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2633   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2634   if (mesh->maxSupportSize) {
2635     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2636     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2637     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2638   }
2639   PetscFunctionReturn(0);
2640 }
2641 
2642 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2643 {
2644   PetscErrorCode ierr;
2645 
2646   PetscFunctionBegin;
2647   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2648   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2649   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2650   if (dm->useNatural && dm->sfMigration) {
2651     PetscSF        sfMigrationInv,sfNatural;
2652     PetscSection   section, sectionSeq;
2653 
2654     (*subdm)->sfMigration = dm->sfMigration;
2655     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2656     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2657     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2658     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2659     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2660 
2661     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2662     (*subdm)->sfNatural = sfNatural;
2663     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2664     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2665   }
2666   PetscFunctionReturn(0);
2667 }
2668 
2669 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2670 {
2671   PetscErrorCode ierr;
2672   PetscInt       i = 0;
2673 
2674   PetscFunctionBegin;
2675   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2676   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2677   (*superdm)->useNatural = PETSC_FALSE;
2678   for (i = 0; i < len; i++){
2679     if (dms[i]->useNatural && dms[i]->sfMigration) {
2680       PetscSF        sfMigrationInv,sfNatural;
2681       PetscSection   section, sectionSeq;
2682 
2683       (*superdm)->sfMigration = dms[i]->sfMigration;
2684       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2685       (*superdm)->useNatural = PETSC_TRUE;
2686       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2687       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2688       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2689       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2690 
2691       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2692       (*superdm)->sfNatural = sfNatural;
2693       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2694       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2695       break;
2696     }
2697   }
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 /*@
2702   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2703 
2704   Not collective
2705 
2706   Input Parameter:
2707 . mesh - The DMPlex
2708 
2709   Output Parameter:
2710 
2711   Note:
2712   This should be called after all calls to DMPlexSetCone()
2713 
2714   Level: beginner
2715 
2716 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2717 @*/
2718 PetscErrorCode DMPlexSymmetrize(DM dm)
2719 {
2720   DM_Plex       *mesh = (DM_Plex*) dm->data;
2721   PetscInt      *offsets;
2722   PetscInt       supportSize;
2723   PetscInt       pStart, pEnd, p;
2724   PetscErrorCode ierr;
2725 
2726   PetscFunctionBegin;
2727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2728   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2729   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2730   /* Calculate support sizes */
2731   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2732   for (p = pStart; p < pEnd; ++p) {
2733     PetscInt dof, off, c;
2734 
2735     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2736     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2737     for (c = off; c < off+dof; ++c) {
2738       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2739     }
2740   }
2741   for (p = pStart; p < pEnd; ++p) {
2742     PetscInt dof;
2743 
2744     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2745 
2746     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2747   }
2748   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2749   /* Calculate supports */
2750   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2751   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2752   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2753   for (p = pStart; p < pEnd; ++p) {
2754     PetscInt dof, off, c;
2755 
2756     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2757     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2758     for (c = off; c < off+dof; ++c) {
2759       const PetscInt q = mesh->cones[c];
2760       PetscInt       offS;
2761 
2762       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2763 
2764       mesh->supports[offS+offsets[q]] = p;
2765       ++offsets[q];
2766     }
2767   }
2768   ierr = PetscFree(offsets);CHKERRQ(ierr);
2769   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2770   PetscFunctionReturn(0);
2771 }
2772 
2773 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2774 {
2775   IS             stratumIS;
2776   PetscErrorCode ierr;
2777 
2778   PetscFunctionBegin;
2779   if (pStart >= pEnd) PetscFunctionReturn(0);
2780 #if defined(PETSC_USE_DEBUG)
2781   {
2782     PetscInt  qStart, qEnd, numLevels, level;
2783     PetscBool overlap = PETSC_FALSE;
2784     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2785     for (level = 0; level < numLevels; level++) {
2786       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2787       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2788     }
2789     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2790   }
2791 #endif
2792   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2793   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2794   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2795   PetscFunctionReturn(0);
2796 }
2797 
2798 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2799 
2800 /*@
2801   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2802   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2803   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2804   the DAG.
2805 
2806   Collective on dm
2807 
2808   Input Parameter:
2809 . mesh - The DMPlex
2810 
2811   Output Parameter:
2812 
2813   Notes:
2814   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2815   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2816   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2817   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2818 
2819   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2820 
2821   Level: beginner
2822 
2823 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2824 @*/
2825 PetscErrorCode DMPlexStratify(DM dm)
2826 {
2827   DM_Plex       *mesh = (DM_Plex*) dm->data;
2828   DMLabel        label;
2829   PetscInt       pStart, pEnd, p;
2830   PetscInt       numRoots = 0, numLeaves = 0;
2831   PetscInt       cMax, fMax, eMax, vMax;
2832   PetscErrorCode ierr;
2833 
2834   PetscFunctionBegin;
2835   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2836   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2837 
2838   /* Create depth label */
2839   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2840   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2841   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2842 
2843   {
2844     /* Initialize roots and count leaves */
2845     PetscInt sMin = PETSC_MAX_INT;
2846     PetscInt sMax = PETSC_MIN_INT;
2847     PetscInt coneSize, supportSize;
2848 
2849     for (p = pStart; p < pEnd; ++p) {
2850       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2851       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2852       if (!coneSize && supportSize) {
2853         sMin = PetscMin(p, sMin);
2854         sMax = PetscMax(p, sMax);
2855         ++numRoots;
2856       } else if (!supportSize && coneSize) {
2857         ++numLeaves;
2858       } else if (!supportSize && !coneSize) {
2859         /* Isolated points */
2860         sMin = PetscMin(p, sMin);
2861         sMax = PetscMax(p, sMax);
2862       }
2863     }
2864     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2865   }
2866 
2867   if (numRoots + numLeaves == (pEnd - pStart)) {
2868     PetscInt sMin = PETSC_MAX_INT;
2869     PetscInt sMax = PETSC_MIN_INT;
2870     PetscInt coneSize, supportSize;
2871 
2872     for (p = pStart; p < pEnd; ++p) {
2873       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2874       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2875       if (!supportSize && coneSize) {
2876         sMin = PetscMin(p, sMin);
2877         sMax = PetscMax(p, sMax);
2878       }
2879     }
2880     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2881   } else {
2882     PetscInt level = 0;
2883     PetscInt qStart, qEnd, q;
2884 
2885     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2886     while (qEnd > qStart) {
2887       PetscInt sMin = PETSC_MAX_INT;
2888       PetscInt sMax = PETSC_MIN_INT;
2889 
2890       for (q = qStart; q < qEnd; ++q) {
2891         const PetscInt *support;
2892         PetscInt        supportSize, s;
2893 
2894         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2895         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2896         for (s = 0; s < supportSize; ++s) {
2897           sMin = PetscMin(support[s], sMin);
2898           sMax = PetscMax(support[s], sMax);
2899         }
2900       }
2901       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2902       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2903       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2904     }
2905   }
2906   { /* just in case there is an empty process */
2907     PetscInt numValues, maxValues = 0, v;
2908 
2909     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2910     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2911     for (v = numValues; v < maxValues; v++) {
2912       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2913     }
2914   }
2915   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2916 
2917   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2918   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2919     PetscInt dim;
2920     DMLabel  dimLabel;
2921 
2922     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2923     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2924     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2925     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2926     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2927     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2928     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2929   }
2930   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2931   PetscFunctionReturn(0);
2932 }
2933 
2934 /*@C
2935   DMPlexGetJoin - Get an array for the join of the set of points
2936 
2937   Not Collective
2938 
2939   Input Parameters:
2940 + dm - The DMPlex object
2941 . numPoints - The number of input points for the join
2942 - points - The input points
2943 
2944   Output Parameters:
2945 + numCoveredPoints - The number of points in the join
2946 - coveredPoints - The points in the join
2947 
2948   Level: intermediate
2949 
2950   Note: Currently, this is restricted to a single level join
2951 
2952   Fortran Notes:
2953   Since it returns an array, this routine is only available in Fortran 90, and you must
2954   include petsc.h90 in your code.
2955 
2956   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2957 
2958 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2959 @*/
2960 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2961 {
2962   DM_Plex       *mesh = (DM_Plex*) dm->data;
2963   PetscInt      *join[2];
2964   PetscInt       joinSize, i = 0;
2965   PetscInt       dof, off, p, c, m;
2966   PetscErrorCode ierr;
2967 
2968   PetscFunctionBegin;
2969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2970   PetscValidIntPointer(points, 3);
2971   PetscValidIntPointer(numCoveredPoints, 4);
2972   PetscValidPointer(coveredPoints, 5);
2973   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2974   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2975   /* Copy in support of first point */
2976   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2977   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2978   for (joinSize = 0; joinSize < dof; ++joinSize) {
2979     join[i][joinSize] = mesh->supports[off+joinSize];
2980   }
2981   /* Check each successive support */
2982   for (p = 1; p < numPoints; ++p) {
2983     PetscInt newJoinSize = 0;
2984 
2985     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2986     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2987     for (c = 0; c < dof; ++c) {
2988       const PetscInt point = mesh->supports[off+c];
2989 
2990       for (m = 0; m < joinSize; ++m) {
2991         if (point == join[i][m]) {
2992           join[1-i][newJoinSize++] = point;
2993           break;
2994         }
2995       }
2996     }
2997     joinSize = newJoinSize;
2998     i        = 1-i;
2999   }
3000   *numCoveredPoints = joinSize;
3001   *coveredPoints    = join[i];
3002   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3003   PetscFunctionReturn(0);
3004 }
3005 
3006 /*@C
3007   DMPlexRestoreJoin - Restore an array for the join of the set of points
3008 
3009   Not Collective
3010 
3011   Input Parameters:
3012 + dm - The DMPlex object
3013 . numPoints - The number of input points for the join
3014 - points - The input points
3015 
3016   Output Parameters:
3017 + numCoveredPoints - The number of points in the join
3018 - coveredPoints - The points in the join
3019 
3020   Fortran Notes:
3021   Since it returns an array, this routine is only available in Fortran 90, and you must
3022   include petsc.h90 in your code.
3023 
3024   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3025 
3026   Level: intermediate
3027 
3028 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3029 @*/
3030 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3031 {
3032   PetscErrorCode ierr;
3033 
3034   PetscFunctionBegin;
3035   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3036   if (points) PetscValidIntPointer(points,3);
3037   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3038   PetscValidPointer(coveredPoints, 5);
3039   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3040   if (numCoveredPoints) *numCoveredPoints = 0;
3041   PetscFunctionReturn(0);
3042 }
3043 
3044 /*@C
3045   DMPlexGetFullJoin - Get an array for the join of the set of points
3046 
3047   Not Collective
3048 
3049   Input Parameters:
3050 + dm - The DMPlex object
3051 . numPoints - The number of input points for the join
3052 - points - The input points
3053 
3054   Output Parameters:
3055 + numCoveredPoints - The number of points in the join
3056 - coveredPoints - The points in the join
3057 
3058   Fortran Notes:
3059   Since it returns an array, this routine is only available in Fortran 90, and you must
3060   include petsc.h90 in your code.
3061 
3062   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3063 
3064   Level: intermediate
3065 
3066 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3067 @*/
3068 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3069 {
3070   DM_Plex       *mesh = (DM_Plex*) dm->data;
3071   PetscInt      *offsets, **closures;
3072   PetscInt      *join[2];
3073   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3074   PetscInt       p, d, c, m, ms;
3075   PetscErrorCode ierr;
3076 
3077   PetscFunctionBegin;
3078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3079   PetscValidIntPointer(points, 3);
3080   PetscValidIntPointer(numCoveredPoints, 4);
3081   PetscValidPointer(coveredPoints, 5);
3082 
3083   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3084   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3085   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3086   ms      = mesh->maxSupportSize;
3087   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3088   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3089   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3090 
3091   for (p = 0; p < numPoints; ++p) {
3092     PetscInt closureSize;
3093 
3094     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3095 
3096     offsets[p*(depth+2)+0] = 0;
3097     for (d = 0; d < depth+1; ++d) {
3098       PetscInt pStart, pEnd, i;
3099 
3100       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3101       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3102         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3103           offsets[p*(depth+2)+d+1] = i;
3104           break;
3105         }
3106       }
3107       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3108     }
3109     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3110   }
3111   for (d = 0; d < depth+1; ++d) {
3112     PetscInt dof;
3113 
3114     /* Copy in support of first point */
3115     dof = offsets[d+1] - offsets[d];
3116     for (joinSize = 0; joinSize < dof; ++joinSize) {
3117       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3118     }
3119     /* Check each successive cone */
3120     for (p = 1; p < numPoints && joinSize; ++p) {
3121       PetscInt newJoinSize = 0;
3122 
3123       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3124       for (c = 0; c < dof; ++c) {
3125         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3126 
3127         for (m = 0; m < joinSize; ++m) {
3128           if (point == join[i][m]) {
3129             join[1-i][newJoinSize++] = point;
3130             break;
3131           }
3132         }
3133       }
3134       joinSize = newJoinSize;
3135       i        = 1-i;
3136     }
3137     if (joinSize) break;
3138   }
3139   *numCoveredPoints = joinSize;
3140   *coveredPoints    = join[i];
3141   for (p = 0; p < numPoints; ++p) {
3142     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3143   }
3144   ierr = PetscFree(closures);CHKERRQ(ierr);
3145   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3146   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3147   PetscFunctionReturn(0);
3148 }
3149 
3150 /*@C
3151   DMPlexGetMeet - Get an array for the meet of the set of points
3152 
3153   Not Collective
3154 
3155   Input Parameters:
3156 + dm - The DMPlex object
3157 . numPoints - The number of input points for the meet
3158 - points - The input points
3159 
3160   Output Parameters:
3161 + numCoveredPoints - The number of points in the meet
3162 - coveredPoints - The points in the meet
3163 
3164   Level: intermediate
3165 
3166   Note: Currently, this is restricted to a single level meet
3167 
3168   Fortran Notes:
3169   Since it returns an array, this routine is only available in Fortran 90, and you must
3170   include petsc.h90 in your code.
3171 
3172   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3173 
3174 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3175 @*/
3176 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3177 {
3178   DM_Plex       *mesh = (DM_Plex*) dm->data;
3179   PetscInt      *meet[2];
3180   PetscInt       meetSize, i = 0;
3181   PetscInt       dof, off, p, c, m;
3182   PetscErrorCode ierr;
3183 
3184   PetscFunctionBegin;
3185   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3186   PetscValidPointer(points, 2);
3187   PetscValidPointer(numCoveringPoints, 3);
3188   PetscValidPointer(coveringPoints, 4);
3189   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3190   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3191   /* Copy in cone of first point */
3192   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3193   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3194   for (meetSize = 0; meetSize < dof; ++meetSize) {
3195     meet[i][meetSize] = mesh->cones[off+meetSize];
3196   }
3197   /* Check each successive cone */
3198   for (p = 1; p < numPoints; ++p) {
3199     PetscInt newMeetSize = 0;
3200 
3201     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3202     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3203     for (c = 0; c < dof; ++c) {
3204       const PetscInt point = mesh->cones[off+c];
3205 
3206       for (m = 0; m < meetSize; ++m) {
3207         if (point == meet[i][m]) {
3208           meet[1-i][newMeetSize++] = point;
3209           break;
3210         }
3211       }
3212     }
3213     meetSize = newMeetSize;
3214     i        = 1-i;
3215   }
3216   *numCoveringPoints = meetSize;
3217   *coveringPoints    = meet[i];
3218   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3219   PetscFunctionReturn(0);
3220 }
3221 
3222 /*@C
3223   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3224 
3225   Not Collective
3226 
3227   Input Parameters:
3228 + dm - The DMPlex object
3229 . numPoints - The number of input points for the meet
3230 - points - The input points
3231 
3232   Output Parameters:
3233 + numCoveredPoints - The number of points in the meet
3234 - coveredPoints - The points in the meet
3235 
3236   Level: intermediate
3237 
3238   Fortran Notes:
3239   Since it returns an array, this routine is only available in Fortran 90, and you must
3240   include petsc.h90 in your code.
3241 
3242   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3243 
3244 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3245 @*/
3246 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3247 {
3248   PetscErrorCode ierr;
3249 
3250   PetscFunctionBegin;
3251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3252   if (points) PetscValidIntPointer(points,3);
3253   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3254   PetscValidPointer(coveredPoints,5);
3255   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3256   if (numCoveredPoints) *numCoveredPoints = 0;
3257   PetscFunctionReturn(0);
3258 }
3259 
3260 /*@C
3261   DMPlexGetFullMeet - Get an array for the meet of the set of points
3262 
3263   Not Collective
3264 
3265   Input Parameters:
3266 + dm - The DMPlex object
3267 . numPoints - The number of input points for the meet
3268 - points - The input points
3269 
3270   Output Parameters:
3271 + numCoveredPoints - The number of points in the meet
3272 - coveredPoints - The points in the meet
3273 
3274   Level: intermediate
3275 
3276   Fortran Notes:
3277   Since it returns an array, this routine is only available in Fortran 90, and you must
3278   include petsc.h90 in your code.
3279 
3280   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3281 
3282 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3283 @*/
3284 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3285 {
3286   DM_Plex       *mesh = (DM_Plex*) dm->data;
3287   PetscInt      *offsets, **closures;
3288   PetscInt      *meet[2];
3289   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3290   PetscInt       p, h, c, m, mc;
3291   PetscErrorCode ierr;
3292 
3293   PetscFunctionBegin;
3294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3295   PetscValidPointer(points, 2);
3296   PetscValidPointer(numCoveredPoints, 3);
3297   PetscValidPointer(coveredPoints, 4);
3298 
3299   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3300   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3301   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3302   mc      = mesh->maxConeSize;
3303   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3304   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3305   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3306 
3307   for (p = 0; p < numPoints; ++p) {
3308     PetscInt closureSize;
3309 
3310     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3311 
3312     offsets[p*(height+2)+0] = 0;
3313     for (h = 0; h < height+1; ++h) {
3314       PetscInt pStart, pEnd, i;
3315 
3316       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3317       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3318         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3319           offsets[p*(height+2)+h+1] = i;
3320           break;
3321         }
3322       }
3323       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3324     }
3325     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3326   }
3327   for (h = 0; h < height+1; ++h) {
3328     PetscInt dof;
3329 
3330     /* Copy in cone of first point */
3331     dof = offsets[h+1] - offsets[h];
3332     for (meetSize = 0; meetSize < dof; ++meetSize) {
3333       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3334     }
3335     /* Check each successive cone */
3336     for (p = 1; p < numPoints && meetSize; ++p) {
3337       PetscInt newMeetSize = 0;
3338 
3339       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3340       for (c = 0; c < dof; ++c) {
3341         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3342 
3343         for (m = 0; m < meetSize; ++m) {
3344           if (point == meet[i][m]) {
3345             meet[1-i][newMeetSize++] = point;
3346             break;
3347           }
3348         }
3349       }
3350       meetSize = newMeetSize;
3351       i        = 1-i;
3352     }
3353     if (meetSize) break;
3354   }
3355   *numCoveredPoints = meetSize;
3356   *coveredPoints    = meet[i];
3357   for (p = 0; p < numPoints; ++p) {
3358     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3359   }
3360   ierr = PetscFree(closures);CHKERRQ(ierr);
3361   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3362   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3363   PetscFunctionReturn(0);
3364 }
3365 
3366 /*@C
3367   DMPlexEqual - Determine if two DMs have the same topology
3368 
3369   Not Collective
3370 
3371   Input Parameters:
3372 + dmA - A DMPlex object
3373 - dmB - A DMPlex object
3374 
3375   Output Parameters:
3376 . equal - PETSC_TRUE if the topologies are identical
3377 
3378   Level: intermediate
3379 
3380   Notes:
3381   We are not solving graph isomorphism, so we do not permutation.
3382 
3383 .seealso: DMPlexGetCone()
3384 @*/
3385 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3386 {
3387   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3388   PetscErrorCode ierr;
3389 
3390   PetscFunctionBegin;
3391   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3392   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3393   PetscValidPointer(equal, 3);
3394 
3395   *equal = PETSC_FALSE;
3396   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3397   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3398   if (depth != depthB) PetscFunctionReturn(0);
3399   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3400   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3401   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3402   for (p = pStart; p < pEnd; ++p) {
3403     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3404     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3405 
3406     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3407     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3408     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3409     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3410     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3411     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3412     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3413     for (c = 0; c < coneSize; ++c) {
3414       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3415       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3416     }
3417     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3418     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3419     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3420     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3421     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3422     for (s = 0; s < supportSize; ++s) {
3423       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3424     }
3425   }
3426   *equal = PETSC_TRUE;
3427   PetscFunctionReturn(0);
3428 }
3429 
3430 /*@C
3431   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3432 
3433   Not Collective
3434 
3435   Input Parameters:
3436 + dm         - The DMPlex
3437 . cellDim    - The cell dimension
3438 - numCorners - The number of vertices on a cell
3439 
3440   Output Parameters:
3441 . numFaceVertices - The number of vertices on a face
3442 
3443   Level: developer
3444 
3445   Notes:
3446   Of course this can only work for a restricted set of symmetric shapes
3447 
3448 .seealso: DMPlexGetCone()
3449 @*/
3450 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3451 {
3452   MPI_Comm       comm;
3453   PetscErrorCode ierr;
3454 
3455   PetscFunctionBegin;
3456   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3457   PetscValidPointer(numFaceVertices,3);
3458   switch (cellDim) {
3459   case 0:
3460     *numFaceVertices = 0;
3461     break;
3462   case 1:
3463     *numFaceVertices = 1;
3464     break;
3465   case 2:
3466     switch (numCorners) {
3467     case 3: /* triangle */
3468       *numFaceVertices = 2; /* Edge has 2 vertices */
3469       break;
3470     case 4: /* quadrilateral */
3471       *numFaceVertices = 2; /* Edge has 2 vertices */
3472       break;
3473     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3474       *numFaceVertices = 3; /* Edge has 3 vertices */
3475       break;
3476     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3477       *numFaceVertices = 3; /* Edge has 3 vertices */
3478       break;
3479     default:
3480       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3481     }
3482     break;
3483   case 3:
3484     switch (numCorners) {
3485     case 4: /* tetradehdron */
3486       *numFaceVertices = 3; /* Face has 3 vertices */
3487       break;
3488     case 6: /* tet cohesive cells */
3489       *numFaceVertices = 4; /* Face has 4 vertices */
3490       break;
3491     case 8: /* hexahedron */
3492       *numFaceVertices = 4; /* Face has 4 vertices */
3493       break;
3494     case 9: /* tet cohesive Lagrange cells */
3495       *numFaceVertices = 6; /* Face has 6 vertices */
3496       break;
3497     case 10: /* quadratic tetrahedron */
3498       *numFaceVertices = 6; /* Face has 6 vertices */
3499       break;
3500     case 12: /* hex cohesive Lagrange cells */
3501       *numFaceVertices = 6; /* Face has 6 vertices */
3502       break;
3503     case 18: /* quadratic tet cohesive Lagrange cells */
3504       *numFaceVertices = 6; /* Face has 6 vertices */
3505       break;
3506     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3507       *numFaceVertices = 9; /* Face has 9 vertices */
3508       break;
3509     default:
3510       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3511     }
3512     break;
3513   default:
3514     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3515   }
3516   PetscFunctionReturn(0);
3517 }
3518 
3519 /*@
3520   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3521 
3522   Not Collective
3523 
3524   Input Parameter:
3525 . dm    - The DMPlex object
3526 
3527   Output Parameter:
3528 . depthLabel - The DMLabel recording point depth
3529 
3530   Level: developer
3531 
3532 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3533 @*/
3534 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3535 {
3536   PetscErrorCode ierr;
3537 
3538   PetscFunctionBegin;
3539   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3540   PetscValidPointer(depthLabel, 2);
3541   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3542   *depthLabel = dm->depthLabel;
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 /*@
3547   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3548 
3549   Not Collective
3550 
3551   Input Parameter:
3552 . dm    - The DMPlex object
3553 
3554   Output Parameter:
3555 . depth - The number of strata (breadth first levels) in the DAG
3556 
3557   Level: developer
3558 
3559 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3560 @*/
3561 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3562 {
3563   DMLabel        label;
3564   PetscInt       d = 0;
3565   PetscErrorCode ierr;
3566 
3567   PetscFunctionBegin;
3568   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3569   PetscValidPointer(depth, 2);
3570   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3571   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3572   *depth = d-1;
3573   PetscFunctionReturn(0);
3574 }
3575 
3576 /*@
3577   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3578 
3579   Not Collective
3580 
3581   Input Parameters:
3582 + dm           - The DMPlex object
3583 - stratumValue - The requested depth
3584 
3585   Output Parameters:
3586 + start - The first point at this depth
3587 - end   - One beyond the last point at this depth
3588 
3589   Notes:
3590   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3591   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3592   higher dimension, e.g., "edges".
3593 
3594   Level: developer
3595 
3596 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3597 @*/
3598 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3599 {
3600   DMLabel        label;
3601   PetscInt       pStart, pEnd;
3602   PetscErrorCode ierr;
3603 
3604   PetscFunctionBegin;
3605   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3606   if (start) {PetscValidPointer(start, 3); *start = 0;}
3607   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3608   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3609   if (pStart == pEnd) PetscFunctionReturn(0);
3610   if (stratumValue < 0) {
3611     if (start) *start = pStart;
3612     if (end)   *end   = pEnd;
3613     PetscFunctionReturn(0);
3614   }
3615   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3616   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3617   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3618   PetscFunctionReturn(0);
3619 }
3620 
3621 /*@
3622   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3623 
3624   Not Collective
3625 
3626   Input Parameters:
3627 + dm           - The DMPlex object
3628 - stratumValue - The requested height
3629 
3630   Output Parameters:
3631 + start - The first point at this height
3632 - end   - One beyond the last point at this height
3633 
3634   Notes:
3635   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3636   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3637   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3638 
3639   Level: developer
3640 
3641 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3642 @*/
3643 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3644 {
3645   DMLabel        label;
3646   PetscInt       depth, pStart, pEnd;
3647   PetscErrorCode ierr;
3648 
3649   PetscFunctionBegin;
3650   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3651   if (start) {PetscValidPointer(start, 3); *start = 0;}
3652   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3653   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3654   if (pStart == pEnd) PetscFunctionReturn(0);
3655   if (stratumValue < 0) {
3656     if (start) *start = pStart;
3657     if (end)   *end   = pEnd;
3658     PetscFunctionReturn(0);
3659   }
3660   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3661   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3662   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3663   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3664   PetscFunctionReturn(0);
3665 }
3666 
3667 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3668 {
3669   PetscSection   section, s;
3670   Mat            m;
3671   PetscInt       maxHeight;
3672   PetscErrorCode ierr;
3673 
3674   PetscFunctionBegin;
3675   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3676   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3677   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3678   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3679   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3680   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3681   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3682   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3683   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3684   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3685   ierr = MatDestroy(&m);CHKERRQ(ierr);
3686 
3687   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3688   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3689   PetscFunctionReturn(0);
3690 }
3691 
3692 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3693 {
3694   Vec            coordsLocal;
3695   DM             coordsDM;
3696   PetscErrorCode ierr;
3697 
3698   PetscFunctionBegin;
3699   *field = NULL;
3700   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3701   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3702   if (coordsLocal && coordsDM) {
3703     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3704   }
3705   PetscFunctionReturn(0);
3706 }
3707 
3708 /*@C
3709   DMPlexGetConeSection - Return a section which describes the layout of cone data
3710 
3711   Not Collective
3712 
3713   Input Parameters:
3714 . dm        - The DMPlex object
3715 
3716   Output Parameter:
3717 . section - The PetscSection object
3718 
3719   Level: developer
3720 
3721 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3722 @*/
3723 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3724 {
3725   DM_Plex *mesh = (DM_Plex*) dm->data;
3726 
3727   PetscFunctionBegin;
3728   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3729   if (section) *section = mesh->coneSection;
3730   PetscFunctionReturn(0);
3731 }
3732 
3733 /*@C
3734   DMPlexGetSupportSection - Return a section which describes the layout of support data
3735 
3736   Not Collective
3737 
3738   Input Parameters:
3739 . dm        - The DMPlex object
3740 
3741   Output Parameter:
3742 . section - The PetscSection object
3743 
3744   Level: developer
3745 
3746 .seealso: DMPlexGetConeSection()
3747 @*/
3748 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3749 {
3750   DM_Plex *mesh = (DM_Plex*) dm->data;
3751 
3752   PetscFunctionBegin;
3753   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3754   if (section) *section = mesh->supportSection;
3755   PetscFunctionReturn(0);
3756 }
3757 
3758 /*@C
3759   DMPlexGetCones - Return cone data
3760 
3761   Not Collective
3762 
3763   Input Parameters:
3764 . dm        - The DMPlex object
3765 
3766   Output Parameter:
3767 . cones - The cone for each point
3768 
3769   Level: developer
3770 
3771 .seealso: DMPlexGetConeSection()
3772 @*/
3773 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3774 {
3775   DM_Plex *mesh = (DM_Plex*) dm->data;
3776 
3777   PetscFunctionBegin;
3778   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3779   if (cones) *cones = mesh->cones;
3780   PetscFunctionReturn(0);
3781 }
3782 
3783 /*@C
3784   DMPlexGetConeOrientations - Return cone orientation data
3785 
3786   Not Collective
3787 
3788   Input Parameters:
3789 . dm        - The DMPlex object
3790 
3791   Output Parameter:
3792 . coneOrientations - The cone orientation for each point
3793 
3794   Level: developer
3795 
3796 .seealso: DMPlexGetConeSection()
3797 @*/
3798 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3799 {
3800   DM_Plex *mesh = (DM_Plex*) dm->data;
3801 
3802   PetscFunctionBegin;
3803   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3804   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3805   PetscFunctionReturn(0);
3806 }
3807 
3808 /******************************** FEM Support **********************************/
3809 
3810 /*
3811  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3812  representing a line in the section.
3813 */
3814 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3815 {
3816   PetscErrorCode ierr;
3817 
3818   PetscFunctionBeginHot;
3819   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3820   if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3821     *k = 1;
3822   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3823     /* An order k SEM disc has k-1 dofs on an edge */
3824     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3825     *k = *k / *Nc + 1;
3826   }
3827   PetscFunctionReturn(0);
3828 }
3829 
3830 /*@
3831 
3832   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3833   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3834   section provided (or the section of the DM).
3835 
3836   Input Parameters:
3837 + dm      - The DM
3838 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3839 - section - The PetscSection to reorder, or NULL for the default section
3840 
3841   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3842   degree of the basis.
3843 
3844   Example:
3845   A typical interpolated single-quad mesh might order points as
3846 .vb
3847   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3848 
3849   v4 -- e6 -- v3
3850   |           |
3851   e7    c0    e8
3852   |           |
3853   v1 -- e5 -- v2
3854 .ve
3855 
3856   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3857   dofs in the order of points, e.g.,
3858 .vb
3859     c0 -> [0,1,2,3]
3860     v1 -> [4]
3861     ...
3862     e5 -> [8, 9]
3863 .ve
3864 
3865   which corresponds to the dofs
3866 .vb
3867     6   10  11  7
3868     13  2   3   15
3869     12  0   1   14
3870     4   8   9   5
3871 .ve
3872 
3873   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3874 .vb
3875   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3876 .ve
3877 
3878   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3879 .vb
3880    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3881 .ve
3882 
3883   Level: developer
3884 
3885 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlocalSection()
3886 @*/
3887 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3888 {
3889   DMLabel        label;
3890   PetscInt      *perm;
3891   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3892   PetscBool      vertexchart;
3893   PetscErrorCode ierr;
3894 
3895   PetscFunctionBegin;
3896   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3897   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3898   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3899   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3900   if (depth == 1) {eStart = point;}
3901   else if  (depth == dim) {
3902     const PetscInt *cone;
3903 
3904     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3905     if (dim == 2) eStart = cone[0];
3906     else if (dim == 3) {
3907       const PetscInt *cone2;
3908       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3909       eStart = cone2[0];
3910     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3911   } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3912   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
3913   {                             /* Determine whether the chart covers all points or just vertices. */
3914     PetscInt pStart,pEnd,cStart,cEnd;
3915     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3916     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3917     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3918     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3919   }
3920   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3921   if (dim < 1) PetscFunctionReturn(0);
3922   for (f = 0; f < Nf; ++f) {
3923     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3924     size += PetscPowInt(k+1, dim)*Nc;
3925   }
3926   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3927   for (f = 0; f < Nf; ++f) {
3928     switch (dim) {
3929     case 1:
3930       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3931       /*
3932         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3933         We want              [ vtx0; edge of length k-1; vtx1 ]
3934       */
3935       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3936       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3937       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3938       foffset = offset;
3939       break;
3940     case 2:
3941       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3942       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3943       /* The SEM order is
3944 
3945          v_lb, {e_b}, v_rb,
3946          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3947          v_lt, reverse {e_t}, v_rt
3948       */
3949       {
3950         const PetscInt of   = 0;
3951         const PetscInt oeb  = of   + PetscSqr(k-1);
3952         const PetscInt oer  = oeb  + (k-1);
3953         const PetscInt oet  = oer  + (k-1);
3954         const PetscInt oel  = oet  + (k-1);
3955         const PetscInt ovlb = oel  + (k-1);
3956         const PetscInt ovrb = ovlb + 1;
3957         const PetscInt ovrt = ovrb + 1;
3958         const PetscInt ovlt = ovrt + 1;
3959         PetscInt       o;
3960 
3961         /* bottom */
3962         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3963         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3964         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3965         /* middle */
3966         for (i = 0; i < k-1; ++i) {
3967           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3968           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3969           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3970         }
3971         /* top */
3972         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3973         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3974         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3975         foffset = offset;
3976       }
3977       break;
3978     case 3:
3979       /* The original hex closure is
3980 
3981          {c,
3982           f_b, f_t, f_f, f_b, f_r, f_l,
3983           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3984           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3985       */
3986       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3987       /* The SEM order is
3988          Bottom Slice
3989          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3990          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3991          v_blb, {e_bb}, v_brb,
3992 
3993          Middle Slice (j)
3994          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3995          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3996          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3997 
3998          Top Slice
3999          v_tlf, {e_tf}, v_trf,
4000          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
4001          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
4002       */
4003       {
4004         const PetscInt oc    = 0;
4005         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4006         const PetscInt oft   = ofb   + PetscSqr(k-1);
4007         const PetscInt off   = oft   + PetscSqr(k-1);
4008         const PetscInt ofk   = off   + PetscSqr(k-1);
4009         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4010         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4011         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4012         const PetscInt oebb  = oebl  + (k-1);
4013         const PetscInt oebr  = oebb  + (k-1);
4014         const PetscInt oebf  = oebr  + (k-1);
4015         const PetscInt oetf  = oebf  + (k-1);
4016         const PetscInt oetr  = oetf  + (k-1);
4017         const PetscInt oetb  = oetr  + (k-1);
4018         const PetscInt oetl  = oetb  + (k-1);
4019         const PetscInt oerf  = oetl  + (k-1);
4020         const PetscInt oelf  = oerf  + (k-1);
4021         const PetscInt oelb  = oelf  + (k-1);
4022         const PetscInt oerb  = oelb  + (k-1);
4023         const PetscInt ovblf = oerb  + (k-1);
4024         const PetscInt ovblb = ovblf + 1;
4025         const PetscInt ovbrb = ovblb + 1;
4026         const PetscInt ovbrf = ovbrb + 1;
4027         const PetscInt ovtlf = ovbrf + 1;
4028         const PetscInt ovtrf = ovtlf + 1;
4029         const PetscInt ovtrb = ovtrf + 1;
4030         const PetscInt ovtlb = ovtrb + 1;
4031         PetscInt       o, n;
4032 
4033         /* Bottom Slice */
4034         /*   bottom */
4035         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4036         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4037         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4038         /*   middle */
4039         for (i = 0; i < k-1; ++i) {
4040           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4041           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4042           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4043         }
4044         /*   top */
4045         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4046         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4047         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4048 
4049         /* Middle Slice */
4050         for (j = 0; j < k-1; ++j) {
4051           /*   bottom */
4052           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4053           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4054           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4055           /*   middle */
4056           for (i = 0; i < k-1; ++i) {
4057             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4058             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4059             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4060           }
4061           /*   top */
4062           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4063           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4064           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4065         }
4066 
4067         /* Top Slice */
4068         /*   bottom */
4069         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4070         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4071         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4072         /*   middle */
4073         for (i = 0; i < k-1; ++i) {
4074           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4075           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4076           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4077         }
4078         /*   top */
4079         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4080         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4081         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4082 
4083         foffset = offset;
4084       }
4085       break;
4086     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4087     }
4088   }
4089   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4090   /* Check permutation */
4091   {
4092     PetscInt *check;
4093 
4094     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4095     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4096     for (i = 0; i < size; ++i) check[perm[i]] = i;
4097     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4098     ierr = PetscFree(check);CHKERRQ(ierr);
4099   }
4100   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4101   PetscFunctionReturn(0);
4102 }
4103 
4104 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4105 {
4106   PetscDS        prob;
4107   PetscInt       depth, Nf, h;
4108   DMLabel        label;
4109   PetscErrorCode ierr;
4110 
4111   PetscFunctionBeginHot;
4112   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4113   Nf      = prob->Nf;
4114   label   = dm->depthLabel;
4115   *dspace = NULL;
4116   if (field < Nf) {
4117     PetscObject disc = prob->disc[field];
4118 
4119     if (disc->classid == PETSCFE_CLASSID) {
4120       PetscDualSpace dsp;
4121 
4122       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4123       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4124       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4125       h    = depth - 1 - h;
4126       if (h) {
4127         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4128       } else {
4129         *dspace = dsp;
4130       }
4131     }
4132   }
4133   PetscFunctionReturn(0);
4134 }
4135 
4136 
4137 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4138 {
4139   PetscScalar    *array, *vArray;
4140   const PetscInt *cone, *coneO;
4141   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4142   PetscErrorCode  ierr;
4143 
4144   PetscFunctionBeginHot;
4145   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4146   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4147   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4148   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4149   if (!values || !*values) {
4150     if ((point >= pStart) && (point < pEnd)) {
4151       PetscInt dof;
4152 
4153       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4154       size += dof;
4155     }
4156     for (p = 0; p < numPoints; ++p) {
4157       const PetscInt cp = cone[p];
4158       PetscInt       dof;
4159 
4160       if ((cp < pStart) || (cp >= pEnd)) continue;
4161       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4162       size += dof;
4163     }
4164     if (!values) {
4165       if (csize) *csize = size;
4166       PetscFunctionReturn(0);
4167     }
4168     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4169   } else {
4170     array = *values;
4171   }
4172   size = 0;
4173   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4174   if ((point >= pStart) && (point < pEnd)) {
4175     PetscInt     dof, off, d;
4176     PetscScalar *varr;
4177 
4178     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4179     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4180     varr = &vArray[off];
4181     for (d = 0; d < dof; ++d, ++offset) {
4182       array[offset] = varr[d];
4183     }
4184     size += dof;
4185   }
4186   for (p = 0; p < numPoints; ++p) {
4187     const PetscInt cp = cone[p];
4188     PetscInt       o  = coneO[p];
4189     PetscInt       dof, off, d;
4190     PetscScalar   *varr;
4191 
4192     if ((cp < pStart) || (cp >= pEnd)) continue;
4193     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4194     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4195     varr = &vArray[off];
4196     if (o >= 0) {
4197       for (d = 0; d < dof; ++d, ++offset) {
4198         array[offset] = varr[d];
4199       }
4200     } else {
4201       for (d = dof-1; d >= 0; --d, ++offset) {
4202         array[offset] = varr[d];
4203       }
4204     }
4205     size += dof;
4206   }
4207   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4208   if (!*values) {
4209     if (csize) *csize = size;
4210     *values = array;
4211   } else {
4212     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4213     *csize = size;
4214   }
4215   PetscFunctionReturn(0);
4216 }
4217 
4218 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4219 {
4220   const PetscInt *cla;
4221   PetscInt       np, *pts = NULL;
4222   PetscErrorCode ierr;
4223 
4224   PetscFunctionBeginHot;
4225   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4226   if (!*clPoints) {
4227     PetscInt pStart, pEnd, p, q;
4228 
4229     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4230     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4231     /* Compress out points not in the section */
4232     for (p = 0, q = 0; p < np; p++) {
4233       PetscInt r = pts[2*p];
4234       if ((r >= pStart) && (r < pEnd)) {
4235         pts[q*2]   = r;
4236         pts[q*2+1] = pts[2*p+1];
4237         ++q;
4238       }
4239     }
4240     np = q;
4241     cla = NULL;
4242   } else {
4243     PetscInt dof, off;
4244 
4245     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4246     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4247     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4248     np   = dof/2;
4249     pts  = (PetscInt *) &cla[off];
4250   }
4251   *numPoints = np;
4252   *points    = pts;
4253   *clp       = cla;
4254 
4255   PetscFunctionReturn(0);
4256 }
4257 
4258 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4259 {
4260   PetscErrorCode ierr;
4261 
4262   PetscFunctionBeginHot;
4263   if (!*clPoints) {
4264     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4265   } else {
4266     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4267   }
4268   *numPoints = 0;
4269   *points    = NULL;
4270   *clSec     = NULL;
4271   *clPoints  = NULL;
4272   *clp       = NULL;
4273   PetscFunctionReturn(0);
4274 }
4275 
4276 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4277 {
4278   PetscInt          offset = 0, p;
4279   const PetscInt    **perms = NULL;
4280   const PetscScalar **flips = NULL;
4281   PetscErrorCode    ierr;
4282 
4283   PetscFunctionBeginHot;
4284   *size = 0;
4285   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4286   for (p = 0; p < numPoints; p++) {
4287     const PetscInt    point = points[2*p];
4288     const PetscInt    *perm = perms ? perms[p] : NULL;
4289     const PetscScalar *flip = flips ? flips[p] : NULL;
4290     PetscInt          dof, off, d;
4291     const PetscScalar *varr;
4292 
4293     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4294     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4295     varr = &vArray[off];
4296     if (clperm) {
4297       if (perm) {
4298         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4299       } else {
4300         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4301       }
4302       if (flip) {
4303         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4304       }
4305     } else {
4306       if (perm) {
4307         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4308       } else {
4309         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4310       }
4311       if (flip) {
4312         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4313       }
4314     }
4315     offset += dof;
4316   }
4317   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4318   *size = offset;
4319   PetscFunctionReturn(0);
4320 }
4321 
4322 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4323 {
4324   PetscInt          offset = 0, f;
4325   PetscErrorCode    ierr;
4326 
4327   PetscFunctionBeginHot;
4328   *size = 0;
4329   for (f = 0; f < numFields; ++f) {
4330     PetscInt          p;
4331     const PetscInt    **perms = NULL;
4332     const PetscScalar **flips = NULL;
4333 
4334     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4335     for (p = 0; p < numPoints; p++) {
4336       const PetscInt    point = points[2*p];
4337       PetscInt          fdof, foff, b;
4338       const PetscScalar *varr;
4339       const PetscInt    *perm = perms ? perms[p] : NULL;
4340       const PetscScalar *flip = flips ? flips[p] : NULL;
4341 
4342       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4343       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4344       varr = &vArray[foff];
4345       if (clperm) {
4346         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4347         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4348         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4349       } else {
4350         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4351         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4352         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4353       }
4354       offset += fdof;
4355     }
4356     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4357   }
4358   *size = offset;
4359   PetscFunctionReturn(0);
4360 }
4361 
4362 /*@C
4363   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4364 
4365   Not collective
4366 
4367   Input Parameters:
4368 + dm - The DM
4369 . section - The section describing the layout in v, or NULL to use the default section
4370 . v - The local vector
4371 . point - The point in the DM
4372 . csize - The size of the input values array, or NULL
4373 - values - An array to use for the values, or NULL to have it allocated automatically
4374 
4375   Output Parameters:
4376 + csize - The number of values in the closure
4377 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4378 
4379 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4380 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4381 $ assembly function, and a user may already have allocated storage for this operation.
4382 $
4383 $ A typical use could be
4384 $
4385 $  values = NULL;
4386 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4387 $  for (cl = 0; cl < clSize; ++cl) {
4388 $    <Compute on closure>
4389 $  }
4390 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4391 $
4392 $ or
4393 $
4394 $  PetscMalloc1(clMaxSize, &values);
4395 $  for (p = pStart; p < pEnd; ++p) {
4396 $    clSize = clMaxSize;
4397 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4398 $    for (cl = 0; cl < clSize; ++cl) {
4399 $      <Compute on closure>
4400 $    }
4401 $  }
4402 $  PetscFree(values);
4403 
4404   Fortran Notes:
4405   Since it returns an array, this routine is only available in Fortran 90, and you must
4406   include petsc.h90 in your code.
4407 
4408   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4409 
4410   Level: intermediate
4411 
4412 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4413 @*/
4414 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4415 {
4416   PetscSection       clSection;
4417   IS                 clPoints;
4418   PetscScalar       *array;
4419   const PetscScalar *vArray;
4420   PetscInt          *points = NULL;
4421   const PetscInt    *clp, *perm;
4422   PetscInt           depth, numFields, numPoints, size;
4423   PetscErrorCode     ierr;
4424 
4425   PetscFunctionBeginHot;
4426   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4427   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4428   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4429   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4430   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4431   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4432   if (depth == 1 && numFields < 2) {
4433     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4434     PetscFunctionReturn(0);
4435   }
4436   /* Get points */
4437   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4438   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4439   /* Get array */
4440   if (!values || !*values) {
4441     PetscInt asize = 0, dof, p;
4442 
4443     for (p = 0; p < numPoints*2; p += 2) {
4444       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4445       asize += dof;
4446     }
4447     if (!values) {
4448       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4449       if (csize) *csize = asize;
4450       PetscFunctionReturn(0);
4451     }
4452     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4453   } else {
4454     array = *values;
4455   }
4456   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4457   /* Get values */
4458   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4459   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4460   /* Cleanup points */
4461   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4462   /* Cleanup array */
4463   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4464   if (!*values) {
4465     if (csize) *csize = size;
4466     *values = array;
4467   } else {
4468     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4469     *csize = size;
4470   }
4471   PetscFunctionReturn(0);
4472 }
4473 
4474 /*@C
4475   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4476 
4477   Not collective
4478 
4479   Input Parameters:
4480 + dm - The DM
4481 . section - The section describing the layout in v, or NULL to use the default section
4482 . v - The local vector
4483 . point - The point in the DM
4484 . csize - The number of values in the closure, or NULL
4485 - values - The array of values, which is a borrowed array and should not be freed
4486 
4487   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4488 
4489   Fortran Notes:
4490   Since it returns an array, this routine is only available in Fortran 90, and you must
4491   include petsc.h90 in your code.
4492 
4493   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4494 
4495   Level: intermediate
4496 
4497 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4498 @*/
4499 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4500 {
4501   PetscInt       size = 0;
4502   PetscErrorCode ierr;
4503 
4504   PetscFunctionBegin;
4505   /* Should work without recalculating size */
4506   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4507   *values = NULL;
4508   PetscFunctionReturn(0);
4509 }
4510 
4511 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4512 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4513 
4514 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4515 {
4516   PetscInt        cdof;   /* The number of constraints on this point */
4517   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4518   PetscScalar    *a;
4519   PetscInt        off, cind = 0, k;
4520   PetscErrorCode  ierr;
4521 
4522   PetscFunctionBegin;
4523   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4524   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4525   a    = &array[off];
4526   if (!cdof || setBC) {
4527     if (clperm) {
4528       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4529       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4530     } else {
4531       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4532       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4533     }
4534   } else {
4535     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4536     if (clperm) {
4537       if (perm) {for (k = 0; k < dof; ++k) {
4538           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4539           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4540         }
4541       } else {
4542         for (k = 0; k < dof; ++k) {
4543           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4544           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4545         }
4546       }
4547     } else {
4548       if (perm) {
4549         for (k = 0; k < dof; ++k) {
4550           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4551           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4552         }
4553       } else {
4554         for (k = 0; k < dof; ++k) {
4555           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4556           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4557         }
4558       }
4559     }
4560   }
4561   PetscFunctionReturn(0);
4562 }
4563 
4564 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4565 {
4566   PetscInt        cdof;   /* The number of constraints on this point */
4567   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4568   PetscScalar    *a;
4569   PetscInt        off, cind = 0, k;
4570   PetscErrorCode  ierr;
4571 
4572   PetscFunctionBegin;
4573   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4574   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4575   a    = &array[off];
4576   if (cdof) {
4577     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4578     if (clperm) {
4579       if (perm) {
4580         for (k = 0; k < dof; ++k) {
4581           if ((cind < cdof) && (k == cdofs[cind])) {
4582             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4583             cind++;
4584           }
4585         }
4586       } else {
4587         for (k = 0; k < dof; ++k) {
4588           if ((cind < cdof) && (k == cdofs[cind])) {
4589             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4590             cind++;
4591           }
4592         }
4593       }
4594     } else {
4595       if (perm) {
4596         for (k = 0; k < dof; ++k) {
4597           if ((cind < cdof) && (k == cdofs[cind])) {
4598             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4599             cind++;
4600           }
4601         }
4602       } else {
4603         for (k = 0; k < dof; ++k) {
4604           if ((cind < cdof) && (k == cdofs[cind])) {
4605             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4606             cind++;
4607           }
4608         }
4609       }
4610     }
4611   }
4612   PetscFunctionReturn(0);
4613 }
4614 
4615 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4616 {
4617   PetscScalar    *a;
4618   PetscInt        fdof, foff, fcdof, foffset = *offset;
4619   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4620   PetscInt        cind = 0, b;
4621   PetscErrorCode  ierr;
4622 
4623   PetscFunctionBegin;
4624   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4625   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4626   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4627   a    = &array[foff];
4628   if (!fcdof || setBC) {
4629     if (clperm) {
4630       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4631       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4632     } else {
4633       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4634       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4635     }
4636   } else {
4637     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4638     if (clperm) {
4639       if (perm) {
4640         for (b = 0; b < fdof; b++) {
4641           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4642           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4643         }
4644       } else {
4645         for (b = 0; b < fdof; b++) {
4646           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4647           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4648         }
4649       }
4650     } else {
4651       if (perm) {
4652         for (b = 0; b < fdof; b++) {
4653           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4654           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4655         }
4656       } else {
4657         for (b = 0; b < fdof; b++) {
4658           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4659           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4660         }
4661       }
4662     }
4663   }
4664   *offset += fdof;
4665   PetscFunctionReturn(0);
4666 }
4667 
4668 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4669 {
4670   PetscScalar    *a;
4671   PetscInt        fdof, foff, fcdof, foffset = *offset;
4672   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4673   PetscInt        cind = 0, ncind = 0, b;
4674   PetscBool       ncSet, fcSet;
4675   PetscErrorCode  ierr;
4676 
4677   PetscFunctionBegin;
4678   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4679   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4680   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4681   a    = &array[foff];
4682   if (fcdof) {
4683     /* We just override fcdof and fcdofs with Ncc and comps */
4684     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4685     if (clperm) {
4686       if (perm) {
4687         if (comps) {
4688           for (b = 0; b < fdof; b++) {
4689             ncSet = fcSet = PETSC_FALSE;
4690             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4691             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4692             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4693           }
4694         } else {
4695           for (b = 0; b < fdof; b++) {
4696             if ((cind < fcdof) && (b == fcdofs[cind])) {
4697               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4698               ++cind;
4699             }
4700           }
4701         }
4702       } else {
4703         if (comps) {
4704           for (b = 0; b < fdof; b++) {
4705             ncSet = fcSet = PETSC_FALSE;
4706             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4707             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4708             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4709           }
4710         } else {
4711           for (b = 0; b < fdof; b++) {
4712             if ((cind < fcdof) && (b == fcdofs[cind])) {
4713               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4714               ++cind;
4715             }
4716           }
4717         }
4718       }
4719     } else {
4720       if (perm) {
4721         if (comps) {
4722           for (b = 0; b < fdof; b++) {
4723             ncSet = fcSet = PETSC_FALSE;
4724             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4725             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4726             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4727           }
4728         } else {
4729           for (b = 0; b < fdof; b++) {
4730             if ((cind < fcdof) && (b == fcdofs[cind])) {
4731               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4732               ++cind;
4733             }
4734           }
4735         }
4736       } else {
4737         if (comps) {
4738           for (b = 0; b < fdof; b++) {
4739             ncSet = fcSet = PETSC_FALSE;
4740             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4741             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4742             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4743           }
4744         } else {
4745           for (b = 0; b < fdof; b++) {
4746             if ((cind < fcdof) && (b == fcdofs[cind])) {
4747               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4748               ++cind;
4749             }
4750           }
4751         }
4752       }
4753     }
4754   }
4755   *offset += fdof;
4756   PetscFunctionReturn(0);
4757 }
4758 
4759 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4760 {
4761   PetscScalar    *array;
4762   const PetscInt *cone, *coneO;
4763   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4764   PetscErrorCode  ierr;
4765 
4766   PetscFunctionBeginHot;
4767   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4768   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4769   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4770   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4771   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4772   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4773     const PetscInt cp = !p ? point : cone[p-1];
4774     const PetscInt o  = !p ? 0     : coneO[p-1];
4775 
4776     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4777     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4778     /* ADD_VALUES */
4779     {
4780       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4781       PetscScalar    *a;
4782       PetscInt        cdof, coff, cind = 0, k;
4783 
4784       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4785       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4786       a    = &array[coff];
4787       if (!cdof) {
4788         if (o >= 0) {
4789           for (k = 0; k < dof; ++k) {
4790             a[k] += values[off+k];
4791           }
4792         } else {
4793           for (k = 0; k < dof; ++k) {
4794             a[k] += values[off+dof-k-1];
4795           }
4796         }
4797       } else {
4798         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4799         if (o >= 0) {
4800           for (k = 0; k < dof; ++k) {
4801             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4802             a[k] += values[off+k];
4803           }
4804         } else {
4805           for (k = 0; k < dof; ++k) {
4806             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4807             a[k] += values[off+dof-k-1];
4808           }
4809         }
4810       }
4811     }
4812   }
4813   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4814   PetscFunctionReturn(0);
4815 }
4816 
4817 /*@C
4818   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4819 
4820   Not collective
4821 
4822   Input Parameters:
4823 + dm - The DM
4824 . section - The section describing the layout in v, or NULL to use the default section
4825 . v - The local vector
4826 . point - The point in the DM
4827 . values - The array of values
4828 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4829          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4830 
4831   Fortran Notes:
4832   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4833 
4834   Level: intermediate
4835 
4836 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4837 @*/
4838 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4839 {
4840   PetscSection    clSection;
4841   IS              clPoints;
4842   PetscScalar    *array;
4843   PetscInt       *points = NULL;
4844   const PetscInt *clp, *clperm;
4845   PetscInt        depth, numFields, numPoints, p;
4846   PetscErrorCode  ierr;
4847 
4848   PetscFunctionBeginHot;
4849   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4850   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4851   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4852   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4853   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4854   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4855   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4856     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4857     PetscFunctionReturn(0);
4858   }
4859   /* Get points */
4860   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4861   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4862   /* Get array */
4863   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4864   /* Get values */
4865   if (numFields > 0) {
4866     PetscInt offset = 0, f;
4867     for (f = 0; f < numFields; ++f) {
4868       const PetscInt    **perms = NULL;
4869       const PetscScalar **flips = NULL;
4870 
4871       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4872       switch (mode) {
4873       case INSERT_VALUES:
4874         for (p = 0; p < numPoints; p++) {
4875           const PetscInt    point = points[2*p];
4876           const PetscInt    *perm = perms ? perms[p] : NULL;
4877           const PetscScalar *flip = flips ? flips[p] : NULL;
4878           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4879         } break;
4880       case INSERT_ALL_VALUES:
4881         for (p = 0; p < numPoints; p++) {
4882           const PetscInt    point = points[2*p];
4883           const PetscInt    *perm = perms ? perms[p] : NULL;
4884           const PetscScalar *flip = flips ? flips[p] : NULL;
4885           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4886         } break;
4887       case INSERT_BC_VALUES:
4888         for (p = 0; p < numPoints; p++) {
4889           const PetscInt    point = points[2*p];
4890           const PetscInt    *perm = perms ? perms[p] : NULL;
4891           const PetscScalar *flip = flips ? flips[p] : NULL;
4892           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4893         } break;
4894       case ADD_VALUES:
4895         for (p = 0; p < numPoints; p++) {
4896           const PetscInt    point = points[2*p];
4897           const PetscInt    *perm = perms ? perms[p] : NULL;
4898           const PetscScalar *flip = flips ? flips[p] : NULL;
4899           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4900         } break;
4901       case ADD_ALL_VALUES:
4902         for (p = 0; p < numPoints; p++) {
4903           const PetscInt    point = points[2*p];
4904           const PetscInt    *perm = perms ? perms[p] : NULL;
4905           const PetscScalar *flip = flips ? flips[p] : NULL;
4906           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4907         } break;
4908       case ADD_BC_VALUES:
4909         for (p = 0; p < numPoints; p++) {
4910           const PetscInt    point = points[2*p];
4911           const PetscInt    *perm = perms ? perms[p] : NULL;
4912           const PetscScalar *flip = flips ? flips[p] : NULL;
4913           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4914         } break;
4915       default:
4916         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4917       }
4918       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4919     }
4920   } else {
4921     PetscInt dof, off;
4922     const PetscInt    **perms = NULL;
4923     const PetscScalar **flips = NULL;
4924 
4925     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4926     switch (mode) {
4927     case INSERT_VALUES:
4928       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4929         const PetscInt    point = points[2*p];
4930         const PetscInt    *perm = perms ? perms[p] : NULL;
4931         const PetscScalar *flip = flips ? flips[p] : NULL;
4932         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4933         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4934       } break;
4935     case INSERT_ALL_VALUES:
4936       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4937         const PetscInt    point = points[2*p];
4938         const PetscInt    *perm = perms ? perms[p] : NULL;
4939         const PetscScalar *flip = flips ? flips[p] : NULL;
4940         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4941         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4942       } break;
4943     case INSERT_BC_VALUES:
4944       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4945         const PetscInt    point = points[2*p];
4946         const PetscInt    *perm = perms ? perms[p] : NULL;
4947         const PetscScalar *flip = flips ? flips[p] : NULL;
4948         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4949         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4950       } break;
4951     case ADD_VALUES:
4952       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4953         const PetscInt    point = points[2*p];
4954         const PetscInt    *perm = perms ? perms[p] : NULL;
4955         const PetscScalar *flip = flips ? flips[p] : NULL;
4956         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4957         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4958       } break;
4959     case ADD_ALL_VALUES:
4960       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4961         const PetscInt    point = points[2*p];
4962         const PetscInt    *perm = perms ? perms[p] : NULL;
4963         const PetscScalar *flip = flips ? flips[p] : NULL;
4964         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4965         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4966       } break;
4967     case ADD_BC_VALUES:
4968       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4969         const PetscInt    point = points[2*p];
4970         const PetscInt    *perm = perms ? perms[p] : NULL;
4971         const PetscScalar *flip = flips ? flips[p] : NULL;
4972         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4973         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4974       } break;
4975     default:
4976       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4977     }
4978     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4979   }
4980   /* Cleanup points */
4981   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4982   /* Cleanup array */
4983   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4984   PetscFunctionReturn(0);
4985 }
4986 
4987 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4988 {
4989   PetscSection      clSection;
4990   IS                clPoints;
4991   PetscScalar       *array;
4992   PetscInt          *points = NULL;
4993   const PetscInt    *clp, *clperm;
4994   PetscInt          numFields, numPoints, p;
4995   PetscInt          offset = 0, f;
4996   PetscErrorCode    ierr;
4997 
4998   PetscFunctionBeginHot;
4999   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5000   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5001   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5002   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5003   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5004   /* Get points */
5005   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5006   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5007   /* Get array */
5008   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5009   /* Get values */
5010   for (f = 0; f < numFields; ++f) {
5011     const PetscInt    **perms = NULL;
5012     const PetscScalar **flips = NULL;
5013 
5014     if (!fieldActive[f]) {
5015       for (p = 0; p < numPoints*2; p += 2) {
5016         PetscInt fdof;
5017         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5018         offset += fdof;
5019       }
5020       continue;
5021     }
5022     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5023     switch (mode) {
5024     case INSERT_VALUES:
5025       for (p = 0; p < numPoints; p++) {
5026         const PetscInt    point = points[2*p];
5027         const PetscInt    *perm = perms ? perms[p] : NULL;
5028         const PetscScalar *flip = flips ? flips[p] : NULL;
5029         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5030       } break;
5031     case INSERT_ALL_VALUES:
5032       for (p = 0; p < numPoints; p++) {
5033         const PetscInt    point = points[2*p];
5034         const PetscInt    *perm = perms ? perms[p] : NULL;
5035         const PetscScalar *flip = flips ? flips[p] : NULL;
5036         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5037         } break;
5038     case INSERT_BC_VALUES:
5039       for (p = 0; p < numPoints; p++) {
5040         const PetscInt    point = points[2*p];
5041         const PetscInt    *perm = perms ? perms[p] : NULL;
5042         const PetscScalar *flip = flips ? flips[p] : NULL;
5043         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
5044       } break;
5045     case ADD_VALUES:
5046       for (p = 0; p < numPoints; p++) {
5047         const PetscInt    point = points[2*p];
5048         const PetscInt    *perm = perms ? perms[p] : NULL;
5049         const PetscScalar *flip = flips ? flips[p] : NULL;
5050         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5051       } break;
5052     case ADD_ALL_VALUES:
5053       for (p = 0; p < numPoints; p++) {
5054         const PetscInt    point = points[2*p];
5055         const PetscInt    *perm = perms ? perms[p] : NULL;
5056         const PetscScalar *flip = flips ? flips[p] : NULL;
5057         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5058       } break;
5059     default:
5060       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5061     }
5062     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5063   }
5064   /* Cleanup points */
5065   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5066   /* Cleanup array */
5067   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5068   PetscFunctionReturn(0);
5069 }
5070 
5071 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5072 {
5073   PetscMPIInt    rank;
5074   PetscInt       i, j;
5075   PetscErrorCode ierr;
5076 
5077   PetscFunctionBegin;
5078   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5079   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5080   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5081   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5082   numCIndices = numCIndices ? numCIndices : numRIndices;
5083   for (i = 0; i < numRIndices; i++) {
5084     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5085     for (j = 0; j < numCIndices; j++) {
5086 #if defined(PETSC_USE_COMPLEX)
5087       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5088 #else
5089       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5090 #endif
5091     }
5092     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5093   }
5094   PetscFunctionReturn(0);
5095 }
5096 
5097 /*
5098   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5099 
5100   Input Parameters:
5101 + section - The section for this data layout
5102 . point   - The point contributing dofs with these indices
5103 . off     - The global offset of this point
5104 . loff    - The local offset of each field
5105 . setBC   - The flag determining whether to include indices of bounsary values
5106 . perm    - A permutation of the dofs on this point, or NULL
5107 - indperm - A permutation of the entire indices array, or NULL
5108 
5109   Output Parameter:
5110 . indices - Indices for dofs on this point
5111 
5112   Level: developer
5113 
5114   Note: The indices could be local or global, depending on the value of 'off'.
5115 */
5116 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5117 {
5118   PetscInt        dof;   /* The number of unknowns on this point */
5119   PetscInt        cdof;  /* The number of constraints on this point */
5120   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5121   PetscInt        cind = 0, k;
5122   PetscErrorCode  ierr;
5123 
5124   PetscFunctionBegin;
5125   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5126   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5127   if (!cdof || setBC) {
5128     for (k = 0; k < dof; ++k) {
5129       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5130       const PetscInt ind    = indperm ? indperm[preind] : preind;
5131 
5132       indices[ind] = off + k;
5133     }
5134   } else {
5135     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5136     for (k = 0; k < dof; ++k) {
5137       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5138       const PetscInt ind    = indperm ? indperm[preind] : preind;
5139 
5140       if ((cind < cdof) && (k == cdofs[cind])) {
5141         /* Insert check for returning constrained indices */
5142         indices[ind] = -(off+k+1);
5143         ++cind;
5144       } else {
5145         indices[ind] = off+k-cind;
5146       }
5147     }
5148   }
5149   *loff += dof;
5150   PetscFunctionReturn(0);
5151 }
5152 
5153 /*
5154   This version only believes the point offset from the globalSection
5155 
5156  . off - The global offset of this point
5157 */
5158 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5159 {
5160   PetscInt       numFields, foff, f;
5161   PetscErrorCode ierr;
5162 
5163   PetscFunctionBegin;
5164   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5165   for (f = 0, foff = 0; f < numFields; ++f) {
5166     PetscInt        fdof, cfdof;
5167     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5168     PetscInt        cind = 0, b;
5169     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5170 
5171     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5172     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5173     if (!cfdof || setBC) {
5174       for (b = 0; b < fdof; ++b) {
5175         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5176         const PetscInt ind    = indperm ? indperm[preind] : preind;
5177 
5178         indices[ind] = off+foff+b;
5179       }
5180     } else {
5181       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5182       for (b = 0; b < fdof; ++b) {
5183         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5184         const PetscInt ind    = indperm ? indperm[preind] : preind;
5185 
5186         if ((cind < cfdof) && (b == fcdofs[cind])) {
5187           indices[ind] = -(off+foff+b+1);
5188           ++cind;
5189         } else {
5190           indices[ind] = off+foff+b-cind;
5191         }
5192       }
5193     }
5194     foff     += (setBC ? fdof : (fdof - cfdof));
5195     foffs[f] += fdof;
5196   }
5197   PetscFunctionReturn(0);
5198 }
5199 
5200 /*
5201   This version believes the globalSection offsets for each field, rather than just the point offset
5202 
5203  . foffs - The offset into 'indices' for each field, since it is segregated by field
5204 */
5205 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5206 {
5207   PetscInt       numFields, foff, f;
5208   PetscErrorCode ierr;
5209 
5210   PetscFunctionBegin;
5211   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5212   for (f = 0; f < numFields; ++f) {
5213     PetscInt        fdof, cfdof;
5214     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5215     PetscInt        cind = 0, b;
5216     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5217 
5218     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5219     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5220     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5221     if (!cfdof || setBC) {
5222       for (b = 0; b < fdof; ++b) {
5223         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5224         const PetscInt ind    = indperm ? indperm[preind] : preind;
5225 
5226         indices[ind] = foff+b;
5227       }
5228     } else {
5229       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5230       for (b = 0; b < fdof; ++b) {
5231         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5232         const PetscInt ind    = indperm ? indperm[preind] : preind;
5233 
5234         if ((cind < cfdof) && (b == fcdofs[cind])) {
5235           indices[ind] = -(foff+b+1);
5236           ++cind;
5237         } else {
5238           indices[ind] = foff+b-cind;
5239         }
5240       }
5241     }
5242     foffs[f] += fdof;
5243   }
5244   PetscFunctionReturn(0);
5245 }
5246 
5247 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5248 {
5249   Mat             cMat;
5250   PetscSection    aSec, cSec;
5251   IS              aIS;
5252   PetscInt        aStart = -1, aEnd = -1;
5253   const PetscInt  *anchors;
5254   PetscInt        numFields, f, p, q, newP = 0;
5255   PetscInt        newNumPoints = 0, newNumIndices = 0;
5256   PetscInt        *newPoints, *indices, *newIndices;
5257   PetscInt        maxAnchor, maxDof;
5258   PetscInt        newOffsets[32];
5259   PetscInt        *pointMatOffsets[32];
5260   PetscInt        *newPointOffsets[32];
5261   PetscScalar     *pointMat[32];
5262   PetscScalar     *newValues=NULL,*tmpValues;
5263   PetscBool       anyConstrained = PETSC_FALSE;
5264   PetscErrorCode  ierr;
5265 
5266   PetscFunctionBegin;
5267   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5268   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5269   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5270 
5271   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5272   /* if there are point-to-point constraints */
5273   if (aSec) {
5274     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5275     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5276     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5277     /* figure out how many points are going to be in the new element matrix
5278      * (we allow double counting, because it's all just going to be summed
5279      * into the global matrix anyway) */
5280     for (p = 0; p < 2*numPoints; p+=2) {
5281       PetscInt b    = points[p];
5282       PetscInt bDof = 0, bSecDof;
5283 
5284       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5285       if (!bSecDof) {
5286         continue;
5287       }
5288       if (b >= aStart && b < aEnd) {
5289         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5290       }
5291       if (bDof) {
5292         /* this point is constrained */
5293         /* it is going to be replaced by its anchors */
5294         PetscInt bOff, q;
5295 
5296         anyConstrained = PETSC_TRUE;
5297         newNumPoints  += bDof;
5298         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5299         for (q = 0; q < bDof; q++) {
5300           PetscInt a = anchors[bOff + q];
5301           PetscInt aDof;
5302 
5303           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5304           newNumIndices += aDof;
5305           for (f = 0; f < numFields; ++f) {
5306             PetscInt fDof;
5307 
5308             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5309             newOffsets[f+1] += fDof;
5310           }
5311         }
5312       }
5313       else {
5314         /* this point is not constrained */
5315         newNumPoints++;
5316         newNumIndices += bSecDof;
5317         for (f = 0; f < numFields; ++f) {
5318           PetscInt fDof;
5319 
5320           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5321           newOffsets[f+1] += fDof;
5322         }
5323       }
5324     }
5325   }
5326   if (!anyConstrained) {
5327     if (outNumPoints)  *outNumPoints  = 0;
5328     if (outNumIndices) *outNumIndices = 0;
5329     if (outPoints)     *outPoints     = NULL;
5330     if (outValues)     *outValues     = NULL;
5331     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5332     PetscFunctionReturn(0);
5333   }
5334 
5335   if (outNumPoints)  *outNumPoints  = newNumPoints;
5336   if (outNumIndices) *outNumIndices = newNumIndices;
5337 
5338   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5339 
5340   if (!outPoints && !outValues) {
5341     if (offsets) {
5342       for (f = 0; f <= numFields; f++) {
5343         offsets[f] = newOffsets[f];
5344       }
5345     }
5346     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5347     PetscFunctionReturn(0);
5348   }
5349 
5350   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5351 
5352   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5353 
5354   /* workspaces */
5355   if (numFields) {
5356     for (f = 0; f < numFields; f++) {
5357       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5358       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5359     }
5360   }
5361   else {
5362     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5363     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5364   }
5365 
5366   /* get workspaces for the point-to-point matrices */
5367   if (numFields) {
5368     PetscInt totalOffset, totalMatOffset;
5369 
5370     for (p = 0; p < numPoints; p++) {
5371       PetscInt b    = points[2*p];
5372       PetscInt bDof = 0, bSecDof;
5373 
5374       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5375       if (!bSecDof) {
5376         for (f = 0; f < numFields; f++) {
5377           newPointOffsets[f][p + 1] = 0;
5378           pointMatOffsets[f][p + 1] = 0;
5379         }
5380         continue;
5381       }
5382       if (b >= aStart && b < aEnd) {
5383         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5384       }
5385       if (bDof) {
5386         for (f = 0; f < numFields; f++) {
5387           PetscInt fDof, q, bOff, allFDof = 0;
5388 
5389           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5390           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5391           for (q = 0; q < bDof; q++) {
5392             PetscInt a = anchors[bOff + q];
5393             PetscInt aFDof;
5394 
5395             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5396             allFDof += aFDof;
5397           }
5398           newPointOffsets[f][p+1] = allFDof;
5399           pointMatOffsets[f][p+1] = fDof * allFDof;
5400         }
5401       }
5402       else {
5403         for (f = 0; f < numFields; f++) {
5404           PetscInt fDof;
5405 
5406           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5407           newPointOffsets[f][p+1] = fDof;
5408           pointMatOffsets[f][p+1] = 0;
5409         }
5410       }
5411     }
5412     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5413       newPointOffsets[f][0] = totalOffset;
5414       pointMatOffsets[f][0] = totalMatOffset;
5415       for (p = 0; p < numPoints; p++) {
5416         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5417         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5418       }
5419       totalOffset    = newPointOffsets[f][numPoints];
5420       totalMatOffset = pointMatOffsets[f][numPoints];
5421       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5422     }
5423   }
5424   else {
5425     for (p = 0; p < numPoints; p++) {
5426       PetscInt b    = points[2*p];
5427       PetscInt bDof = 0, bSecDof;
5428 
5429       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5430       if (!bSecDof) {
5431         newPointOffsets[0][p + 1] = 0;
5432         pointMatOffsets[0][p + 1] = 0;
5433         continue;
5434       }
5435       if (b >= aStart && b < aEnd) {
5436         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5437       }
5438       if (bDof) {
5439         PetscInt bOff, q, allDof = 0;
5440 
5441         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5442         for (q = 0; q < bDof; q++) {
5443           PetscInt a = anchors[bOff + q], aDof;
5444 
5445           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5446           allDof += aDof;
5447         }
5448         newPointOffsets[0][p+1] = allDof;
5449         pointMatOffsets[0][p+1] = bSecDof * allDof;
5450       }
5451       else {
5452         newPointOffsets[0][p+1] = bSecDof;
5453         pointMatOffsets[0][p+1] = 0;
5454       }
5455     }
5456     newPointOffsets[0][0] = 0;
5457     pointMatOffsets[0][0] = 0;
5458     for (p = 0; p < numPoints; p++) {
5459       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5460       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5461     }
5462     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5463   }
5464 
5465   /* output arrays */
5466   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5467 
5468   /* get the point-to-point matrices; construct newPoints */
5469   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5470   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5471   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5472   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5473   if (numFields) {
5474     for (p = 0, newP = 0; p < numPoints; p++) {
5475       PetscInt b    = points[2*p];
5476       PetscInt o    = points[2*p+1];
5477       PetscInt bDof = 0, bSecDof;
5478 
5479       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5480       if (!bSecDof) {
5481         continue;
5482       }
5483       if (b >= aStart && b < aEnd) {
5484         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5485       }
5486       if (bDof) {
5487         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5488 
5489         fStart[0] = 0;
5490         fEnd[0]   = 0;
5491         for (f = 0; f < numFields; f++) {
5492           PetscInt fDof;
5493 
5494           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5495           fStart[f+1] = fStart[f] + fDof;
5496           fEnd[f+1]   = fStart[f+1];
5497         }
5498         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5499         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5500 
5501         fAnchorStart[0] = 0;
5502         fAnchorEnd[0]   = 0;
5503         for (f = 0; f < numFields; f++) {
5504           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5505 
5506           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5507           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5508         }
5509         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5510         for (q = 0; q < bDof; q++) {
5511           PetscInt a = anchors[bOff + q], aOff;
5512 
5513           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5514           newPoints[2*(newP + q)]     = a;
5515           newPoints[2*(newP + q) + 1] = 0;
5516           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5517           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5518         }
5519         newP += bDof;
5520 
5521         if (outValues) {
5522           /* get the point-to-point submatrix */
5523           for (f = 0; f < numFields; f++) {
5524             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5525           }
5526         }
5527       }
5528       else {
5529         newPoints[2 * newP]     = b;
5530         newPoints[2 * newP + 1] = o;
5531         newP++;
5532       }
5533     }
5534   } else {
5535     for (p = 0; p < numPoints; p++) {
5536       PetscInt b    = points[2*p];
5537       PetscInt o    = points[2*p+1];
5538       PetscInt bDof = 0, bSecDof;
5539 
5540       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5541       if (!bSecDof) {
5542         continue;
5543       }
5544       if (b >= aStart && b < aEnd) {
5545         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5546       }
5547       if (bDof) {
5548         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5549 
5550         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5551         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5552 
5553         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5554         for (q = 0; q < bDof; q++) {
5555           PetscInt a = anchors[bOff + q], aOff;
5556 
5557           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5558 
5559           newPoints[2*(newP + q)]     = a;
5560           newPoints[2*(newP + q) + 1] = 0;
5561           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5562           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5563         }
5564         newP += bDof;
5565 
5566         /* get the point-to-point submatrix */
5567         if (outValues) {
5568           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5569         }
5570       }
5571       else {
5572         newPoints[2 * newP]     = b;
5573         newPoints[2 * newP + 1] = o;
5574         newP++;
5575       }
5576     }
5577   }
5578 
5579   if (outValues) {
5580     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5581     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5582     /* multiply constraints on the right */
5583     if (numFields) {
5584       for (f = 0; f < numFields; f++) {
5585         PetscInt oldOff = offsets[f];
5586 
5587         for (p = 0; p < numPoints; p++) {
5588           PetscInt cStart = newPointOffsets[f][p];
5589           PetscInt b      = points[2 * p];
5590           PetscInt c, r, k;
5591           PetscInt dof;
5592 
5593           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5594           if (!dof) {
5595             continue;
5596           }
5597           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5598             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5599             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5600 
5601             for (r = 0; r < numIndices; r++) {
5602               for (c = 0; c < nCols; c++) {
5603                 for (k = 0; k < dof; k++) {
5604                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5605                 }
5606               }
5607             }
5608           }
5609           else {
5610             /* copy this column as is */
5611             for (r = 0; r < numIndices; r++) {
5612               for (c = 0; c < dof; c++) {
5613                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5614               }
5615             }
5616           }
5617           oldOff += dof;
5618         }
5619       }
5620     }
5621     else {
5622       PetscInt oldOff = 0;
5623       for (p = 0; p < numPoints; p++) {
5624         PetscInt cStart = newPointOffsets[0][p];
5625         PetscInt b      = points[2 * p];
5626         PetscInt c, r, k;
5627         PetscInt dof;
5628 
5629         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5630         if (!dof) {
5631           continue;
5632         }
5633         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5634           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5635           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5636 
5637           for (r = 0; r < numIndices; r++) {
5638             for (c = 0; c < nCols; c++) {
5639               for (k = 0; k < dof; k++) {
5640                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5641               }
5642             }
5643           }
5644         }
5645         else {
5646           /* copy this column as is */
5647           for (r = 0; r < numIndices; r++) {
5648             for (c = 0; c < dof; c++) {
5649               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5650             }
5651           }
5652         }
5653         oldOff += dof;
5654       }
5655     }
5656 
5657     if (multiplyLeft) {
5658       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5659       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5660       /* multiply constraints transpose on the left */
5661       if (numFields) {
5662         for (f = 0; f < numFields; f++) {
5663           PetscInt oldOff = offsets[f];
5664 
5665           for (p = 0; p < numPoints; p++) {
5666             PetscInt rStart = newPointOffsets[f][p];
5667             PetscInt b      = points[2 * p];
5668             PetscInt c, r, k;
5669             PetscInt dof;
5670 
5671             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5672             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5673               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5674               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5675 
5676               for (r = 0; r < nRows; r++) {
5677                 for (c = 0; c < newNumIndices; c++) {
5678                   for (k = 0; k < dof; k++) {
5679                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5680                   }
5681                 }
5682               }
5683             }
5684             else {
5685               /* copy this row as is */
5686               for (r = 0; r < dof; r++) {
5687                 for (c = 0; c < newNumIndices; c++) {
5688                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5689                 }
5690               }
5691             }
5692             oldOff += dof;
5693           }
5694         }
5695       }
5696       else {
5697         PetscInt oldOff = 0;
5698 
5699         for (p = 0; p < numPoints; p++) {
5700           PetscInt rStart = newPointOffsets[0][p];
5701           PetscInt b      = points[2 * p];
5702           PetscInt c, r, k;
5703           PetscInt dof;
5704 
5705           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5706           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5707             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5708             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5709 
5710             for (r = 0; r < nRows; r++) {
5711               for (c = 0; c < newNumIndices; c++) {
5712                 for (k = 0; k < dof; k++) {
5713                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5714                 }
5715               }
5716             }
5717           }
5718           else {
5719             /* copy this row as is */
5720             for (r = 0; r < dof; r++) {
5721               for (c = 0; c < newNumIndices; c++) {
5722                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5723               }
5724             }
5725           }
5726           oldOff += dof;
5727         }
5728       }
5729 
5730       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5731     }
5732     else {
5733       newValues = tmpValues;
5734     }
5735   }
5736 
5737   /* clean up */
5738   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5739   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5740 
5741   if (numFields) {
5742     for (f = 0; f < numFields; f++) {
5743       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5744       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5745       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5746     }
5747   }
5748   else {
5749     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5750     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5751     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5752   }
5753   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5754 
5755   /* output */
5756   if (outPoints) {
5757     *outPoints = newPoints;
5758   }
5759   else {
5760     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5761   }
5762   if (outValues) {
5763     *outValues = newValues;
5764   }
5765   for (f = 0; f <= numFields; f++) {
5766     offsets[f] = newOffsets[f];
5767   }
5768   PetscFunctionReturn(0);
5769 }
5770 
5771 /*@C
5772   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5773 
5774   Not collective
5775 
5776   Input Parameters:
5777 + dm - The DM
5778 . section - The section describing the layout in v, or NULL to use the default section
5779 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5780 - point - The mesh point
5781 
5782   Output parameters:
5783 + numIndices - The number of indices
5784 . indices - The indices
5785 - outOffsets - Field offset if not NULL
5786 
5787   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5788 
5789   Level: advanced
5790 
5791 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5792 @*/
5793 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5794 {
5795   PetscSection    clSection;
5796   IS              clPoints;
5797   const PetscInt *clp, *clperm;
5798   const PetscInt  **perms[32] = {NULL};
5799   PetscInt       *points = NULL, *pointsNew;
5800   PetscInt        numPoints, numPointsNew;
5801   PetscInt        offsets[32];
5802   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5803   PetscErrorCode  ierr;
5804 
5805   PetscFunctionBegin;
5806   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5807   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5808   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5809   if (numIndices) PetscValidPointer(numIndices, 4);
5810   PetscValidPointer(indices, 5);
5811   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5812   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5813   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5814   /* Get points in closure */
5815   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5816   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5817   /* Get number of indices and indices per field */
5818   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5819     PetscInt dof, fdof;
5820 
5821     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5822     for (f = 0; f < Nf; ++f) {
5823       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5824       offsets[f+1] += fdof;
5825     }
5826     Nind += dof;
5827   }
5828   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5829   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5830   if (!Nf) offsets[1] = Nind;
5831   /* Get dual space symmetries */
5832   for (f = 0; f < PetscMax(1,Nf); f++) {
5833     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5834     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5835   }
5836   /* Correct for hanging node constraints */
5837   {
5838     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5839     if (numPointsNew) {
5840       for (f = 0; f < PetscMax(1,Nf); f++) {
5841         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5842         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5843       }
5844       for (f = 0; f < PetscMax(1,Nf); f++) {
5845         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5846         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5847       }
5848       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5849       numPoints = numPointsNew;
5850       Nind      = NindNew;
5851       points    = pointsNew;
5852     }
5853   }
5854   /* Calculate indices */
5855   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5856   if (Nf) {
5857     if (outOffsets) {
5858       PetscInt f;
5859 
5860       for (f = 0; f <= Nf; f++) {
5861         outOffsets[f] = offsets[f];
5862       }
5863     }
5864     for (p = 0; p < numPoints; p++) {
5865       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5866       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);
5867     }
5868   } else {
5869     for (p = 0, off = 0; p < numPoints; p++) {
5870       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5871 
5872       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5873       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);
5874     }
5875   }
5876   /* Cleanup points */
5877   for (f = 0; f < PetscMax(1,Nf); f++) {
5878     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5879     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5880   }
5881   if (numPointsNew) {
5882     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5883   } else {
5884     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5885   }
5886   if (numIndices) *numIndices = Nind;
5887   PetscFunctionReturn(0);
5888 }
5889 
5890 /*@C
5891   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5892 
5893   Not collective
5894 
5895   Input Parameters:
5896 + dm - The DM
5897 . section - The section describing the layout in v, or NULL to use the default section
5898 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5899 . point - The mesh point
5900 . numIndices - The number of indices
5901 . indices - The indices
5902 - outOffsets - Field offset if not NULL
5903 
5904   Level: advanced
5905 
5906 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5907 @*/
5908 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5909 {
5910   PetscErrorCode ierr;
5911 
5912   PetscFunctionBegin;
5913   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5914   PetscValidPointer(indices, 5);
5915   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5916   PetscFunctionReturn(0);
5917 }
5918 
5919 /*@C
5920   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5921 
5922   Not collective
5923 
5924   Input Parameters:
5925 + dm - The DM
5926 . section - The section describing the layout in v, or NULL to use the default section
5927 . globalSection - The section describing the layout in v, or NULL to use the default global section
5928 . A - The matrix
5929 . point - The point in the DM
5930 . values - The array of values
5931 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5932 
5933   Fortran Notes:
5934   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5935 
5936   Level: intermediate
5937 
5938 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5939 @*/
5940 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5941 {
5942   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5943   PetscSection        clSection;
5944   IS                  clPoints;
5945   PetscInt           *points = NULL, *newPoints;
5946   const PetscInt     *clp, *clperm;
5947   PetscInt           *indices;
5948   PetscInt            offsets[32];
5949   const PetscInt    **perms[32] = {NULL};
5950   const PetscScalar **flips[32] = {NULL};
5951   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5952   PetscScalar        *valCopy = NULL;
5953   PetscScalar        *newValues;
5954   PetscErrorCode      ierr;
5955 
5956   PetscFunctionBegin;
5957   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5958   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5959   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5960   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5961   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5962   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5963   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5964   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5965   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5966   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5967   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5968   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5969     PetscInt fdof;
5970 
5971     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5972     for (f = 0; f < numFields; ++f) {
5973       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5974       offsets[f+1] += fdof;
5975     }
5976     numIndices += dof;
5977   }
5978   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5979 
5980   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5981   /* Get symmetries */
5982   for (f = 0; f < PetscMax(1,numFields); f++) {
5983     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5984     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5985     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5986       PetscInt foffset = offsets[f];
5987 
5988       for (p = 0; p < numPoints; p++) {
5989         PetscInt point          = points[2*p], fdof;
5990         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5991 
5992         if (!numFields) {
5993           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5994         } else {
5995           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5996         }
5997         if (flip) {
5998           PetscInt i, j, k;
5999 
6000           if (!valCopy) {
6001             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6002             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6003             values = valCopy;
6004           }
6005           for (i = 0; i < fdof; i++) {
6006             PetscScalar fval = flip[i];
6007 
6008             for (k = 0; k < numIndices; k++) {
6009               valCopy[numIndices * (foffset + i) + k] *= fval;
6010               valCopy[numIndices * k + (foffset + i)] *= fval;
6011             }
6012           }
6013         }
6014         foffset += fdof;
6015       }
6016     }
6017   }
6018   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6019   if (newNumPoints) {
6020     if (valCopy) {
6021       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6022     }
6023     for (f = 0; f < PetscMax(1,numFields); f++) {
6024       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6025       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6026     }
6027     for (f = 0; f < PetscMax(1,numFields); f++) {
6028       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6029       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6030     }
6031     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6032     numPoints  = newNumPoints;
6033     numIndices = newNumIndices;
6034     points     = newPoints;
6035     values     = newValues;
6036   }
6037   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6038   if (numFields) {
6039     PetscBool useFieldOffsets;
6040 
6041     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6042     if (useFieldOffsets) {
6043       for (p = 0; p < numPoints; p++) {
6044         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
6045       }
6046     } else {
6047       for (p = 0; p < numPoints; p++) {
6048         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6049         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
6050       }
6051     }
6052   } else {
6053     for (p = 0, off = 0; p < numPoints; p++) {
6054       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6055       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6056       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
6057     }
6058   }
6059   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6060   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
6061   if (mesh->printFEM > 1) {
6062     PetscInt i;
6063     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6064     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6065     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6066   }
6067   if (ierr) {
6068     PetscMPIInt    rank;
6069     PetscErrorCode ierr2;
6070 
6071     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6072     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6073     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6074     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6075     CHKERRQ(ierr);
6076   }
6077   for (f = 0; f < PetscMax(1,numFields); f++) {
6078     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6079     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6080   }
6081   if (newNumPoints) {
6082     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6083     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6084   }
6085   else {
6086     if (valCopy) {
6087       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6088     }
6089     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6090   }
6091   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6092   PetscFunctionReturn(0);
6093 }
6094 
6095 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6096 {
6097   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6098   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6099   PetscInt       *cpoints = NULL;
6100   PetscInt       *findices, *cindices;
6101   const PetscInt *fclperm, *cclperm;
6102   PetscInt        foffsets[32], coffsets[32];
6103   CellRefiner     cellRefiner;
6104   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6105   PetscErrorCode  ierr;
6106 
6107   PetscFunctionBegin;
6108   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6109   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6110   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6111   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6112   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6113   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6114   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6115   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6116   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6117   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6118   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6119   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6120   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6121   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6122   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6123   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6124   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6125   /* Column indices */
6126   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6127   maxFPoints = numCPoints;
6128   /* Compress out points not in the section */
6129   /*   TODO: Squeeze out points with 0 dof as well */
6130   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6131   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6132     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6133       cpoints[q*2]   = cpoints[p];
6134       cpoints[q*2+1] = cpoints[p+1];
6135       ++q;
6136     }
6137   }
6138   numCPoints = q;
6139   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6140     PetscInt fdof;
6141 
6142     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6143     if (!dof) continue;
6144     for (f = 0; f < numFields; ++f) {
6145       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6146       coffsets[f+1] += fdof;
6147     }
6148     numCIndices += dof;
6149   }
6150   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6151   /* Row indices */
6152   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6153   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6154   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6155   for (r = 0, q = 0; r < numSubcells; ++r) {
6156     /* TODO Map from coarse to fine cells */
6157     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6158     /* Compress out points not in the section */
6159     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6160     for (p = 0; p < numFPoints*2; p += 2) {
6161       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6162         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6163         if (!dof) continue;
6164         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6165         if (s < q) continue;
6166         ftotpoints[q*2]   = fpoints[p];
6167         ftotpoints[q*2+1] = fpoints[p+1];
6168         ++q;
6169       }
6170     }
6171     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6172   }
6173   numFPoints = q;
6174   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6175     PetscInt fdof;
6176 
6177     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6178     if (!dof) continue;
6179     for (f = 0; f < numFields; ++f) {
6180       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6181       foffsets[f+1] += fdof;
6182     }
6183     numFIndices += dof;
6184   }
6185   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6186 
6187   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6188   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6189   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6190   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6191   if (numFields) {
6192     const PetscInt **permsF[32] = {NULL};
6193     const PetscInt **permsC[32] = {NULL};
6194 
6195     for (f = 0; f < numFields; f++) {
6196       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6197       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6198     }
6199     for (p = 0; p < numFPoints; p++) {
6200       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6201       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6202     }
6203     for (p = 0; p < numCPoints; p++) {
6204       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6205       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6206     }
6207     for (f = 0; f < numFields; f++) {
6208       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6209       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6210     }
6211   } else {
6212     const PetscInt **permsF = NULL;
6213     const PetscInt **permsC = NULL;
6214 
6215     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6216     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6217     for (p = 0, off = 0; p < numFPoints; p++) {
6218       const PetscInt *perm = permsF ? permsF[p] : NULL;
6219 
6220       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6221       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6222     }
6223     for (p = 0, off = 0; p < numCPoints; p++) {
6224       const PetscInt *perm = permsC ? permsC[p] : NULL;
6225 
6226       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6227       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6228     }
6229     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6230     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6231   }
6232   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6233   /* TODO: flips */
6234   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6235   if (ierr) {
6236     PetscMPIInt    rank;
6237     PetscErrorCode ierr2;
6238 
6239     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6240     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6241     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6242     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6243     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6244     CHKERRQ(ierr);
6245   }
6246   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6247   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6248   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6249   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6250   PetscFunctionReturn(0);
6251 }
6252 
6253 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6254 {
6255   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6256   PetscInt      *cpoints = NULL;
6257   PetscInt       foffsets[32], coffsets[32];
6258   const PetscInt *fclperm, *cclperm;
6259   CellRefiner    cellRefiner;
6260   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6261   PetscErrorCode ierr;
6262 
6263   PetscFunctionBegin;
6264   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6265   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6266   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6267   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6268   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6269   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6270   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6271   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6272   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6273   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6274   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6275   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6276   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6277   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6278   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6279   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6280   /* Column indices */
6281   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6282   maxFPoints = numCPoints;
6283   /* Compress out points not in the section */
6284   /*   TODO: Squeeze out points with 0 dof as well */
6285   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6286   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6287     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6288       cpoints[q*2]   = cpoints[p];
6289       cpoints[q*2+1] = cpoints[p+1];
6290       ++q;
6291     }
6292   }
6293   numCPoints = q;
6294   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6295     PetscInt fdof;
6296 
6297     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6298     if (!dof) continue;
6299     for (f = 0; f < numFields; ++f) {
6300       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6301       coffsets[f+1] += fdof;
6302     }
6303     numCIndices += dof;
6304   }
6305   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6306   /* Row indices */
6307   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6308   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6309   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6310   for (r = 0, q = 0; r < numSubcells; ++r) {
6311     /* TODO Map from coarse to fine cells */
6312     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6313     /* Compress out points not in the section */
6314     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6315     for (p = 0; p < numFPoints*2; p += 2) {
6316       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6317         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6318         if (!dof) continue;
6319         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6320         if (s < q) continue;
6321         ftotpoints[q*2]   = fpoints[p];
6322         ftotpoints[q*2+1] = fpoints[p+1];
6323         ++q;
6324       }
6325     }
6326     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6327   }
6328   numFPoints = q;
6329   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6330     PetscInt fdof;
6331 
6332     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6333     if (!dof) continue;
6334     for (f = 0; f < numFields; ++f) {
6335       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6336       foffsets[f+1] += fdof;
6337     }
6338     numFIndices += dof;
6339   }
6340   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6341 
6342   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6343   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6344   if (numFields) {
6345     const PetscInt **permsF[32] = {NULL};
6346     const PetscInt **permsC[32] = {NULL};
6347 
6348     for (f = 0; f < numFields; f++) {
6349       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6350       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6351     }
6352     for (p = 0; p < numFPoints; p++) {
6353       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6354       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6355     }
6356     for (p = 0; p < numCPoints; p++) {
6357       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6358       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6359     }
6360     for (f = 0; f < numFields; f++) {
6361       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6362       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6363     }
6364   } else {
6365     const PetscInt **permsF = NULL;
6366     const PetscInt **permsC = NULL;
6367 
6368     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6369     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6370     for (p = 0, off = 0; p < numFPoints; p++) {
6371       const PetscInt *perm = permsF ? permsF[p] : NULL;
6372 
6373       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6374       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6375     }
6376     for (p = 0, off = 0; p < numCPoints; p++) {
6377       const PetscInt *perm = permsC ? permsC[p] : NULL;
6378 
6379       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6380       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6381     }
6382     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6383     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6384   }
6385   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6386   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6387   PetscFunctionReturn(0);
6388 }
6389 
6390 /*@
6391   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6392 
6393   Input Parameter:
6394 . dm - The DMPlex object
6395 
6396   Output Parameters:
6397 + cMax - The first hybrid cell
6398 . fMax - The first hybrid face
6399 . eMax - The first hybrid edge
6400 - vMax - The first hybrid vertex
6401 
6402   Level: developer
6403 
6404 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6405 @*/
6406 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6407 {
6408   DM_Plex       *mesh = (DM_Plex*) dm->data;
6409   PetscInt       dim;
6410   PetscErrorCode ierr;
6411 
6412   PetscFunctionBegin;
6413   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6414   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6415   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6416   if (cMax) *cMax = mesh->hybridPointMax[dim];
6417   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6418   if (eMax) *eMax = mesh->hybridPointMax[1];
6419   if (vMax) *vMax = mesh->hybridPointMax[0];
6420   PetscFunctionReturn(0);
6421 }
6422 
6423 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6424 {
6425   IS             is, his;
6426   PetscInt       first = 0, stride;
6427   PetscBool      isStride;
6428   PetscErrorCode ierr;
6429 
6430   PetscFunctionBegin;
6431   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6432   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6433   if (isStride) {
6434     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6435   }
6436   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6437   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6438   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6439   ierr = ISDestroy(&his);CHKERRQ(ierr);
6440   ierr = ISDestroy(&is);CHKERRQ(ierr);
6441   PetscFunctionReturn(0);
6442 }
6443 
6444 /*@
6445   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6446 
6447   Input Parameters:
6448 + dm   - The DMPlex object
6449 . cMax - The first hybrid cell
6450 . fMax - The first hybrid face
6451 . eMax - The first hybrid edge
6452 - vMax - The first hybrid vertex
6453 
6454   Level: developer
6455 
6456 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6457 @*/
6458 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6459 {
6460   DM_Plex       *mesh = (DM_Plex*) dm->data;
6461   PetscInt       dim;
6462   PetscErrorCode ierr;
6463 
6464   PetscFunctionBegin;
6465   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6466   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6467   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6468   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6469   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6470   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6471   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6472   PetscFunctionReturn(0);
6473 }
6474 
6475 /*@C
6476   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6477 
6478   Input Parameter:
6479 . dm   - The DMPlex object
6480 
6481   Output Parameter:
6482 . cellHeight - The height of a cell
6483 
6484   Level: developer
6485 
6486 .seealso DMPlexSetVTKCellHeight()
6487 @*/
6488 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6489 {
6490   DM_Plex *mesh = (DM_Plex*) dm->data;
6491 
6492   PetscFunctionBegin;
6493   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6494   PetscValidPointer(cellHeight, 2);
6495   *cellHeight = mesh->vtkCellHeight;
6496   PetscFunctionReturn(0);
6497 }
6498 
6499 /*@C
6500   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6501 
6502   Input Parameters:
6503 + dm   - The DMPlex object
6504 - cellHeight - The height of a cell
6505 
6506   Level: developer
6507 
6508 .seealso DMPlexGetVTKCellHeight()
6509 @*/
6510 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6511 {
6512   DM_Plex *mesh = (DM_Plex*) dm->data;
6513 
6514   PetscFunctionBegin;
6515   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6516   mesh->vtkCellHeight = cellHeight;
6517   PetscFunctionReturn(0);
6518 }
6519 
6520 /* We can easily have a form that takes an IS instead */
6521 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6522 {
6523   PetscSection   section, globalSection;
6524   PetscInt      *numbers, p;
6525   PetscErrorCode ierr;
6526 
6527   PetscFunctionBegin;
6528   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6529   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6530   for (p = pStart; p < pEnd; ++p) {
6531     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6532   }
6533   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6534   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6535   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6536   for (p = pStart; p < pEnd; ++p) {
6537     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6538     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6539     else                       numbers[p-pStart] += shift;
6540   }
6541   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6542   if (globalSize) {
6543     PetscLayout layout;
6544     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6545     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6546     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6547   }
6548   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6549   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6550   PetscFunctionReturn(0);
6551 }
6552 
6553 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6554 {
6555   PetscInt       cellHeight, cStart, cEnd, cMax;
6556   PetscErrorCode ierr;
6557 
6558   PetscFunctionBegin;
6559   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6560   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6561   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6562   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6563   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6564   PetscFunctionReturn(0);
6565 }
6566 
6567 /*@
6568   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6569 
6570   Input Parameter:
6571 . dm   - The DMPlex object
6572 
6573   Output Parameter:
6574 . globalCellNumbers - Global cell numbers for all cells on this process
6575 
6576   Level: developer
6577 
6578 .seealso DMPlexGetVertexNumbering()
6579 @*/
6580 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6581 {
6582   DM_Plex       *mesh = (DM_Plex*) dm->data;
6583   PetscErrorCode ierr;
6584 
6585   PetscFunctionBegin;
6586   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6587   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6588   *globalCellNumbers = mesh->globalCellNumbers;
6589   PetscFunctionReturn(0);
6590 }
6591 
6592 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6593 {
6594   PetscInt       vStart, vEnd, vMax;
6595   PetscErrorCode ierr;
6596 
6597   PetscFunctionBegin;
6598   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6599   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6600   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6601   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6602   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6603   PetscFunctionReturn(0);
6604 }
6605 
6606 /*@
6607   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
6608 
6609   Input Parameter:
6610 . dm   - The DMPlex object
6611 
6612   Output Parameter:
6613 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6614 
6615   Level: developer
6616 
6617 .seealso DMPlexGetCellNumbering()
6618 @*/
6619 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6620 {
6621   DM_Plex       *mesh = (DM_Plex*) dm->data;
6622   PetscErrorCode ierr;
6623 
6624   PetscFunctionBegin;
6625   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6626   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6627   *globalVertexNumbers = mesh->globalVertexNumbers;
6628   PetscFunctionReturn(0);
6629 }
6630 
6631 /*@
6632   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6633 
6634   Input Parameter:
6635 . dm   - The DMPlex object
6636 
6637   Output Parameter:
6638 . globalPointNumbers - Global numbers for all points on this process
6639 
6640   Level: developer
6641 
6642 .seealso DMPlexGetCellNumbering()
6643 @*/
6644 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6645 {
6646   IS             nums[4];
6647   PetscInt       depths[4], gdepths[4], starts[4];
6648   PetscInt       depth, d, shift = 0;
6649   PetscErrorCode ierr;
6650 
6651   PetscFunctionBegin;
6652   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6653   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6654   /* For unstratified meshes use dim instead of depth */
6655   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6656   for (d = 0; d <= depth; ++d) {
6657     PetscInt end;
6658 
6659     depths[d] = depth-d;
6660     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6661     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6662   }
6663   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6664   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6665   for (d = 0; d <= depth; ++d) {
6666     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6667   }
6668   for (d = 0; d <= depth; ++d) {
6669     PetscInt pStart, pEnd, gsize;
6670 
6671     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6672     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6673     shift += gsize;
6674   }
6675   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6676   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6677   PetscFunctionReturn(0);
6678 }
6679 
6680 
6681 /*@
6682   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6683 
6684   Input Parameter:
6685 . dm - The DMPlex object
6686 
6687   Output Parameter:
6688 . ranks - The rank field
6689 
6690   Options Database Keys:
6691 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6692 
6693   Level: intermediate
6694 
6695 .seealso: DMView()
6696 @*/
6697 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6698 {
6699   DM             rdm;
6700   PetscFE        fe;
6701   PetscScalar   *r;
6702   PetscMPIInt    rank;
6703   PetscInt       dim, cStart, cEnd, c;
6704   PetscErrorCode ierr;
6705 
6706   PetscFunctionBeginUser;
6707   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6708   PetscValidPointer(ranks, 2);
6709   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6710   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6711   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6712   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6713   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6714   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6715   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6716   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6717   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6718   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6719   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6720   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6721   for (c = cStart; c < cEnd; ++c) {
6722     PetscScalar *lr;
6723 
6724     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6725     *lr = rank;
6726   }
6727   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6728   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6729   PetscFunctionReturn(0);
6730 }
6731 
6732 /*@
6733   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6734 
6735   Input Parameters:
6736 + dm    - The DMPlex
6737 - label - The DMLabel
6738 
6739   Output Parameter:
6740 . val - The label value field
6741 
6742   Options Database Keys:
6743 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6744 
6745   Level: intermediate
6746 
6747 .seealso: DMView()
6748 @*/
6749 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6750 {
6751   DM             rdm;
6752   PetscFE        fe;
6753   PetscScalar   *v;
6754   PetscInt       dim, cStart, cEnd, c;
6755   PetscErrorCode ierr;
6756 
6757   PetscFunctionBeginUser;
6758   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6759   PetscValidPointer(label, 2);
6760   PetscValidPointer(val, 3);
6761   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6762   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6763   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6764   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6765   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6766   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6767   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6768   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6769   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6770   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6771   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6772   for (c = cStart; c < cEnd; ++c) {
6773     PetscScalar *lv;
6774     PetscInt     cval;
6775 
6776     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6777     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6778     *lv = cval;
6779   }
6780   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6781   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6782   PetscFunctionReturn(0);
6783 }
6784 
6785 /*@
6786   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6787 
6788   Input Parameter:
6789 . dm - The DMPlex object
6790 
6791   Note: This is a useful diagnostic when creating meshes programmatically.
6792 
6793   Level: developer
6794 
6795 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6796 @*/
6797 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6798 {
6799   PetscSection    coneSection, supportSection;
6800   const PetscInt *cone, *support;
6801   PetscInt        coneSize, c, supportSize, s;
6802   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6803   PetscBool       storagecheck = PETSC_TRUE;
6804   PetscErrorCode  ierr;
6805 
6806   PetscFunctionBegin;
6807   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6808   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6809   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6810   /* Check that point p is found in the support of its cone points, and vice versa */
6811   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6812   for (p = pStart; p < pEnd; ++p) {
6813     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6814     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6815     for (c = 0; c < coneSize; ++c) {
6816       PetscBool dup = PETSC_FALSE;
6817       PetscInt  d;
6818       for (d = c-1; d >= 0; --d) {
6819         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6820       }
6821       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6822       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6823       for (s = 0; s < supportSize; ++s) {
6824         if (support[s] == p) break;
6825       }
6826       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6827         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6828         for (s = 0; s < coneSize; ++s) {
6829           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6830         }
6831         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6832         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6833         for (s = 0; s < supportSize; ++s) {
6834           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6835         }
6836         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6837         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6838         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6839       }
6840     }
6841     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6842     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6843     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6844     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6845     for (s = 0; s < supportSize; ++s) {
6846       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6847       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6848       for (c = 0; c < coneSize; ++c) {
6849         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6850         if (cone[c] != pp) { c = 0; break; }
6851         if (cone[c] == p) break;
6852       }
6853       if (c >= coneSize) {
6854         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6855         for (c = 0; c < supportSize; ++c) {
6856           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6857         }
6858         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6859         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6860         for (c = 0; c < coneSize; ++c) {
6861           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6862         }
6863         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6864         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6865       }
6866     }
6867   }
6868   if (storagecheck) {
6869     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6870     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6871     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6872   }
6873   PetscFunctionReturn(0);
6874 }
6875 
6876 /*@
6877   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6878 
6879   Input Parameters:
6880 + dm - The DMPlex object
6881 - cellHeight - Normally 0
6882 
6883   Note: This is a useful diagnostic when creating meshes programmatically.
6884   Currently applicable only to homogeneous simplex or tensor meshes.
6885 
6886   Level: developer
6887 
6888 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6889 @*/
6890 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6891 {
6892   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6893   PetscBool      isSimplex = PETSC_FALSE;
6894   PetscErrorCode ierr;
6895 
6896   PetscFunctionBegin;
6897   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6898   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6899   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6900   if (cStart < cEnd) {
6901     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6902     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6903   }
6904   switch (dim) {
6905   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6906   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6907   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6908   default:
6909     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6910   }
6911   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6912   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6913   cMax = cMax >= 0 ? cMax : cEnd;
6914   for (c = cStart; c < cMax; ++c) {
6915     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6916 
6917     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6918     for (cl = 0; cl < closureSize*2; cl += 2) {
6919       const PetscInt p = closure[cl];
6920       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6921     }
6922     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6923     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6924   }
6925   for (c = cMax; c < cEnd; ++c) {
6926     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6927 
6928     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6929     for (cl = 0; cl < closureSize*2; cl += 2) {
6930       const PetscInt p = closure[cl];
6931       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6932     }
6933     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6934     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6935   }
6936   PetscFunctionReturn(0);
6937 }
6938 
6939 /*@
6940   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6941 
6942   Input Parameters:
6943 + dm - The DMPlex object
6944 - cellHeight - Normally 0
6945 
6946   Note: This is a useful diagnostic when creating meshes programmatically.
6947 
6948   Level: developer
6949 
6950 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6951 @*/
6952 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6953 {
6954   PetscInt       pMax[4];
6955   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6956   PetscErrorCode ierr;
6957 
6958   PetscFunctionBegin;
6959   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6960   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6961   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6962   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6963   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6964   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6965     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6966     for (c = cStart; c < cEnd; ++c) {
6967       const PetscInt *cone, *ornt, *faces;
6968       PetscInt        numFaces, faceSize, coneSize,f;
6969       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6970 
6971       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6972       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6973       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6974       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6975       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6976       for (cl = 0; cl < closureSize*2; cl += 2) {
6977         const PetscInt p = closure[cl];
6978         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6979       }
6980       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6981       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6982       for (f = 0; f < numFaces; ++f) {
6983         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6984 
6985         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6986         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6987           const PetscInt p = fclosure[cl];
6988           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6989         }
6990         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6991         for (v = 0; v < fnumCorners; ++v) {
6992           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6993         }
6994         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6995       }
6996       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6997       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6998     }
6999   }
7000   PetscFunctionReturn(0);
7001 }
7002 
7003 /*@
7004   DMPlexCheckGeometry - Check the geometry of mesh cells
7005 
7006   Input Parameter:
7007 . dm - The DMPlex object
7008 
7009   Note: This is a useful diagnostic when creating meshes programmatically.
7010 
7011   Level: developer
7012 
7013 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
7014 @*/
7015 PetscErrorCode DMPlexCheckGeometry(DM dm)
7016 {
7017   PetscReal      detJ, J[9], refVol = 1.0;
7018   PetscReal      vol;
7019   PetscInt       dim, depth, d, cStart, cEnd, c, cMax;
7020   PetscErrorCode ierr;
7021 
7022   PetscFunctionBegin;
7023   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7024   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7025   for (d = 0; d < dim; ++d) refVol *= 2.0;
7026   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7027   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7028   cMax = cMax < 0 ? cEnd : cMax;
7029   for (c = cStart; c < cMax; ++c) {
7030     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7031     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7032     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7033     if (depth > 1) {
7034       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7035       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7036       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7037     }
7038   }
7039   PetscFunctionReturn(0);
7040 }
7041 
7042 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
7043 {
7044   PetscInt i,l,n;
7045   const PetscInt *cone;
7046   PetscErrorCode ierr;
7047 
7048   PetscFunctionBegin;
7049   *missingPoint = -1;
7050   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
7051   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
7052   for (i=0; i<n; i++) {
7053     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
7054     if (l < 0) {
7055       *missingPoint = cone[i];
7056       break;
7057     }
7058   }
7059   PetscFunctionReturn(0);
7060 }
7061 
7062 /*@
7063   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
7064 
7065   Input Parameters:
7066 . dm - The DMPlex object
7067 
7068   Notes:
7069   This is mainly intended for debugging/testing purposes.
7070 
7071   Level: developer
7072 
7073 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7074 @*/
7075 PetscErrorCode DMPlexCheckPointSF(DM dm)
7076 {
7077   PetscSF         sf;
7078   PetscInt        d,depth,i,nleaves,p,plo,phi,missingPoint;
7079   const PetscInt *locals;
7080   PetscErrorCode  ierr;
7081 
7082   PetscFunctionBegin;
7083   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7084   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7085   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
7086   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
7087 
7088   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7089   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
7090   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
7091   for (i=0; i<nleaves; i++) {
7092     p = locals[i];
7093     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
7094   }
7095 
7096   /* 2) if some point is in interface, then all its cone points must be also in interface  */
7097   for (i=0; i<nleaves; i++) {
7098     p = locals[i];
7099     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
7100     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
7101   }
7102   PetscFunctionReturn(0);
7103 }
7104 
7105 typedef struct cell_stats
7106 {
7107   PetscReal min, max, sum, squaresum;
7108   PetscInt  count;
7109 } cell_stats_t;
7110 
7111 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7112 {
7113   PetscInt i, N = *len;
7114 
7115   for (i = 0; i < N; i++) {
7116     cell_stats_t *A = (cell_stats_t *) a;
7117     cell_stats_t *B = (cell_stats_t *) b;
7118 
7119     B->min = PetscMin(A->min,B->min);
7120     B->max = PetscMax(A->max,B->max);
7121     B->sum += A->sum;
7122     B->squaresum += A->squaresum;
7123     B->count += A->count;
7124   }
7125 }
7126 
7127 /*@
7128   DMPlexCheckCellShape - Checks the Jacobian of the mapping and computes some minimal statistics.
7129 
7130   Input Parameters:
7131 + dm - The DMPlex object
7132 - output - If true, statistics will be displayed on stdout
7133 
7134   Note: This is mainly intended for debugging/testing purposes.
7135 
7136   Level: developer
7137 
7138 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7139 @*/
7140 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output)
7141 {
7142   PetscMPIInt    rank,size;
7143   PetscInt       dim, c, cStart, cEnd, cMax, count = 0;
7144   cell_stats_t   stats, globalStats;
7145   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7146   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7147   DM             dmCoarse;
7148   PetscErrorCode ierr;
7149 
7150   PetscFunctionBegin;
7151   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7152   stats.min   = PETSC_MAX_REAL;
7153   stats.max   = PETSC_MIN_REAL;
7154   stats.sum   = stats.squaresum = 0.;
7155   stats.count = 0;
7156 
7157   ierr = DMGetCoordinateDim(dm,&dim);CHKERRQ(ierr);
7158   ierr = PetscMalloc2(dim * dim, &J, dim * dim, &invJ);CHKERRQ(ierr);
7159   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7160   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7161   cMax = cMax < 0 ? cEnd : cMax;
7162   for (c = cStart; c < cMax; c++) {
7163     PetscInt  i;
7164     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7165 
7166     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7167     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7168     for (i = 0; i < dim * dim; i++) {
7169       frobJ    += J[i] * J[i];
7170       frobInvJ += invJ[i] * invJ[i];
7171     }
7172     cond2 = frobJ * frobInvJ;
7173     cond  = PetscSqrtReal(cond2);
7174 
7175     stats.min        = PetscMin(stats.min,cond);
7176     stats.max        = PetscMax(stats.max,cond);
7177     stats.sum       += cond;
7178     stats.squaresum += cond2;
7179     stats.count++;
7180   }
7181 
7182   ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
7183   if (size > 1) {
7184     PetscMPIInt   blockLengths[2] = {4,1};
7185     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7186     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7187     MPI_Op        statReduce;
7188 
7189     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7190     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7191     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7192     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7193     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7194     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7195   } else {
7196     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7197   }
7198 
7199   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
7200   if (!rank) {
7201     count = globalStats.count;
7202     min   = globalStats.min;
7203     max   = globalStats.max;
7204     mean  = globalStats.sum / globalStats.count;
7205     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7206   }
7207 
7208   if (output) {
7209     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7210   }
7211   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7212 
7213   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7214   if (dmCoarse) {
7215     PetscBool isplex;
7216 
7217     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7218     if (isplex) {
7219       ierr = DMPlexCheckCellShape(dmCoarse,output);CHKERRQ(ierr);
7220     }
7221   }
7222   PetscFunctionReturn(0);
7223 }
7224 
7225 /* Pointwise interpolation
7226      Just code FEM for now
7227      u^f = I u^c
7228      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7229      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7230      I_{ij} = psi^f_i phi^c_j
7231 */
7232 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7233 {
7234   PetscSection   gsc, gsf;
7235   PetscInt       m, n;
7236   void          *ctx;
7237   DM             cdm;
7238   PetscBool      regular, ismatis;
7239   PetscErrorCode ierr;
7240 
7241   PetscFunctionBegin;
7242   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7243   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7244   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7245   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7246 
7247   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7248   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7249   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7250   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7251   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7252 
7253   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7254   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7255   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7256   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7257   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7258   if (scaling) {
7259     /* Use naive scaling */
7260     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7261   }
7262   PetscFunctionReturn(0);
7263 }
7264 
7265 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7266 {
7267   PetscErrorCode ierr;
7268   VecScatter     ctx;
7269 
7270   PetscFunctionBegin;
7271   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7272   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7273   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7274   PetscFunctionReturn(0);
7275 }
7276 
7277 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7278 {
7279   PetscSection   gsc, gsf;
7280   PetscInt       m, n;
7281   void          *ctx;
7282   DM             cdm;
7283   PetscBool      regular;
7284   PetscErrorCode ierr;
7285 
7286   PetscFunctionBegin;
7287   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7288   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7289   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7290   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7291 
7292   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7293   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7294   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7295   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7296 
7297   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7298   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7299   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7300   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7301   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7302   PetscFunctionReturn(0);
7303 }
7304 
7305 /*@
7306   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7307 
7308   Input Parameter:
7309 . dm - The DMPlex object
7310 
7311   Output Parameter:
7312 . regular - The flag
7313 
7314   Level: intermediate
7315 
7316 .seealso: DMPlexSetRegularRefinement()
7317 @*/
7318 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7319 {
7320   PetscFunctionBegin;
7321   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7322   PetscValidPointer(regular, 2);
7323   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7324   PetscFunctionReturn(0);
7325 }
7326 
7327 /*@
7328   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7329 
7330   Input Parameters:
7331 + dm - The DMPlex object
7332 - regular - The flag
7333 
7334   Level: intermediate
7335 
7336 .seealso: DMPlexGetRegularRefinement()
7337 @*/
7338 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7339 {
7340   PetscFunctionBegin;
7341   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7342   ((DM_Plex *) dm->data)->regularRefinement = regular;
7343   PetscFunctionReturn(0);
7344 }
7345 
7346 /* anchors */
7347 /*@
7348   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7349   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7350 
7351   not collective
7352 
7353   Input Parameters:
7354 . dm - The DMPlex object
7355 
7356   Output Parameters:
7357 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7358 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7359 
7360 
7361   Level: intermediate
7362 
7363 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7364 @*/
7365 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7366 {
7367   DM_Plex *plex = (DM_Plex *)dm->data;
7368   PetscErrorCode ierr;
7369 
7370   PetscFunctionBegin;
7371   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7372   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7373   if (anchorSection) *anchorSection = plex->anchorSection;
7374   if (anchorIS) *anchorIS = plex->anchorIS;
7375   PetscFunctionReturn(0);
7376 }
7377 
7378 /*@
7379   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7380   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7381   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7382 
7383   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7384   DMGetConstraints() and filling in the entries in the constraint matrix.
7385 
7386   collective on dm
7387 
7388   Input Parameters:
7389 + dm - The DMPlex object
7390 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7391 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7392 
7393   The reference counts of anchorSection and anchorIS are incremented.
7394 
7395   Level: intermediate
7396 
7397 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7398 @*/
7399 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7400 {
7401   DM_Plex        *plex = (DM_Plex *)dm->data;
7402   PetscMPIInt    result;
7403   PetscErrorCode ierr;
7404 
7405   PetscFunctionBegin;
7406   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7407   if (anchorSection) {
7408     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7409     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7410     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7411   }
7412   if (anchorIS) {
7413     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7414     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7415     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7416   }
7417 
7418   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7419   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7420   plex->anchorSection = anchorSection;
7421 
7422   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7423   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7424   plex->anchorIS = anchorIS;
7425 
7426 #if defined(PETSC_USE_DEBUG)
7427   if (anchorIS && anchorSection) {
7428     PetscInt size, a, pStart, pEnd;
7429     const PetscInt *anchors;
7430 
7431     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7432     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7433     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7434     for (a = 0; a < size; a++) {
7435       PetscInt p;
7436 
7437       p = anchors[a];
7438       if (p >= pStart && p < pEnd) {
7439         PetscInt dof;
7440 
7441         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7442         if (dof) {
7443           PetscErrorCode ierr2;
7444 
7445           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7446           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7447         }
7448       }
7449     }
7450     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7451   }
7452 #endif
7453   /* reset the generic constraints */
7454   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7455   PetscFunctionReturn(0);
7456 }
7457 
7458 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7459 {
7460   PetscSection anchorSection;
7461   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7462   PetscErrorCode ierr;
7463 
7464   PetscFunctionBegin;
7465   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7466   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7467   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7468   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7469   if (numFields) {
7470     PetscInt f;
7471     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7472 
7473     for (f = 0; f < numFields; f++) {
7474       PetscInt numComp;
7475 
7476       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7477       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7478     }
7479   }
7480   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7481   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7482   pStart = PetscMax(pStart,sStart);
7483   pEnd   = PetscMin(pEnd,sEnd);
7484   pEnd   = PetscMax(pStart,pEnd);
7485   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7486   for (p = pStart; p < pEnd; p++) {
7487     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7488     if (dof) {
7489       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7490       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7491       for (f = 0; f < numFields; f++) {
7492         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7493         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7494       }
7495     }
7496   }
7497   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7498   PetscFunctionReturn(0);
7499 }
7500 
7501 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7502 {
7503   PetscSection aSec;
7504   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7505   const PetscInt *anchors;
7506   PetscInt numFields, f;
7507   IS aIS;
7508   PetscErrorCode ierr;
7509 
7510   PetscFunctionBegin;
7511   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7512   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7513   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7514   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7515   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7516   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7517   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7518   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7519   /* cSec will be a subset of aSec and section */
7520   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7521   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7522   i[0] = 0;
7523   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7524   for (p = pStart; p < pEnd; p++) {
7525     PetscInt rDof, rOff, r;
7526 
7527     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7528     if (!rDof) continue;
7529     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7530     if (numFields) {
7531       for (f = 0; f < numFields; f++) {
7532         annz = 0;
7533         for (r = 0; r < rDof; r++) {
7534           a = anchors[rOff + r];
7535           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7536           annz += aDof;
7537         }
7538         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7539         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7540         for (q = 0; q < dof; q++) {
7541           i[off + q + 1] = i[off + q] + annz;
7542         }
7543       }
7544     }
7545     else {
7546       annz = 0;
7547       for (q = 0; q < dof; q++) {
7548         a = anchors[off + q];
7549         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7550         annz += aDof;
7551       }
7552       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7553       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7554       for (q = 0; q < dof; q++) {
7555         i[off + q + 1] = i[off + q] + annz;
7556       }
7557     }
7558   }
7559   nnz = i[m];
7560   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7561   offset = 0;
7562   for (p = pStart; p < pEnd; p++) {
7563     if (numFields) {
7564       for (f = 0; f < numFields; f++) {
7565         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7566         for (q = 0; q < dof; q++) {
7567           PetscInt rDof, rOff, r;
7568           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7569           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7570           for (r = 0; r < rDof; r++) {
7571             PetscInt s;
7572 
7573             a = anchors[rOff + r];
7574             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7575             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7576             for (s = 0; s < aDof; s++) {
7577               j[offset++] = aOff + s;
7578             }
7579           }
7580         }
7581       }
7582     }
7583     else {
7584       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7585       for (q = 0; q < dof; q++) {
7586         PetscInt rDof, rOff, r;
7587         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7588         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7589         for (r = 0; r < rDof; r++) {
7590           PetscInt s;
7591 
7592           a = anchors[rOff + r];
7593           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7594           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7595           for (s = 0; s < aDof; s++) {
7596             j[offset++] = aOff + s;
7597           }
7598         }
7599       }
7600     }
7601   }
7602   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7603   ierr = PetscFree(i);CHKERRQ(ierr);
7604   ierr = PetscFree(j);CHKERRQ(ierr);
7605   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7606   PetscFunctionReturn(0);
7607 }
7608 
7609 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7610 {
7611   DM_Plex        *plex = (DM_Plex *)dm->data;
7612   PetscSection   anchorSection, section, cSec;
7613   Mat            cMat;
7614   PetscErrorCode ierr;
7615 
7616   PetscFunctionBegin;
7617   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7618   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7619   if (anchorSection) {
7620     PetscInt Nf;
7621 
7622     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
7623     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7624     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7625     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7626     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7627     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7628     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7629     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7630   }
7631   PetscFunctionReturn(0);
7632 }
7633 
7634 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7635 {
7636   IS             subis;
7637   PetscSection   section, subsection;
7638   PetscErrorCode ierr;
7639 
7640   PetscFunctionBegin;
7641   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
7642   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7643   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7644   /* Create subdomain */
7645   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7646   /* Create submodel */
7647   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7648   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7649   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7650   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
7651   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7652   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7653   /* Create map from submodel to global model */
7654   if (is) {
7655     PetscSection    sectionGlobal, subsectionGlobal;
7656     IS              spIS;
7657     const PetscInt *spmap;
7658     PetscInt       *subIndices;
7659     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7660     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7661 
7662     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7663     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7664     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7665     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7666     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7667     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7668     for (p = pStart; p < pEnd; ++p) {
7669       PetscInt gdof, pSubSize  = 0;
7670 
7671       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7672       if (gdof > 0) {
7673         for (f = 0; f < Nf; ++f) {
7674           PetscInt fdof, fcdof;
7675 
7676           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7677           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7678           pSubSize += fdof-fcdof;
7679         }
7680         subSize += pSubSize;
7681         if (pSubSize) {
7682           if (bs < 0) {
7683             bs = pSubSize;
7684           } else if (bs != pSubSize) {
7685             /* Layout does not admit a pointwise block size */
7686             bs = 1;
7687           }
7688         }
7689       }
7690     }
7691     /* Must have same blocksize on all procs (some might have no points) */
7692     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7693     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7694     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7695     else                            {bs = bsMinMax[0];}
7696     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7697     for (p = pStart; p < pEnd; ++p) {
7698       PetscInt gdof, goff;
7699 
7700       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7701       if (gdof > 0) {
7702         const PetscInt point = spmap[p];
7703 
7704         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7705         for (f = 0; f < Nf; ++f) {
7706           PetscInt fdof, fcdof, fc, f2, poff = 0;
7707 
7708           /* Can get rid of this loop by storing field information in the global section */
7709           for (f2 = 0; f2 < f; ++f2) {
7710             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7711             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7712             poff += fdof-fcdof;
7713           }
7714           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7715           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7716           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7717             subIndices[subOff] = goff+poff+fc;
7718           }
7719         }
7720       }
7721     }
7722     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7723     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7724     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7725     if (bs > 1) {
7726       /* We need to check that the block size does not come from non-contiguous fields */
7727       PetscInt i, j, set = 1;
7728       for (i = 0; i < subSize; i += bs) {
7729         for (j = 0; j < bs; ++j) {
7730           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7731         }
7732       }
7733       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7734     }
7735     /* Attach nullspace */
7736     for (f = 0; f < Nf; ++f) {
7737       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7738       if ((*subdm)->nullspaceConstructors[f]) break;
7739     }
7740     if (f < Nf) {
7741       MatNullSpace nullSpace;
7742 
7743       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7744       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7745       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7746     }
7747   }
7748   PetscFunctionReturn(0);
7749 }
7750