xref: /petsc/src/dm/impls/plex/plex.c (revision 3ca0882ebe6e6bc3c96f853b3244799ed8a662a6)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     char         lname[PETSC_MAX_PATH_LEN];
675     PetscReal    scale         = 2.0;
676     PetscReal    tikzscale     = 1.0;
677     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
678     double       tcoords[3];
679     PetscScalar *coords;
680     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
681     PetscMPIInt  rank, size;
682     char         **names, **colors, **lcolors;
683     PetscBool    plotEdges, flg, lflg;
684     PetscBT      wp = NULL;
685     PetscInt     pEnd, pStart;
686 
687     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
688     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
689     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
690     numLabels  = PetscMax(numLabels, 10);
691     numColors  = 10;
692     numLColors = 10;
693     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
694     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
695     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
696     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
697     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
698     if (!useLabels) numLabels = 0;
699     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
700     if (!useColors) {
701       numColors = 3;
702       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
703     }
704     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
705     if (!useColors) {
706       numLColors = 4;
707       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
708     }
709     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
710     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
711     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
712     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
713     if (depth < dim) plotEdges = PETSC_FALSE;
714 
715     /* filter points with labelvalue != labeldefaultvalue */
716     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
717     if (lflg) {
718       DMLabel lbl;
719 
720       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
721       if (lbl) {
722         PetscInt val, defval;
723 
724         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
725         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
726         for (c = pStart;  c < pEnd; c++) {
727           PetscInt *closure = NULL;
728           PetscInt  closureSize;
729 
730           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
731           if (val == defval) continue;
732 
733           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
734           for (p = 0; p < closureSize*2; p += 2) {
735             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
736           }
737           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
738         }
739       }
740     }
741 
742     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
743     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
744     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
745     ierr = PetscViewerASCIIPrintf(viewer, "\
746 \\documentclass[tikz]{standalone}\n\n\
747 \\usepackage{pgflibraryshapes}\n\
748 \\usetikzlibrary{backgrounds}\n\
749 \\usetikzlibrary{arrows}\n\
750 \\begin{document}\n");CHKERRQ(ierr);
751     if (size > 1) {
752       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
753       for (p = 0; p < size; ++p) {
754         if (p > 0 && p == size-1) {
755           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
756         } else if (p > 0) {
757           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
758         }
759         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
760       }
761       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
762     }
763     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", tikzscale);CHKERRQ(ierr);
764 
765     /* Plot vertices */
766     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
767     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
768     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
769     for (v = vStart; v < vEnd; ++v) {
770       PetscInt  off, dof, d;
771       PetscBool isLabeled = PETSC_FALSE;
772 
773       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
774       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
775       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
776       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
777       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
778       for (d = 0; d < dof; ++d) {
779         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
780         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
781       }
782       /* Rotate coordinates since PGF makes z point out of the page instead of up */
783       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
784       for (d = 0; d < dof; ++d) {
785         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
786         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", tcoords[d]);CHKERRQ(ierr);
787       }
788       color = colors[rank%numColors];
789       for (l = 0; l < numLabels; ++l) {
790         PetscInt val;
791         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
792         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
793       }
794       if (useNumbers) {
795         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
796       } else {
797         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
798       }
799     }
800     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
801     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
802     /* Plot cells */
803     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
804     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
805     if (dim == 3 || !useNumbers) {
806       for (e = eStart; e < eEnd; ++e) {
807         const PetscInt *cone;
808 
809         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
810         color = colors[rank%numColors];
811         for (l = 0; l < numLabels; ++l) {
812           PetscInt val;
813           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
814           if (val >= 0) {color = lcolors[l%numLColors]; break;}
815         }
816         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
817         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
818       }
819     } else {
820       for (c = cStart; c < cEnd; ++c) {
821         PetscInt *closure = NULL;
822         PetscInt  closureSize, firstPoint = -1;
823 
824         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
825         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
826         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
827         for (p = 0; p < closureSize*2; p += 2) {
828           const PetscInt point = closure[p];
829 
830           if ((point < vStart) || (point >= vEnd)) continue;
831           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
832           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
833           if (firstPoint < 0) firstPoint = point;
834         }
835         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
836         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
837         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
838       }
839     }
840     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
841     for (c = cStart; c < cEnd; ++c) {
842       double    ccoords[3] = {0.0, 0.0, 0.0};
843       PetscBool isLabeled  = PETSC_FALSE;
844       PetscInt *closure    = NULL;
845       PetscInt  closureSize, dof, d, n = 0;
846 
847       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
848       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
849       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
850       for (p = 0; p < closureSize*2; p += 2) {
851         const PetscInt point = closure[p];
852         PetscInt       off;
853 
854         if ((point < vStart) || (point >= vEnd)) continue;
855         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
856         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
857         for (d = 0; d < dof; ++d) {
858           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
859           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
860         }
861         /* Rotate coordinates since PGF makes z point out of the page instead of up */
862         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
863         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
864         ++n;
865       }
866       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
867       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
868       for (d = 0; d < dof; ++d) {
869         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
870         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", ccoords[d]);CHKERRQ(ierr);
871       }
872       color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (useNumbers) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
880       } else {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       }
883     }
884     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
885     /* Plot edges */
886     if (plotEdges) {
887       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
888       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
889       for (e = eStart; e < eEnd; ++e) {
890         const PetscInt *cone;
891         PetscInt        coneSize, offA, offB, dof, d;
892 
893         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
894         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
895         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
896         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
897         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
898         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
899         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
900         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
901         for (d = 0; d < dof; ++d) {
902           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
903           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
904         }
905         /* Rotate coordinates since PGF makes z point out of the page instead of up */
906         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
907         for (d = 0; d < dof; ++d) {
908           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
909           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
910         }
911         color = colors[rank%numColors];
912         for (l = 0; l < numLabels; ++l) {
913           PetscInt val;
914           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
915           if (val >= 0) {color = lcolors[l%numLColors]; break;}
916         }
917         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
918       }
919       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
920       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
921       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
922     }
923     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
924     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
925     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
926     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
927     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
928     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
929     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
930     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
931     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
932   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
933     Vec                    cown,acown;
934     VecScatter             sct;
935     ISLocalToGlobalMapping g2l;
936     IS                     gid,acis;
937     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
938     MPI_Group              ggroup,ngroup;
939     PetscScalar            *array,nid;
940     const PetscInt         *idxs;
941     PetscInt               *idxs2,*start,*adjacency,*work;
942     PetscInt64             lm[3],gm[3];
943     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
944     PetscMPIInt            d1,d2,rank;
945 
946     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
947     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
948 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
949     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
950 #endif
951     if (ncomm != MPI_COMM_NULL) {
952       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
953       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
954       d1   = 0;
955       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
956       nid  = d2;
957       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
958       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
959       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
960     } else nid = 0.0;
961 
962     /* Get connectivity */
963     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
964     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
965 
966     /* filter overlapped local cells */
967     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
968     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
969     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
970     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
971     for (c = cStart, cum = 0; c < cEnd; c++) {
972       if (idxs[c-cStart] < 0) continue;
973       idxs2[cum++] = idxs[c-cStart];
974     }
975     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
976     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
977     ierr = ISDestroy(&gid);CHKERRQ(ierr);
978     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
979 
980     /* support for node-aware cell locality */
981     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
982     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
983     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
984     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
985     for (c = 0; c < numVertices; c++) array[c] = nid;
986     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
987     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
988     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
989     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
990     ierr = ISDestroy(&acis);CHKERRQ(ierr);
991     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
992     ierr = VecDestroy(&cown);CHKERRQ(ierr);
993 
994     /* compute edgeCut */
995     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
996     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
997     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
998     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
999     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1000     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1001     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1002       PetscInt totl;
1003 
1004       totl = start[c+1]-start[c];
1005       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1006       for (i = 0; i < totl; i++) {
1007         if (work[i] < 0) {
1008           ect  += 1;
1009           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1010         }
1011       }
1012     }
1013     ierr  = PetscFree(work);CHKERRQ(ierr);
1014     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1015     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1016     lm[1] = -numVertices;
1017     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1018     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1019     lm[0] = ect; /* edgeCut */
1020     lm[1] = ectn; /* node-aware edgeCut */
1021     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1022     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1023     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1024 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
1025     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1026 #else
1027     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1028 #endif
1029     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1030     ierr  = PetscFree(start);CHKERRQ(ierr);
1031     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1032     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1033   } else {
1034     MPI_Comm    comm;
1035     PetscInt   *sizes, *hybsizes;
1036     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
1037     PetscInt    pStart, pEnd, p;
1038     PetscInt    numLabels, l;
1039     const char *name;
1040     PetscMPIInt size;
1041 
1042     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1043     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1044     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1045     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1046     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1047     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1048     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1049     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1050     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1051     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1052     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1053     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1054     if (depth == 1) {
1055       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1056       pEnd = pEnd - pStart;
1057       pMax[0] -= pStart;
1058       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1059       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1060       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1061       for (p = 0; p < size; ++p) {
1062         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1063         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1064       }
1065       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1066       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1067       pEnd = pEnd - pStart;
1068       pMax[depth] -= pStart;
1069       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1070       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1071       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1072       for (p = 0; p < size; ++p) {
1073         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1074         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1075       }
1076       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1077     } else {
1078       PetscMPIInt rank;
1079       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1080       for (d = 0; d <= dim; d++) {
1081         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1082         pEnd    -= pStart;
1083         pMax[d] -= pStart;
1084         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1085         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1086         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1087         for (p = 0; p < size; ++p) {
1088           if (!rank) {
1089             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1090             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1091           }
1092         }
1093         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1094       }
1095     }
1096     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1097     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1098     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1099     for (l = 0; l < numLabels; ++l) {
1100       DMLabel         label;
1101       const char     *name;
1102       IS              valueIS;
1103       const PetscInt *values;
1104       PetscInt        numValues, v;
1105 
1106       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1107       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1108       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1109       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1110       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1111       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1112       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1113       for (v = 0; v < numValues; ++v) {
1114         PetscInt size;
1115 
1116         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1117         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1118         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1119       }
1120       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1122       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1123       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1124     }
1125     /* If no fields are specified, people do not want to see adjacency */
1126     if (dm->Nf) {
1127       PetscInt f;
1128 
1129       for (f = 0; f < dm->Nf; ++f) {
1130         const char *name;
1131 
1132         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1133         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1134         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1135         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1136         if (dm->fields[f].adjacency[0]) {
1137           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1138           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1139         } else {
1140           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1141           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1142         }
1143         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1144       }
1145     }
1146     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1147     if (cdm) {
1148       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1149       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1150       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1151     }
1152   }
1153   PetscFunctionReturn(0);
1154 }
1155 
1156 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1157 {
1158   PetscDraw          draw;
1159   DM                 cdm;
1160   PetscSection       coordSection;
1161   Vec                coordinates;
1162   const PetscScalar *coords;
1163   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1164   PetscBool          isnull;
1165   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1166   PetscMPIInt        rank;
1167   PetscErrorCode     ierr;
1168 
1169   PetscFunctionBegin;
1170   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1171   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1172   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1173   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1174   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1175   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1176   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1177 
1178   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1179   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1180   if (isnull) PetscFunctionReturn(0);
1181   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1182 
1183   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1184   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1185   for (c = 0; c < N; c += dim) {
1186     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1187     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1188   }
1189   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1190   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1191   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1192   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1193   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1194 
1195   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1196   for (c = cStart; c < cEnd; ++c) {
1197     PetscScalar *coords = NULL;
1198     PetscInt     numCoords,coneSize;
1199 
1200     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1201     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202     switch (coneSize) {
1203     case 3:
1204       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1205                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1206                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1207                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1208       break;
1209     case 4:
1210       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1211                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1212                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1213                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1214                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1215       break;
1216     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1217     }
1218     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1219   }
1220   for (c = cStart; c < cEnd; ++c) {
1221     PetscScalar *coords = NULL;
1222     PetscInt     numCoords,coneSize;
1223 
1224     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1225     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1226     switch (coneSize) {
1227     case 3:
1228       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1229       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1230       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1231       break;
1232     case 4:
1233       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1234       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1235       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1236       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237       break;
1238     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1239     }
1240     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1241   }
1242   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1243   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1244   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1249 {
1250   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1251   char           name[PETSC_MAX_PATH_LEN];
1252   PetscErrorCode ierr;
1253 
1254   PetscFunctionBegin;
1255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1256   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1257   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1258   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1259   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1260   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1261   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1262   if (iascii) {
1263     PetscViewerFormat format;
1264     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1265     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1266       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1267     } else {
1268       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1269     }
1270   } else if (ishdf5) {
1271 #if defined(PETSC_HAVE_HDF5)
1272     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1273 #else
1274     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1275 #endif
1276   } else if (isvtk) {
1277     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1278   } else if (isdraw) {
1279     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1280   } else if (isglvis) {
1281     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1282   } else {
1283     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1284   }
1285   /* Optionally view the partition */
1286   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1287   if (flg) {
1288     Vec ranks;
1289     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1290     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1291     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1292   }
1293   /* Optionally view a label */
1294   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1295   if (flg) {
1296     DMLabel label;
1297     Vec     val;
1298 
1299     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1300     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1301     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1302     ierr = VecView(val, viewer);CHKERRQ(ierr);
1303     ierr = VecDestroy(&val);CHKERRQ(ierr);
1304   }
1305   PetscFunctionReturn(0);
1306 }
1307 
1308 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1309 {
1310   PetscBool      ishdf5;
1311   PetscErrorCode ierr;
1312 
1313   PetscFunctionBegin;
1314   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1315   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1316   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1317   if (ishdf5) {
1318 #if defined(PETSC_HAVE_HDF5)
1319     PetscViewerFormat format;
1320     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1321     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1322       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1323     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1324       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1325     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1326 #else
1327     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1328 #endif
1329   } else {
1330     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1331   }
1332   PetscFunctionReturn(0);
1333 }
1334 
1335 PetscErrorCode DMDestroy_Plex(DM dm)
1336 {
1337   DM_Plex       *mesh = (DM_Plex*) dm->data;
1338   PetscErrorCode ierr;
1339 
1340   PetscFunctionBegin;
1341   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1342   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1343   if (--mesh->refct > 0) PetscFunctionReturn(0);
1344   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1345   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1346   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1347   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1348   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1349   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1350   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1351   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1352   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1353   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1354   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1355   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1356   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1357   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1358   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1359   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1360   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1361   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1362   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1363   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1364   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1365   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1366   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1367   ierr = PetscFree(mesh);CHKERRQ(ierr);
1368   PetscFunctionReturn(0);
1369 }
1370 
1371 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1372 {
1373   PetscSection           sectionGlobal;
1374   PetscInt               bs = -1, mbs;
1375   PetscInt               localSize;
1376   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1377   PetscErrorCode         ierr;
1378   MatType                mtype;
1379   ISLocalToGlobalMapping ltog;
1380 
1381   PetscFunctionBegin;
1382   ierr = MatInitializePackage();CHKERRQ(ierr);
1383   mtype = dm->mattype;
1384   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1385   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1386   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1387   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1388   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1389   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1390   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1391   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1392   if (mbs > 1) bs = mbs;
1393   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1394   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1395   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1396   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1397   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1398   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1399   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1400   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1401   if (!isShell) {
1402     PetscSection subSection;
1403     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1404     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1405     PetscInt     pStart, pEnd, p, dof, cdof;
1406 
1407     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1408     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1409       PetscSection section;
1410       PetscInt     size;
1411 
1412       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1413       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1414       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1415       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1416     } else {
1417       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1418     }
1419     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1420     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1421       PetscInt bdof;
1422 
1423       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1424       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1425       dof  = dof < 0 ? -(dof+1) : dof;
1426       bdof = cdof && (dof-cdof) ? 1 : dof;
1427       if (dof) {
1428         if (bs < 0)          {bs = bdof;}
1429         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1430       }
1431       if (isMatIS) {
1432         PetscInt loff,c,off;
1433         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1434         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1435         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1436       }
1437     }
1438     /* Must have same blocksize on all procs (some might have no points) */
1439     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1440     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1441     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1442     else                            {bs = bsMinMax[0];}
1443     bs = PetscMax(1,bs);
1444     if (isMatIS) { /* Must reduce indices by blocksize */
1445       PetscInt l;
1446 
1447       lsize = lsize/bs;
1448       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1449       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1450     }
1451     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1452     if (isMatIS) {
1453       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1454     }
1455     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1456     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1457     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1458   }
1459   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1460   PetscFunctionReturn(0);
1461 }
1462 
1463 /*@
1464   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1465 
1466   Not collective
1467 
1468   Input Parameter:
1469 . mesh - The DMPlex
1470 
1471   Output Parameters:
1472 . subsection - The subdomain section
1473 
1474   Level: developer
1475 
1476 .seealso:
1477 @*/
1478 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1479 {
1480   DM_Plex       *mesh = (DM_Plex*) dm->data;
1481   PetscErrorCode ierr;
1482 
1483   PetscFunctionBegin;
1484   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1485   if (!mesh->subdomainSection) {
1486     PetscSection section;
1487     PetscSF      sf;
1488 
1489     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1490     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1491     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1492     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1493   }
1494   *subsection = mesh->subdomainSection;
1495   PetscFunctionReturn(0);
1496 }
1497 
1498 /*@
1499   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1500 
1501   Not collective
1502 
1503   Input Parameter:
1504 . mesh - The DMPlex
1505 
1506   Output Parameters:
1507 + pStart - The first mesh point
1508 - pEnd   - The upper bound for mesh points
1509 
1510   Level: beginner
1511 
1512 .seealso: DMPlexCreate(), DMPlexSetChart()
1513 @*/
1514 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1515 {
1516   DM_Plex       *mesh = (DM_Plex*) dm->data;
1517   PetscErrorCode ierr;
1518 
1519   PetscFunctionBegin;
1520   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1521   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1522   PetscFunctionReturn(0);
1523 }
1524 
1525 /*@
1526   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1527 
1528   Not collective
1529 
1530   Input Parameters:
1531 + mesh - The DMPlex
1532 . pStart - The first mesh point
1533 - pEnd   - The upper bound for mesh points
1534 
1535   Output Parameters:
1536 
1537   Level: beginner
1538 
1539 .seealso: DMPlexCreate(), DMPlexGetChart()
1540 @*/
1541 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1542 {
1543   DM_Plex       *mesh = (DM_Plex*) dm->data;
1544   PetscErrorCode ierr;
1545 
1546   PetscFunctionBegin;
1547   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1548   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1549   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1550   PetscFunctionReturn(0);
1551 }
1552 
1553 /*@
1554   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1555 
1556   Not collective
1557 
1558   Input Parameters:
1559 + mesh - The DMPlex
1560 - p - The point, which must lie in the chart set with DMPlexSetChart()
1561 
1562   Output Parameter:
1563 . size - The cone size for point p
1564 
1565   Level: beginner
1566 
1567 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1568 @*/
1569 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1570 {
1571   DM_Plex       *mesh = (DM_Plex*) dm->data;
1572   PetscErrorCode ierr;
1573 
1574   PetscFunctionBegin;
1575   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1576   PetscValidPointer(size, 3);
1577   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1578   PetscFunctionReturn(0);
1579 }
1580 
1581 /*@
1582   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1583 
1584   Not collective
1585 
1586   Input Parameters:
1587 + mesh - The DMPlex
1588 . p - The point, which must lie in the chart set with DMPlexSetChart()
1589 - size - The cone size for point p
1590 
1591   Output Parameter:
1592 
1593   Note:
1594   This should be called after DMPlexSetChart().
1595 
1596   Level: beginner
1597 
1598 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1599 @*/
1600 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1601 {
1602   DM_Plex       *mesh = (DM_Plex*) dm->data;
1603   PetscErrorCode ierr;
1604 
1605   PetscFunctionBegin;
1606   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1607   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1608 
1609   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1610   PetscFunctionReturn(0);
1611 }
1612 
1613 /*@
1614   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1615 
1616   Not collective
1617 
1618   Input Parameters:
1619 + mesh - The DMPlex
1620 . p - The point, which must lie in the chart set with DMPlexSetChart()
1621 - size - The additional cone size for point p
1622 
1623   Output Parameter:
1624 
1625   Note:
1626   This should be called after DMPlexSetChart().
1627 
1628   Level: beginner
1629 
1630 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1631 @*/
1632 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1633 {
1634   DM_Plex       *mesh = (DM_Plex*) dm->data;
1635   PetscInt       csize;
1636   PetscErrorCode ierr;
1637 
1638   PetscFunctionBegin;
1639   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1640   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1641   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1642 
1643   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1644   PetscFunctionReturn(0);
1645 }
1646 
1647 /*@C
1648   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1649 
1650   Not collective
1651 
1652   Input Parameters:
1653 + dm - The DMPlex
1654 - p - The point, which must lie in the chart set with DMPlexSetChart()
1655 
1656   Output Parameter:
1657 . cone - An array of points which are on the in-edges for point p
1658 
1659   Level: beginner
1660 
1661   Fortran Notes:
1662   Since it returns an array, this routine is only available in Fortran 90, and you must
1663   include petsc.h90 in your code.
1664   You must also call DMPlexRestoreCone() after you finish using the returned array.
1665   DMPlexRestoreCone() is not needed/available in C.
1666 
1667 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1668 @*/
1669 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1670 {
1671   DM_Plex       *mesh = (DM_Plex*) dm->data;
1672   PetscInt       off;
1673   PetscErrorCode ierr;
1674 
1675   PetscFunctionBegin;
1676   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1677   PetscValidPointer(cone, 3);
1678   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1679   *cone = &mesh->cones[off];
1680   PetscFunctionReturn(0);
1681 }
1682 
1683 /*@C
1684   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1685 
1686   Not collective
1687 
1688   Input Parameters:
1689 + dm - The DMPlex
1690 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1691 
1692   Output Parameter:
1693 + pConesSection - PetscSection describing the layout of pCones
1694 - pCones - An array of points which are on the in-edges for the point set p
1695 
1696   Level: intermediate
1697 
1698 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1699 @*/
1700 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1701 {
1702   PetscSection        cs, newcs;
1703   PetscInt            *cones;
1704   PetscInt            *newarr=NULL;
1705   PetscInt            n;
1706   PetscErrorCode      ierr;
1707 
1708   PetscFunctionBegin;
1709   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1710   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1711   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1712   if (pConesSection) *pConesSection = newcs;
1713   if (pCones) {
1714     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1715     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1716   }
1717   PetscFunctionReturn(0);
1718 }
1719 
1720 /*@
1721   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1722 
1723   Not collective
1724 
1725   Input Parameters:
1726 + dm - The DMPlex
1727 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1728 
1729   Output Parameter:
1730 . expandedPoints - An array of vertices recursively expanded from input points
1731 
1732   Level: advanced
1733 
1734   Notes:
1735   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1736   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1737 
1738 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1739 @*/
1740 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1741 {
1742   IS                  *expandedPointsAll;
1743   PetscInt            depth;
1744   PetscErrorCode      ierr;
1745 
1746   PetscFunctionBegin;
1747   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1748   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1749   PetscValidPointer(expandedPoints, 3);
1750   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1751   *expandedPoints = expandedPointsAll[0];
1752   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1753   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1754   PetscFunctionReturn(0);
1755 }
1756 
1757 /*@
1758   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1759 
1760   Not collective
1761 
1762   Input Parameters:
1763 + dm - The DMPlex
1764 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1765 
1766   Output Parameter:
1767 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1768 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1769 - sections - (optional) An array of sections which describe mappings from points to their cone points
1770 
1771   Level: advanced
1772 
1773   Notes:
1774   Like DMPlexGetConeTuple() but recursive.
1775 
1776   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1777   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1778 
1779   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1780   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1781   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1782 
1783 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1784 @*/
1785 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1786 {
1787   const PetscInt      *arr0=NULL, *cone=NULL;
1788   PetscInt            *arr=NULL, *newarr=NULL;
1789   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1790   IS                  *expandedPoints_;
1791   PetscSection        *sections_;
1792   PetscErrorCode      ierr;
1793 
1794   PetscFunctionBegin;
1795   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1796   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1797   if (depth) PetscValidIntPointer(depth, 3);
1798   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1799   if (sections) PetscValidPointer(sections, 5);
1800   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1801   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1802   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1803   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1804   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1805   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1806   for (d=depth_-1; d>=0; d--) {
1807     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1808     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1809     for (i=0; i<n; i++) {
1810       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1811       if (arr[i] >= start && arr[i] < end) {
1812         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1813         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1814       } else {
1815         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1816       }
1817     }
1818     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1819     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1820     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1821     for (i=0; i<n; i++) {
1822       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1823       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1824       if (cn > 1) {
1825         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1826         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1827       } else {
1828         newarr[co] = arr[i];
1829       }
1830     }
1831     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1832     arr = newarr;
1833     n = newn;
1834   }
1835   *depth = depth_;
1836   if (expandedPoints) *expandedPoints = expandedPoints_;
1837   else {
1838     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1839     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1840   }
1841   if (sections) *sections = sections_;
1842   else {
1843     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1844     ierr = PetscFree(sections_);CHKERRQ(ierr);
1845   }
1846   PetscFunctionReturn(0);
1847 }
1848 
1849 /*@
1850   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1851 
1852   Not collective
1853 
1854   Input Parameters:
1855 + dm - The DMPlex
1856 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1857 
1858   Output Parameter:
1859 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1860 . expandedPoints - (optional) An array of recursively expanded cones
1861 - sections - (optional) An array of sections which describe mappings from points to their cone points
1862 
1863   Level: advanced
1864 
1865   Notes:
1866   See DMPlexGetConeRecursive() for details.
1867 
1868 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1869 @*/
1870 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1871 {
1872   PetscInt            d, depth_;
1873   PetscErrorCode      ierr;
1874 
1875   PetscFunctionBegin;
1876   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1877   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1878   if (depth) *depth = 0;
1879   if (expandedPoints) {
1880     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1881     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1882   }
1883   if (sections)  {
1884     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1885     ierr = PetscFree(*sections);CHKERRQ(ierr);
1886   }
1887   PetscFunctionReturn(0);
1888 }
1889 
1890 /*@
1891   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1892 
1893   Not collective
1894 
1895   Input Parameters:
1896 + mesh - The DMPlex
1897 . p - The point, which must lie in the chart set with DMPlexSetChart()
1898 - cone - An array of points which are on the in-edges for point p
1899 
1900   Output Parameter:
1901 
1902   Note:
1903   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1904 
1905   Developer Note: Why not call this DMPlexSetCover()
1906 
1907   Level: beginner
1908 
1909 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1910 @*/
1911 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1912 {
1913   DM_Plex       *mesh = (DM_Plex*) dm->data;
1914   PetscInt       pStart, pEnd;
1915   PetscInt       dof, off, c;
1916   PetscErrorCode ierr;
1917 
1918   PetscFunctionBegin;
1919   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1920   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1921   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1922   if (dof) PetscValidPointer(cone, 3);
1923   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1924   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1925   for (c = 0; c < dof; ++c) {
1926     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1927     mesh->cones[off+c] = cone[c];
1928   }
1929   PetscFunctionReturn(0);
1930 }
1931 
1932 /*@C
1933   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1934 
1935   Not collective
1936 
1937   Input Parameters:
1938 + mesh - The DMPlex
1939 - p - The point, which must lie in the chart set with DMPlexSetChart()
1940 
1941   Output Parameter:
1942 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1943                     integer giving the prescription for cone traversal. If it is negative, the cone is
1944                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1945                     the index of the cone point on which to start.
1946 
1947   Level: beginner
1948 
1949   Fortran Notes:
1950   Since it returns an array, this routine is only available in Fortran 90, and you must
1951   include petsc.h90 in your code.
1952   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1953   DMPlexRestoreConeOrientation() is not needed/available in C.
1954 
1955 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1956 @*/
1957 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1958 {
1959   DM_Plex       *mesh = (DM_Plex*) dm->data;
1960   PetscInt       off;
1961   PetscErrorCode ierr;
1962 
1963   PetscFunctionBegin;
1964   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1965 #if defined(PETSC_USE_DEBUG)
1966   {
1967     PetscInt dof;
1968     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1969     if (dof) PetscValidPointer(coneOrientation, 3);
1970   }
1971 #endif
1972   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1973 
1974   *coneOrientation = &mesh->coneOrientations[off];
1975   PetscFunctionReturn(0);
1976 }
1977 
1978 /*@
1979   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1980 
1981   Not collective
1982 
1983   Input Parameters:
1984 + mesh - The DMPlex
1985 . p - The point, which must lie in the chart set with DMPlexSetChart()
1986 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1987                     integer giving the prescription for cone traversal. If it is negative, the cone is
1988                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1989                     the index of the cone point on which to start.
1990 
1991   Output Parameter:
1992 
1993   Note:
1994   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1995 
1996   Level: beginner
1997 
1998 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1999 @*/
2000 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2001 {
2002   DM_Plex       *mesh = (DM_Plex*) dm->data;
2003   PetscInt       pStart, pEnd;
2004   PetscInt       dof, off, c;
2005   PetscErrorCode ierr;
2006 
2007   PetscFunctionBegin;
2008   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2009   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2010   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2011   if (dof) PetscValidPointer(coneOrientation, 3);
2012   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2013   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2014   for (c = 0; c < dof; ++c) {
2015     PetscInt cdof, o = coneOrientation[c];
2016 
2017     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2018     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2019     mesh->coneOrientations[off+c] = o;
2020   }
2021   PetscFunctionReturn(0);
2022 }
2023 
2024 /*@
2025   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2026 
2027   Not collective
2028 
2029   Input Parameters:
2030 + mesh - The DMPlex
2031 . p - The point, which must lie in the chart set with DMPlexSetChart()
2032 . conePos - The local index in the cone where the point should be put
2033 - conePoint - The mesh point to insert
2034 
2035   Level: beginner
2036 
2037 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2038 @*/
2039 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2040 {
2041   DM_Plex       *mesh = (DM_Plex*) dm->data;
2042   PetscInt       pStart, pEnd;
2043   PetscInt       dof, off;
2044   PetscErrorCode ierr;
2045 
2046   PetscFunctionBegin;
2047   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2048   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2049   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2050   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2051   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2052   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2053   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2054   mesh->cones[off+conePos] = conePoint;
2055   PetscFunctionReturn(0);
2056 }
2057 
2058 /*@
2059   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2060 
2061   Not collective
2062 
2063   Input Parameters:
2064 + mesh - The DMPlex
2065 . p - The point, which must lie in the chart set with DMPlexSetChart()
2066 . conePos - The local index in the cone where the point should be put
2067 - coneOrientation - The point orientation to insert
2068 
2069   Level: beginner
2070 
2071 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2072 @*/
2073 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2074 {
2075   DM_Plex       *mesh = (DM_Plex*) dm->data;
2076   PetscInt       pStart, pEnd;
2077   PetscInt       dof, off;
2078   PetscErrorCode ierr;
2079 
2080   PetscFunctionBegin;
2081   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2082   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2083   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2084   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2085   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2086   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2087   mesh->coneOrientations[off+conePos] = coneOrientation;
2088   PetscFunctionReturn(0);
2089 }
2090 
2091 /*@
2092   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2093 
2094   Not collective
2095 
2096   Input Parameters:
2097 + mesh - The DMPlex
2098 - p - The point, which must lie in the chart set with DMPlexSetChart()
2099 
2100   Output Parameter:
2101 . size - The support size for point p
2102 
2103   Level: beginner
2104 
2105 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2106 @*/
2107 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2108 {
2109   DM_Plex       *mesh = (DM_Plex*) dm->data;
2110   PetscErrorCode ierr;
2111 
2112   PetscFunctionBegin;
2113   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2114   PetscValidPointer(size, 3);
2115   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2116   PetscFunctionReturn(0);
2117 }
2118 
2119 /*@
2120   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2121 
2122   Not collective
2123 
2124   Input Parameters:
2125 + mesh - The DMPlex
2126 . p - The point, which must lie in the chart set with DMPlexSetChart()
2127 - size - The support size for point p
2128 
2129   Output Parameter:
2130 
2131   Note:
2132   This should be called after DMPlexSetChart().
2133 
2134   Level: beginner
2135 
2136 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2137 @*/
2138 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2139 {
2140   DM_Plex       *mesh = (DM_Plex*) dm->data;
2141   PetscErrorCode ierr;
2142 
2143   PetscFunctionBegin;
2144   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2145   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2146 
2147   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2148   PetscFunctionReturn(0);
2149 }
2150 
2151 /*@C
2152   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2153 
2154   Not collective
2155 
2156   Input Parameters:
2157 + mesh - The DMPlex
2158 - p - The point, which must lie in the chart set with DMPlexSetChart()
2159 
2160   Output Parameter:
2161 . support - An array of points which are on the out-edges for point p
2162 
2163   Level: beginner
2164 
2165   Fortran Notes:
2166   Since it returns an array, this routine is only available in Fortran 90, and you must
2167   include petsc.h90 in your code.
2168   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2169   DMPlexRestoreSupport() is not needed/available in C.
2170 
2171 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2172 @*/
2173 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2174 {
2175   DM_Plex       *mesh = (DM_Plex*) dm->data;
2176   PetscInt       off;
2177   PetscErrorCode ierr;
2178 
2179   PetscFunctionBegin;
2180   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2181   PetscValidPointer(support, 3);
2182   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2183   *support = &mesh->supports[off];
2184   PetscFunctionReturn(0);
2185 }
2186 
2187 /*@
2188   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2189 
2190   Not collective
2191 
2192   Input Parameters:
2193 + mesh - The DMPlex
2194 . p - The point, which must lie in the chart set with DMPlexSetChart()
2195 - support - An array of points which are on the out-edges for point p
2196 
2197   Output Parameter:
2198 
2199   Note:
2200   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2201 
2202   Level: beginner
2203 
2204 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2205 @*/
2206 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2207 {
2208   DM_Plex       *mesh = (DM_Plex*) dm->data;
2209   PetscInt       pStart, pEnd;
2210   PetscInt       dof, off, c;
2211   PetscErrorCode ierr;
2212 
2213   PetscFunctionBegin;
2214   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2215   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2216   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2217   if (dof) PetscValidPointer(support, 3);
2218   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2219   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2220   for (c = 0; c < dof; ++c) {
2221     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2222     mesh->supports[off+c] = support[c];
2223   }
2224   PetscFunctionReturn(0);
2225 }
2226 
2227 /*@
2228   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2229 
2230   Not collective
2231 
2232   Input Parameters:
2233 + mesh - The DMPlex
2234 . p - The point, which must lie in the chart set with DMPlexSetChart()
2235 . supportPos - The local index in the cone where the point should be put
2236 - supportPoint - The mesh point to insert
2237 
2238   Level: beginner
2239 
2240 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2241 @*/
2242 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2243 {
2244   DM_Plex       *mesh = (DM_Plex*) dm->data;
2245   PetscInt       pStart, pEnd;
2246   PetscInt       dof, off;
2247   PetscErrorCode ierr;
2248 
2249   PetscFunctionBegin;
2250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2251   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2252   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2253   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2254   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2255   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2256   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2257   mesh->supports[off+supportPos] = supportPoint;
2258   PetscFunctionReturn(0);
2259 }
2260 
2261 /*@C
2262   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2263 
2264   Not collective
2265 
2266   Input Parameters:
2267 + mesh - The DMPlex
2268 . p - The point, which must lie in the chart set with DMPlexSetChart()
2269 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2270 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2271 
2272   Output Parameters:
2273 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2274 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2275 
2276   Note:
2277   If using internal storage (points is NULL on input), each call overwrites the last output.
2278 
2279   Fortran Notes:
2280   Since it returns an array, this routine is only available in Fortran 90, and you must
2281   include petsc.h90 in your code.
2282 
2283   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2284 
2285   Level: beginner
2286 
2287 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2288 @*/
2289 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2290 {
2291   DM_Plex        *mesh = (DM_Plex*) dm->data;
2292   PetscInt       *closure, *fifo;
2293   const PetscInt *tmp = NULL, *tmpO = NULL;
2294   PetscInt        tmpSize, t;
2295   PetscInt        depth       = 0, maxSize;
2296   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2297   PetscErrorCode  ierr;
2298 
2299   PetscFunctionBegin;
2300   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2301   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2302   /* This is only 1-level */
2303   if (useCone) {
2304     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2305     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2306     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2307   } else {
2308     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2309     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2310   }
2311   if (depth == 1) {
2312     if (*points) {
2313       closure = *points;
2314     } else {
2315       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2316       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2317     }
2318     closure[0] = p; closure[1] = 0;
2319     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2320       closure[closureSize]   = tmp[t];
2321       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2322     }
2323     if (numPoints) *numPoints = closureSize/2;
2324     if (points)    *points    = closure;
2325     PetscFunctionReturn(0);
2326   }
2327   {
2328     PetscInt c, coneSeries, s,supportSeries;
2329 
2330     c = mesh->maxConeSize;
2331     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2332     s = mesh->maxSupportSize;
2333     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2334     maxSize = 2*PetscMax(coneSeries,supportSeries);
2335   }
2336   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2337   if (*points) {
2338     closure = *points;
2339   } else {
2340     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2341   }
2342   closure[0] = p; closure[1] = 0;
2343   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2344     const PetscInt cp = tmp[t];
2345     const PetscInt co = tmpO ? tmpO[t] : 0;
2346 
2347     closure[closureSize]   = cp;
2348     closure[closureSize+1] = co;
2349     fifo[fifoSize]         = cp;
2350     fifo[fifoSize+1]       = co;
2351   }
2352   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2353   while (fifoSize - fifoStart) {
2354     const PetscInt q   = fifo[fifoStart];
2355     const PetscInt o   = fifo[fifoStart+1];
2356     const PetscInt rev = o >= 0 ? 0 : 1;
2357     const PetscInt off = rev ? -(o+1) : o;
2358 
2359     if (useCone) {
2360       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2361       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2362       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2363     } else {
2364       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2365       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2366       tmpO = NULL;
2367     }
2368     for (t = 0; t < tmpSize; ++t) {
2369       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2370       const PetscInt cp = tmp[i];
2371       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2372       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2373        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2374       PetscInt       co = tmpO ? tmpO[i] : 0;
2375       PetscInt       c;
2376 
2377       if (rev) {
2378         PetscInt childSize, coff;
2379         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2380         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2381         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2382       }
2383       /* Check for duplicate */
2384       for (c = 0; c < closureSize; c += 2) {
2385         if (closure[c] == cp) break;
2386       }
2387       if (c == closureSize) {
2388         closure[closureSize]   = cp;
2389         closure[closureSize+1] = co;
2390         fifo[fifoSize]         = cp;
2391         fifo[fifoSize+1]       = co;
2392         closureSize           += 2;
2393         fifoSize              += 2;
2394       }
2395     }
2396     fifoStart += 2;
2397   }
2398   if (numPoints) *numPoints = closureSize/2;
2399   if (points)    *points    = closure;
2400   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2401   PetscFunctionReturn(0);
2402 }
2403 
2404 /*@C
2405   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2406 
2407   Not collective
2408 
2409   Input Parameters:
2410 + mesh - The DMPlex
2411 . p - The point, which must lie in the chart set with DMPlexSetChart()
2412 . orientation - The orientation of the point
2413 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2414 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2415 
2416   Output Parameters:
2417 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2418 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2419 
2420   Note:
2421   If using internal storage (points is NULL on input), each call overwrites the last output.
2422 
2423   Fortran Notes:
2424   Since it returns an array, this routine is only available in Fortran 90, and you must
2425   include petsc.h90 in your code.
2426 
2427   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2428 
2429   Level: beginner
2430 
2431 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2432 @*/
2433 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2434 {
2435   DM_Plex        *mesh = (DM_Plex*) dm->data;
2436   PetscInt       *closure, *fifo;
2437   const PetscInt *tmp = NULL, *tmpO = NULL;
2438   PetscInt        tmpSize, t;
2439   PetscInt        depth       = 0, maxSize;
2440   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2441   PetscErrorCode  ierr;
2442 
2443   PetscFunctionBegin;
2444   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2445   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2446   /* This is only 1-level */
2447   if (useCone) {
2448     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2449     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2450     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2451   } else {
2452     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2453     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2454   }
2455   if (depth == 1) {
2456     if (*points) {
2457       closure = *points;
2458     } else {
2459       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2460       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2461     }
2462     closure[0] = p; closure[1] = ornt;
2463     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2464       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2465       closure[closureSize]   = tmp[i];
2466       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2467     }
2468     if (numPoints) *numPoints = closureSize/2;
2469     if (points)    *points    = closure;
2470     PetscFunctionReturn(0);
2471   }
2472   {
2473     PetscInt c, coneSeries, s,supportSeries;
2474 
2475     c = mesh->maxConeSize;
2476     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2477     s = mesh->maxSupportSize;
2478     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2479     maxSize = 2*PetscMax(coneSeries,supportSeries);
2480   }
2481   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2482   if (*points) {
2483     closure = *points;
2484   } else {
2485     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2486   }
2487   closure[0] = p; closure[1] = ornt;
2488   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2489     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2490     const PetscInt cp = tmp[i];
2491     PetscInt       co = tmpO ? tmpO[i] : 0;
2492 
2493     if (ornt < 0) {
2494       PetscInt childSize, coff;
2495       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2496       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2497       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2498     }
2499     closure[closureSize]   = cp;
2500     closure[closureSize+1] = co;
2501     fifo[fifoSize]         = cp;
2502     fifo[fifoSize+1]       = co;
2503   }
2504   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2505   while (fifoSize - fifoStart) {
2506     const PetscInt q   = fifo[fifoStart];
2507     const PetscInt o   = fifo[fifoStart+1];
2508     const PetscInt rev = o >= 0 ? 0 : 1;
2509     const PetscInt off = rev ? -(o+1) : o;
2510 
2511     if (useCone) {
2512       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2513       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2514       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2515     } else {
2516       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2517       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2518       tmpO = NULL;
2519     }
2520     for (t = 0; t < tmpSize; ++t) {
2521       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2522       const PetscInt cp = tmp[i];
2523       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2524       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2525        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2526       PetscInt       co = tmpO ? tmpO[i] : 0;
2527       PetscInt       c;
2528 
2529       if (rev) {
2530         PetscInt childSize, coff;
2531         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2532         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2533         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2534       }
2535       /* Check for duplicate */
2536       for (c = 0; c < closureSize; c += 2) {
2537         if (closure[c] == cp) break;
2538       }
2539       if (c == closureSize) {
2540         closure[closureSize]   = cp;
2541         closure[closureSize+1] = co;
2542         fifo[fifoSize]         = cp;
2543         fifo[fifoSize+1]       = co;
2544         closureSize           += 2;
2545         fifoSize              += 2;
2546       }
2547     }
2548     fifoStart += 2;
2549   }
2550   if (numPoints) *numPoints = closureSize/2;
2551   if (points)    *points    = closure;
2552   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2553   PetscFunctionReturn(0);
2554 }
2555 
2556 /*@C
2557   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2558 
2559   Not collective
2560 
2561   Input Parameters:
2562 + mesh - The DMPlex
2563 . p - The point, which must lie in the chart set with DMPlexSetChart()
2564 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2565 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2566 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2567 
2568   Note:
2569   If not using internal storage (points is not NULL on input), this call is unnecessary
2570 
2571   Fortran Notes:
2572   Since it returns an array, this routine is only available in Fortran 90, and you must
2573   include petsc.h90 in your code.
2574 
2575   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2576 
2577   Level: beginner
2578 
2579 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2580 @*/
2581 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2582 {
2583   PetscErrorCode ierr;
2584 
2585   PetscFunctionBegin;
2586   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2587   if (numPoints) PetscValidIntPointer(numPoints,4);
2588   if (points) PetscValidPointer(points,5);
2589   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2590   if (numPoints) *numPoints = 0;
2591   PetscFunctionReturn(0);
2592 }
2593 
2594 /*@
2595   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2596 
2597   Not collective
2598 
2599   Input Parameter:
2600 . mesh - The DMPlex
2601 
2602   Output Parameters:
2603 + maxConeSize - The maximum number of in-edges
2604 - maxSupportSize - The maximum number of out-edges
2605 
2606   Level: beginner
2607 
2608 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2609 @*/
2610 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2611 {
2612   DM_Plex *mesh = (DM_Plex*) dm->data;
2613 
2614   PetscFunctionBegin;
2615   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2616   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2617   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2618   PetscFunctionReturn(0);
2619 }
2620 
2621 PetscErrorCode DMSetUp_Plex(DM dm)
2622 {
2623   DM_Plex       *mesh = (DM_Plex*) dm->data;
2624   PetscInt       size;
2625   PetscErrorCode ierr;
2626 
2627   PetscFunctionBegin;
2628   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2629   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2630   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2631   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2632   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2633   if (mesh->maxSupportSize) {
2634     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2635     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2636     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2637   }
2638   PetscFunctionReturn(0);
2639 }
2640 
2641 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2642 {
2643   PetscErrorCode ierr;
2644 
2645   PetscFunctionBegin;
2646   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2647   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2648   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2649   if (dm->useNatural && dm->sfMigration) {
2650     PetscSF        sfMigrationInv,sfNatural;
2651     PetscSection   section, sectionSeq;
2652 
2653     (*subdm)->sfMigration = dm->sfMigration;
2654     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2655     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2656     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2657     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2658     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2659 
2660     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2661     (*subdm)->sfNatural = sfNatural;
2662     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2663     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2664   }
2665   PetscFunctionReturn(0);
2666 }
2667 
2668 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2669 {
2670   PetscErrorCode ierr;
2671   PetscInt       i = 0;
2672 
2673   PetscFunctionBegin;
2674   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2675   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2676   (*superdm)->useNatural = PETSC_FALSE;
2677   for (i = 0; i < len; i++){
2678     if (dms[i]->useNatural && dms[i]->sfMigration) {
2679       PetscSF        sfMigrationInv,sfNatural;
2680       PetscSection   section, sectionSeq;
2681 
2682       (*superdm)->sfMigration = dms[i]->sfMigration;
2683       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2684       (*superdm)->useNatural = PETSC_TRUE;
2685       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2686       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2687       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2688       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2689 
2690       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2691       (*superdm)->sfNatural = sfNatural;
2692       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2693       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2694       break;
2695     }
2696   }
2697   PetscFunctionReturn(0);
2698 }
2699 
2700 /*@
2701   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2702 
2703   Not collective
2704 
2705   Input Parameter:
2706 . mesh - The DMPlex
2707 
2708   Output Parameter:
2709 
2710   Note:
2711   This should be called after all calls to DMPlexSetCone()
2712 
2713   Level: beginner
2714 
2715 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2716 @*/
2717 PetscErrorCode DMPlexSymmetrize(DM dm)
2718 {
2719   DM_Plex       *mesh = (DM_Plex*) dm->data;
2720   PetscInt      *offsets;
2721   PetscInt       supportSize;
2722   PetscInt       pStart, pEnd, p;
2723   PetscErrorCode ierr;
2724 
2725   PetscFunctionBegin;
2726   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2727   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2728   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2729   /* Calculate support sizes */
2730   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2731   for (p = pStart; p < pEnd; ++p) {
2732     PetscInt dof, off, c;
2733 
2734     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2735     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2736     for (c = off; c < off+dof; ++c) {
2737       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2738     }
2739   }
2740   for (p = pStart; p < pEnd; ++p) {
2741     PetscInt dof;
2742 
2743     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2744 
2745     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2746   }
2747   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2748   /* Calculate supports */
2749   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2750   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2751   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2752   for (p = pStart; p < pEnd; ++p) {
2753     PetscInt dof, off, c;
2754 
2755     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2756     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2757     for (c = off; c < off+dof; ++c) {
2758       const PetscInt q = mesh->cones[c];
2759       PetscInt       offS;
2760 
2761       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2762 
2763       mesh->supports[offS+offsets[q]] = p;
2764       ++offsets[q];
2765     }
2766   }
2767   ierr = PetscFree(offsets);CHKERRQ(ierr);
2768   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2769   PetscFunctionReturn(0);
2770 }
2771 
2772 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2773 {
2774   IS             stratumIS;
2775   PetscErrorCode ierr;
2776 
2777   PetscFunctionBegin;
2778   if (pStart >= pEnd) PetscFunctionReturn(0);
2779 #if defined(PETSC_USE_DEBUG)
2780   {
2781     PetscInt  qStart, qEnd, numLevels, level;
2782     PetscBool overlap = PETSC_FALSE;
2783     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2784     for (level = 0; level < numLevels; level++) {
2785       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2786       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2787     }
2788     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2789   }
2790 #endif
2791   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2792   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2793   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2794   PetscFunctionReturn(0);
2795 }
2796 
2797 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2798 
2799 /*@
2800   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2801   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2802   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2803   the DAG.
2804 
2805   Collective on dm
2806 
2807   Input Parameter:
2808 . mesh - The DMPlex
2809 
2810   Output Parameter:
2811 
2812   Notes:
2813   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2814   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2815   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2816   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2817 
2818   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2819 
2820   Level: beginner
2821 
2822 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2823 @*/
2824 PetscErrorCode DMPlexStratify(DM dm)
2825 {
2826   DM_Plex       *mesh = (DM_Plex*) dm->data;
2827   DMLabel        label;
2828   PetscInt       pStart, pEnd, p;
2829   PetscInt       numRoots = 0, numLeaves = 0;
2830   PetscInt       cMax, fMax, eMax, vMax;
2831   PetscErrorCode ierr;
2832 
2833   PetscFunctionBegin;
2834   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2835   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2836 
2837   /* Create depth label */
2838   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2839   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2840   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2841 
2842   {
2843     /* Initialize roots and count leaves */
2844     PetscInt sMin = PETSC_MAX_INT;
2845     PetscInt sMax = PETSC_MIN_INT;
2846     PetscInt coneSize, supportSize;
2847 
2848     for (p = pStart; p < pEnd; ++p) {
2849       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2850       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2851       if (!coneSize && supportSize) {
2852         sMin = PetscMin(p, sMin);
2853         sMax = PetscMax(p, sMax);
2854         ++numRoots;
2855       } else if (!supportSize && coneSize) {
2856         ++numLeaves;
2857       } else if (!supportSize && !coneSize) {
2858         /* Isolated points */
2859         sMin = PetscMin(p, sMin);
2860         sMax = PetscMax(p, sMax);
2861       }
2862     }
2863     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2864   }
2865 
2866   if (numRoots + numLeaves == (pEnd - pStart)) {
2867     PetscInt sMin = PETSC_MAX_INT;
2868     PetscInt sMax = PETSC_MIN_INT;
2869     PetscInt coneSize, supportSize;
2870 
2871     for (p = pStart; p < pEnd; ++p) {
2872       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2873       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2874       if (!supportSize && coneSize) {
2875         sMin = PetscMin(p, sMin);
2876         sMax = PetscMax(p, sMax);
2877       }
2878     }
2879     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2880   } else {
2881     PetscInt level = 0;
2882     PetscInt qStart, qEnd, q;
2883 
2884     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2885     while (qEnd > qStart) {
2886       PetscInt sMin = PETSC_MAX_INT;
2887       PetscInt sMax = PETSC_MIN_INT;
2888 
2889       for (q = qStart; q < qEnd; ++q) {
2890         const PetscInt *support;
2891         PetscInt        supportSize, s;
2892 
2893         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2894         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2895         for (s = 0; s < supportSize; ++s) {
2896           sMin = PetscMin(support[s], sMin);
2897           sMax = PetscMax(support[s], sMax);
2898         }
2899       }
2900       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2901       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2902       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2903     }
2904   }
2905   { /* just in case there is an empty process */
2906     PetscInt numValues, maxValues = 0, v;
2907 
2908     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2909     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2910     for (v = numValues; v < maxValues; v++) {
2911       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2912     }
2913   }
2914   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2915 
2916   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2917   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2918     PetscInt dim;
2919     DMLabel  dimLabel;
2920 
2921     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2922     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2923     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2924     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2925     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2926     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2927     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2928   }
2929   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2930   PetscFunctionReturn(0);
2931 }
2932 
2933 /*@C
2934   DMPlexGetJoin - Get an array for the join of the set of points
2935 
2936   Not Collective
2937 
2938   Input Parameters:
2939 + dm - The DMPlex object
2940 . numPoints - The number of input points for the join
2941 - points - The input points
2942 
2943   Output Parameters:
2944 + numCoveredPoints - The number of points in the join
2945 - coveredPoints - The points in the join
2946 
2947   Level: intermediate
2948 
2949   Note: Currently, this is restricted to a single level join
2950 
2951   Fortran Notes:
2952   Since it returns an array, this routine is only available in Fortran 90, and you must
2953   include petsc.h90 in your code.
2954 
2955   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2956 
2957 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2958 @*/
2959 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2960 {
2961   DM_Plex       *mesh = (DM_Plex*) dm->data;
2962   PetscInt      *join[2];
2963   PetscInt       joinSize, i = 0;
2964   PetscInt       dof, off, p, c, m;
2965   PetscErrorCode ierr;
2966 
2967   PetscFunctionBegin;
2968   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2969   PetscValidIntPointer(points, 3);
2970   PetscValidIntPointer(numCoveredPoints, 4);
2971   PetscValidPointer(coveredPoints, 5);
2972   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2973   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2974   /* Copy in support of first point */
2975   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2976   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2977   for (joinSize = 0; joinSize < dof; ++joinSize) {
2978     join[i][joinSize] = mesh->supports[off+joinSize];
2979   }
2980   /* Check each successive support */
2981   for (p = 1; p < numPoints; ++p) {
2982     PetscInt newJoinSize = 0;
2983 
2984     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2985     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2986     for (c = 0; c < dof; ++c) {
2987       const PetscInt point = mesh->supports[off+c];
2988 
2989       for (m = 0; m < joinSize; ++m) {
2990         if (point == join[i][m]) {
2991           join[1-i][newJoinSize++] = point;
2992           break;
2993         }
2994       }
2995     }
2996     joinSize = newJoinSize;
2997     i        = 1-i;
2998   }
2999   *numCoveredPoints = joinSize;
3000   *coveredPoints    = join[i];
3001   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3002   PetscFunctionReturn(0);
3003 }
3004 
3005 /*@C
3006   DMPlexRestoreJoin - Restore an array for the join of the set of points
3007 
3008   Not Collective
3009 
3010   Input Parameters:
3011 + dm - The DMPlex object
3012 . numPoints - The number of input points for the join
3013 - points - The input points
3014 
3015   Output Parameters:
3016 + numCoveredPoints - The number of points in the join
3017 - coveredPoints - The points in the join
3018 
3019   Fortran Notes:
3020   Since it returns an array, this routine is only available in Fortran 90, and you must
3021   include petsc.h90 in your code.
3022 
3023   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3024 
3025   Level: intermediate
3026 
3027 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3028 @*/
3029 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3030 {
3031   PetscErrorCode ierr;
3032 
3033   PetscFunctionBegin;
3034   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3035   if (points) PetscValidIntPointer(points,3);
3036   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3037   PetscValidPointer(coveredPoints, 5);
3038   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3039   if (numCoveredPoints) *numCoveredPoints = 0;
3040   PetscFunctionReturn(0);
3041 }
3042 
3043 /*@C
3044   DMPlexGetFullJoin - Get an array for the join of the set of points
3045 
3046   Not Collective
3047 
3048   Input Parameters:
3049 + dm - The DMPlex object
3050 . numPoints - The number of input points for the join
3051 - points - The input points
3052 
3053   Output Parameters:
3054 + numCoveredPoints - The number of points in the join
3055 - coveredPoints - The points in the join
3056 
3057   Fortran Notes:
3058   Since it returns an array, this routine is only available in Fortran 90, and you must
3059   include petsc.h90 in your code.
3060 
3061   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3062 
3063   Level: intermediate
3064 
3065 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3066 @*/
3067 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3068 {
3069   DM_Plex       *mesh = (DM_Plex*) dm->data;
3070   PetscInt      *offsets, **closures;
3071   PetscInt      *join[2];
3072   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3073   PetscInt       p, d, c, m, ms;
3074   PetscErrorCode ierr;
3075 
3076   PetscFunctionBegin;
3077   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3078   PetscValidIntPointer(points, 3);
3079   PetscValidIntPointer(numCoveredPoints, 4);
3080   PetscValidPointer(coveredPoints, 5);
3081 
3082   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3083   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3084   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3085   ms      = mesh->maxSupportSize;
3086   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3087   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3088   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3089 
3090   for (p = 0; p < numPoints; ++p) {
3091     PetscInt closureSize;
3092 
3093     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3094 
3095     offsets[p*(depth+2)+0] = 0;
3096     for (d = 0; d < depth+1; ++d) {
3097       PetscInt pStart, pEnd, i;
3098 
3099       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3100       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3101         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3102           offsets[p*(depth+2)+d+1] = i;
3103           break;
3104         }
3105       }
3106       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3107     }
3108     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3109   }
3110   for (d = 0; d < depth+1; ++d) {
3111     PetscInt dof;
3112 
3113     /* Copy in support of first point */
3114     dof = offsets[d+1] - offsets[d];
3115     for (joinSize = 0; joinSize < dof; ++joinSize) {
3116       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3117     }
3118     /* Check each successive cone */
3119     for (p = 1; p < numPoints && joinSize; ++p) {
3120       PetscInt newJoinSize = 0;
3121 
3122       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3123       for (c = 0; c < dof; ++c) {
3124         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3125 
3126         for (m = 0; m < joinSize; ++m) {
3127           if (point == join[i][m]) {
3128             join[1-i][newJoinSize++] = point;
3129             break;
3130           }
3131         }
3132       }
3133       joinSize = newJoinSize;
3134       i        = 1-i;
3135     }
3136     if (joinSize) break;
3137   }
3138   *numCoveredPoints = joinSize;
3139   *coveredPoints    = join[i];
3140   for (p = 0; p < numPoints; ++p) {
3141     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3142   }
3143   ierr = PetscFree(closures);CHKERRQ(ierr);
3144   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3145   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3146   PetscFunctionReturn(0);
3147 }
3148 
3149 /*@C
3150   DMPlexGetMeet - Get an array for the meet of the set of points
3151 
3152   Not Collective
3153 
3154   Input Parameters:
3155 + dm - The DMPlex object
3156 . numPoints - The number of input points for the meet
3157 - points - The input points
3158 
3159   Output Parameters:
3160 + numCoveredPoints - The number of points in the meet
3161 - coveredPoints - The points in the meet
3162 
3163   Level: intermediate
3164 
3165   Note: Currently, this is restricted to a single level meet
3166 
3167   Fortran Notes:
3168   Since it returns an array, this routine is only available in Fortran 90, and you must
3169   include petsc.h90 in your code.
3170 
3171   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3172 
3173 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3174 @*/
3175 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3176 {
3177   DM_Plex       *mesh = (DM_Plex*) dm->data;
3178   PetscInt      *meet[2];
3179   PetscInt       meetSize, i = 0;
3180   PetscInt       dof, off, p, c, m;
3181   PetscErrorCode ierr;
3182 
3183   PetscFunctionBegin;
3184   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3185   PetscValidPointer(points, 2);
3186   PetscValidPointer(numCoveringPoints, 3);
3187   PetscValidPointer(coveringPoints, 4);
3188   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3189   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3190   /* Copy in cone of first point */
3191   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3192   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3193   for (meetSize = 0; meetSize < dof; ++meetSize) {
3194     meet[i][meetSize] = mesh->cones[off+meetSize];
3195   }
3196   /* Check each successive cone */
3197   for (p = 1; p < numPoints; ++p) {
3198     PetscInt newMeetSize = 0;
3199 
3200     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3201     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3202     for (c = 0; c < dof; ++c) {
3203       const PetscInt point = mesh->cones[off+c];
3204 
3205       for (m = 0; m < meetSize; ++m) {
3206         if (point == meet[i][m]) {
3207           meet[1-i][newMeetSize++] = point;
3208           break;
3209         }
3210       }
3211     }
3212     meetSize = newMeetSize;
3213     i        = 1-i;
3214   }
3215   *numCoveringPoints = meetSize;
3216   *coveringPoints    = meet[i];
3217   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3218   PetscFunctionReturn(0);
3219 }
3220 
3221 /*@C
3222   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3223 
3224   Not Collective
3225 
3226   Input Parameters:
3227 + dm - The DMPlex object
3228 . numPoints - The number of input points for the meet
3229 - points - The input points
3230 
3231   Output Parameters:
3232 + numCoveredPoints - The number of points in the meet
3233 - coveredPoints - The points in the meet
3234 
3235   Level: intermediate
3236 
3237   Fortran Notes:
3238   Since it returns an array, this routine is only available in Fortran 90, and you must
3239   include petsc.h90 in your code.
3240 
3241   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3242 
3243 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3244 @*/
3245 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3246 {
3247   PetscErrorCode ierr;
3248 
3249   PetscFunctionBegin;
3250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3251   if (points) PetscValidIntPointer(points,3);
3252   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3253   PetscValidPointer(coveredPoints,5);
3254   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3255   if (numCoveredPoints) *numCoveredPoints = 0;
3256   PetscFunctionReturn(0);
3257 }
3258 
3259 /*@C
3260   DMPlexGetFullMeet - Get an array for the meet of the set of points
3261 
3262   Not Collective
3263 
3264   Input Parameters:
3265 + dm - The DMPlex object
3266 . numPoints - The number of input points for the meet
3267 - points - The input points
3268 
3269   Output Parameters:
3270 + numCoveredPoints - The number of points in the meet
3271 - coveredPoints - The points in the meet
3272 
3273   Level: intermediate
3274 
3275   Fortran Notes:
3276   Since it returns an array, this routine is only available in Fortran 90, and you must
3277   include petsc.h90 in your code.
3278 
3279   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3280 
3281 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3282 @*/
3283 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3284 {
3285   DM_Plex       *mesh = (DM_Plex*) dm->data;
3286   PetscInt      *offsets, **closures;
3287   PetscInt      *meet[2];
3288   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3289   PetscInt       p, h, c, m, mc;
3290   PetscErrorCode ierr;
3291 
3292   PetscFunctionBegin;
3293   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3294   PetscValidPointer(points, 2);
3295   PetscValidPointer(numCoveredPoints, 3);
3296   PetscValidPointer(coveredPoints, 4);
3297 
3298   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3299   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3300   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3301   mc      = mesh->maxConeSize;
3302   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3303   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3304   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3305 
3306   for (p = 0; p < numPoints; ++p) {
3307     PetscInt closureSize;
3308 
3309     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3310 
3311     offsets[p*(height+2)+0] = 0;
3312     for (h = 0; h < height+1; ++h) {
3313       PetscInt pStart, pEnd, i;
3314 
3315       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3316       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3317         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3318           offsets[p*(height+2)+h+1] = i;
3319           break;
3320         }
3321       }
3322       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3323     }
3324     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3325   }
3326   for (h = 0; h < height+1; ++h) {
3327     PetscInt dof;
3328 
3329     /* Copy in cone of first point */
3330     dof = offsets[h+1] - offsets[h];
3331     for (meetSize = 0; meetSize < dof; ++meetSize) {
3332       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3333     }
3334     /* Check each successive cone */
3335     for (p = 1; p < numPoints && meetSize; ++p) {
3336       PetscInt newMeetSize = 0;
3337 
3338       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3339       for (c = 0; c < dof; ++c) {
3340         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3341 
3342         for (m = 0; m < meetSize; ++m) {
3343           if (point == meet[i][m]) {
3344             meet[1-i][newMeetSize++] = point;
3345             break;
3346           }
3347         }
3348       }
3349       meetSize = newMeetSize;
3350       i        = 1-i;
3351     }
3352     if (meetSize) break;
3353   }
3354   *numCoveredPoints = meetSize;
3355   *coveredPoints    = meet[i];
3356   for (p = 0; p < numPoints; ++p) {
3357     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3358   }
3359   ierr = PetscFree(closures);CHKERRQ(ierr);
3360   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3361   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3362   PetscFunctionReturn(0);
3363 }
3364 
3365 /*@C
3366   DMPlexEqual - Determine if two DMs have the same topology
3367 
3368   Not Collective
3369 
3370   Input Parameters:
3371 + dmA - A DMPlex object
3372 - dmB - A DMPlex object
3373 
3374   Output Parameters:
3375 . equal - PETSC_TRUE if the topologies are identical
3376 
3377   Level: intermediate
3378 
3379   Notes:
3380   We are not solving graph isomorphism, so we do not permutation.
3381 
3382 .seealso: DMPlexGetCone()
3383 @*/
3384 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3385 {
3386   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3387   PetscErrorCode ierr;
3388 
3389   PetscFunctionBegin;
3390   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3391   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3392   PetscValidPointer(equal, 3);
3393 
3394   *equal = PETSC_FALSE;
3395   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3396   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3397   if (depth != depthB) PetscFunctionReturn(0);
3398   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3399   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3400   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3401   for (p = pStart; p < pEnd; ++p) {
3402     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3403     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3404 
3405     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3406     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3407     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3408     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3409     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3410     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3411     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3412     for (c = 0; c < coneSize; ++c) {
3413       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3414       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3415     }
3416     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3417     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3418     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3419     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3420     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3421     for (s = 0; s < supportSize; ++s) {
3422       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3423     }
3424   }
3425   *equal = PETSC_TRUE;
3426   PetscFunctionReturn(0);
3427 }
3428 
3429 /*@C
3430   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3431 
3432   Not Collective
3433 
3434   Input Parameters:
3435 + dm         - The DMPlex
3436 . cellDim    - The cell dimension
3437 - numCorners - The number of vertices on a cell
3438 
3439   Output Parameters:
3440 . numFaceVertices - The number of vertices on a face
3441 
3442   Level: developer
3443 
3444   Notes:
3445   Of course this can only work for a restricted set of symmetric shapes
3446 
3447 .seealso: DMPlexGetCone()
3448 @*/
3449 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3450 {
3451   MPI_Comm       comm;
3452   PetscErrorCode ierr;
3453 
3454   PetscFunctionBegin;
3455   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3456   PetscValidPointer(numFaceVertices,3);
3457   switch (cellDim) {
3458   case 0:
3459     *numFaceVertices = 0;
3460     break;
3461   case 1:
3462     *numFaceVertices = 1;
3463     break;
3464   case 2:
3465     switch (numCorners) {
3466     case 3: /* triangle */
3467       *numFaceVertices = 2; /* Edge has 2 vertices */
3468       break;
3469     case 4: /* quadrilateral */
3470       *numFaceVertices = 2; /* Edge has 2 vertices */
3471       break;
3472     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3473       *numFaceVertices = 3; /* Edge has 3 vertices */
3474       break;
3475     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3476       *numFaceVertices = 3; /* Edge has 3 vertices */
3477       break;
3478     default:
3479       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3480     }
3481     break;
3482   case 3:
3483     switch (numCorners) {
3484     case 4: /* tetradehdron */
3485       *numFaceVertices = 3; /* Face has 3 vertices */
3486       break;
3487     case 6: /* tet cohesive cells */
3488       *numFaceVertices = 4; /* Face has 4 vertices */
3489       break;
3490     case 8: /* hexahedron */
3491       *numFaceVertices = 4; /* Face has 4 vertices */
3492       break;
3493     case 9: /* tet cohesive Lagrange cells */
3494       *numFaceVertices = 6; /* Face has 6 vertices */
3495       break;
3496     case 10: /* quadratic tetrahedron */
3497       *numFaceVertices = 6; /* Face has 6 vertices */
3498       break;
3499     case 12: /* hex cohesive Lagrange cells */
3500       *numFaceVertices = 6; /* Face has 6 vertices */
3501       break;
3502     case 18: /* quadratic tet cohesive Lagrange cells */
3503       *numFaceVertices = 6; /* Face has 6 vertices */
3504       break;
3505     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3506       *numFaceVertices = 9; /* Face has 9 vertices */
3507       break;
3508     default:
3509       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3510     }
3511     break;
3512   default:
3513     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3514   }
3515   PetscFunctionReturn(0);
3516 }
3517 
3518 /*@
3519   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3520 
3521   Not Collective
3522 
3523   Input Parameter:
3524 . dm    - The DMPlex object
3525 
3526   Output Parameter:
3527 . depthLabel - The DMLabel recording point depth
3528 
3529   Level: developer
3530 
3531 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3532 @*/
3533 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3534 {
3535   PetscErrorCode ierr;
3536 
3537   PetscFunctionBegin;
3538   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3539   PetscValidPointer(depthLabel, 2);
3540   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3541   *depthLabel = dm->depthLabel;
3542   PetscFunctionReturn(0);
3543 }
3544 
3545 /*@
3546   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3547 
3548   Not Collective
3549 
3550   Input Parameter:
3551 . dm    - The DMPlex object
3552 
3553   Output Parameter:
3554 . depth - The number of strata (breadth first levels) in the DAG
3555 
3556   Level: developer
3557 
3558 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3559 @*/
3560 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3561 {
3562   DMLabel        label;
3563   PetscInt       d = 0;
3564   PetscErrorCode ierr;
3565 
3566   PetscFunctionBegin;
3567   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3568   PetscValidPointer(depth, 2);
3569   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3570   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3571   *depth = d-1;
3572   PetscFunctionReturn(0);
3573 }
3574 
3575 /*@
3576   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3577 
3578   Not Collective
3579 
3580   Input Parameters:
3581 + dm           - The DMPlex object
3582 - stratumValue - The requested depth
3583 
3584   Output Parameters:
3585 + start - The first point at this depth
3586 - end   - One beyond the last point at this depth
3587 
3588   Notes:
3589   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3590   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3591   higher dimension, e.g., "edges".
3592 
3593   Level: developer
3594 
3595 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3596 @*/
3597 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3598 {
3599   DMLabel        label;
3600   PetscInt       pStart, pEnd;
3601   PetscErrorCode ierr;
3602 
3603   PetscFunctionBegin;
3604   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3605   if (start) {PetscValidPointer(start, 3); *start = 0;}
3606   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3607   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3608   if (pStart == pEnd) PetscFunctionReturn(0);
3609   if (stratumValue < 0) {
3610     if (start) *start = pStart;
3611     if (end)   *end   = pEnd;
3612     PetscFunctionReturn(0);
3613   }
3614   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3615   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3616   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3617   PetscFunctionReturn(0);
3618 }
3619 
3620 /*@
3621   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3622 
3623   Not Collective
3624 
3625   Input Parameters:
3626 + dm           - The DMPlex object
3627 - stratumValue - The requested height
3628 
3629   Output Parameters:
3630 + start - The first point at this height
3631 - end   - One beyond the last point at this height
3632 
3633   Notes:
3634   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3635   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3636   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3637 
3638   Level: developer
3639 
3640 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3641 @*/
3642 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3643 {
3644   DMLabel        label;
3645   PetscInt       depth, pStart, pEnd;
3646   PetscErrorCode ierr;
3647 
3648   PetscFunctionBegin;
3649   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3650   if (start) {PetscValidPointer(start, 3); *start = 0;}
3651   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3652   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3653   if (pStart == pEnd) PetscFunctionReturn(0);
3654   if (stratumValue < 0) {
3655     if (start) *start = pStart;
3656     if (end)   *end   = pEnd;
3657     PetscFunctionReturn(0);
3658   }
3659   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3660   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3661   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3662   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3663   PetscFunctionReturn(0);
3664 }
3665 
3666 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3667 {
3668   PetscSection   section, s;
3669   Mat            m;
3670   PetscInt       maxHeight;
3671   PetscErrorCode ierr;
3672 
3673   PetscFunctionBegin;
3674   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3675   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3676   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3677   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3678   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3679   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3680   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3681   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3682   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3683   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3684   ierr = MatDestroy(&m);CHKERRQ(ierr);
3685 
3686   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3687   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3688   PetscFunctionReturn(0);
3689 }
3690 
3691 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3692 {
3693   Vec            coordsLocal;
3694   DM             coordsDM;
3695   PetscErrorCode ierr;
3696 
3697   PetscFunctionBegin;
3698   *field = NULL;
3699   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3700   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3701   if (coordsLocal && coordsDM) {
3702     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3703   }
3704   PetscFunctionReturn(0);
3705 }
3706 
3707 /*@C
3708   DMPlexGetConeSection - Return a section which describes the layout of cone data
3709 
3710   Not Collective
3711 
3712   Input Parameters:
3713 . dm        - The DMPlex object
3714 
3715   Output Parameter:
3716 . section - The PetscSection object
3717 
3718   Level: developer
3719 
3720 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3721 @*/
3722 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3723 {
3724   DM_Plex *mesh = (DM_Plex*) dm->data;
3725 
3726   PetscFunctionBegin;
3727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3728   if (section) *section = mesh->coneSection;
3729   PetscFunctionReturn(0);
3730 }
3731 
3732 /*@C
3733   DMPlexGetSupportSection - Return a section which describes the layout of support data
3734 
3735   Not Collective
3736 
3737   Input Parameters:
3738 . dm        - The DMPlex object
3739 
3740   Output Parameter:
3741 . section - The PetscSection object
3742 
3743   Level: developer
3744 
3745 .seealso: DMPlexGetConeSection()
3746 @*/
3747 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3748 {
3749   DM_Plex *mesh = (DM_Plex*) dm->data;
3750 
3751   PetscFunctionBegin;
3752   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3753   if (section) *section = mesh->supportSection;
3754   PetscFunctionReturn(0);
3755 }
3756 
3757 /*@C
3758   DMPlexGetCones - Return cone data
3759 
3760   Not Collective
3761 
3762   Input Parameters:
3763 . dm        - The DMPlex object
3764 
3765   Output Parameter:
3766 . cones - The cone for each point
3767 
3768   Level: developer
3769 
3770 .seealso: DMPlexGetConeSection()
3771 @*/
3772 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3773 {
3774   DM_Plex *mesh = (DM_Plex*) dm->data;
3775 
3776   PetscFunctionBegin;
3777   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3778   if (cones) *cones = mesh->cones;
3779   PetscFunctionReturn(0);
3780 }
3781 
3782 /*@C
3783   DMPlexGetConeOrientations - Return cone orientation data
3784 
3785   Not Collective
3786 
3787   Input Parameters:
3788 . dm        - The DMPlex object
3789 
3790   Output Parameter:
3791 . coneOrientations - The cone orientation for each point
3792 
3793   Level: developer
3794 
3795 .seealso: DMPlexGetConeSection()
3796 @*/
3797 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3798 {
3799   DM_Plex *mesh = (DM_Plex*) dm->data;
3800 
3801   PetscFunctionBegin;
3802   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3803   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3804   PetscFunctionReturn(0);
3805 }
3806 
3807 /******************************** FEM Support **********************************/
3808 
3809 /*
3810  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3811  representing a line in the section.
3812 */
3813 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3814 {
3815   PetscErrorCode ierr;
3816 
3817   PetscFunctionBeginHot;
3818   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3819   if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3820     *k = 1;
3821   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3822     /* An order k SEM disc has k-1 dofs on an edge */
3823     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3824     *k = *k / *Nc + 1;
3825   }
3826   PetscFunctionReturn(0);
3827 }
3828 
3829 /*@
3830 
3831   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3832   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3833   section provided (or the section of the DM).
3834 
3835   Input Parameters:
3836 + dm      - The DM
3837 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3838 - section - The PetscSection to reorder, or NULL for the default section
3839 
3840   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3841   degree of the basis.
3842 
3843   Example:
3844   A typical interpolated single-quad mesh might order points as
3845 .vb
3846   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3847 
3848   v4 -- e6 -- v3
3849   |           |
3850   e7    c0    e8
3851   |           |
3852   v1 -- e5 -- v2
3853 .ve
3854 
3855   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3856   dofs in the order of points, e.g.,
3857 .vb
3858     c0 -> [0,1,2,3]
3859     v1 -> [4]
3860     ...
3861     e5 -> [8, 9]
3862 .ve
3863 
3864   which corresponds to the dofs
3865 .vb
3866     6   10  11  7
3867     13  2   3   15
3868     12  0   1   14
3869     4   8   9   5
3870 .ve
3871 
3872   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3873 .vb
3874   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3875 .ve
3876 
3877   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3878 .vb
3879    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3880 .ve
3881 
3882   Level: developer
3883 
3884 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlocalSection()
3885 @*/
3886 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3887 {
3888   DMLabel        label;
3889   PetscInt      *perm;
3890   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3891   PetscBool      vertexchart;
3892   PetscErrorCode ierr;
3893 
3894   PetscFunctionBegin;
3895   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3896   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3897   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3898   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3899   if (depth == 1) {eStart = point;}
3900   else if  (depth == dim) {
3901     const PetscInt *cone;
3902 
3903     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3904     if (dim == 2) eStart = cone[0];
3905     else if (dim == 3) {
3906       const PetscInt *cone2;
3907       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3908       eStart = cone2[0];
3909     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3910   } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3911   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
3912   {                             /* Determine whether the chart covers all points or just vertices. */
3913     PetscInt pStart,pEnd,cStart,cEnd;
3914     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3915     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3916     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3917     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3918   }
3919   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3920   if (dim < 1) PetscFunctionReturn(0);
3921   for (f = 0; f < Nf; ++f) {
3922     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3923     size += PetscPowInt(k+1, dim)*Nc;
3924   }
3925   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3926   for (f = 0; f < Nf; ++f) {
3927     switch (dim) {
3928     case 1:
3929       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3930       /*
3931         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3932         We want              [ vtx0; edge of length k-1; vtx1 ]
3933       */
3934       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3935       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3936       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3937       foffset = offset;
3938       break;
3939     case 2:
3940       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3941       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3942       /* The SEM order is
3943 
3944          v_lb, {e_b}, v_rb,
3945          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3946          v_lt, reverse {e_t}, v_rt
3947       */
3948       {
3949         const PetscInt of   = 0;
3950         const PetscInt oeb  = of   + PetscSqr(k-1);
3951         const PetscInt oer  = oeb  + (k-1);
3952         const PetscInt oet  = oer  + (k-1);
3953         const PetscInt oel  = oet  + (k-1);
3954         const PetscInt ovlb = oel  + (k-1);
3955         const PetscInt ovrb = ovlb + 1;
3956         const PetscInt ovrt = ovrb + 1;
3957         const PetscInt ovlt = ovrt + 1;
3958         PetscInt       o;
3959 
3960         /* bottom */
3961         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3962         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3963         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3964         /* middle */
3965         for (i = 0; i < k-1; ++i) {
3966           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3967           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3968           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3969         }
3970         /* top */
3971         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3972         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3973         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3974         foffset = offset;
3975       }
3976       break;
3977     case 3:
3978       /* The original hex closure is
3979 
3980          {c,
3981           f_b, f_t, f_f, f_b, f_r, f_l,
3982           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3983           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3984       */
3985       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3986       /* The SEM order is
3987          Bottom Slice
3988          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3989          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3990          v_blb, {e_bb}, v_brb,
3991 
3992          Middle Slice (j)
3993          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3994          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3995          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3996 
3997          Top Slice
3998          v_tlf, {e_tf}, v_trf,
3999          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
4000          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
4001       */
4002       {
4003         const PetscInt oc    = 0;
4004         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4005         const PetscInt oft   = ofb   + PetscSqr(k-1);
4006         const PetscInt off   = oft   + PetscSqr(k-1);
4007         const PetscInt ofk   = off   + PetscSqr(k-1);
4008         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4009         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4010         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4011         const PetscInt oebb  = oebl  + (k-1);
4012         const PetscInt oebr  = oebb  + (k-1);
4013         const PetscInt oebf  = oebr  + (k-1);
4014         const PetscInt oetf  = oebf  + (k-1);
4015         const PetscInt oetr  = oetf  + (k-1);
4016         const PetscInt oetb  = oetr  + (k-1);
4017         const PetscInt oetl  = oetb  + (k-1);
4018         const PetscInt oerf  = oetl  + (k-1);
4019         const PetscInt oelf  = oerf  + (k-1);
4020         const PetscInt oelb  = oelf  + (k-1);
4021         const PetscInt oerb  = oelb  + (k-1);
4022         const PetscInt ovblf = oerb  + (k-1);
4023         const PetscInt ovblb = ovblf + 1;
4024         const PetscInt ovbrb = ovblb + 1;
4025         const PetscInt ovbrf = ovbrb + 1;
4026         const PetscInt ovtlf = ovbrf + 1;
4027         const PetscInt ovtrf = ovtlf + 1;
4028         const PetscInt ovtrb = ovtrf + 1;
4029         const PetscInt ovtlb = ovtrb + 1;
4030         PetscInt       o, n;
4031 
4032         /* Bottom Slice */
4033         /*   bottom */
4034         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4035         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4036         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4037         /*   middle */
4038         for (i = 0; i < k-1; ++i) {
4039           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4040           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4041           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4042         }
4043         /*   top */
4044         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4045         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4046         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4047 
4048         /* Middle Slice */
4049         for (j = 0; j < k-1; ++j) {
4050           /*   bottom */
4051           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4052           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4053           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4054           /*   middle */
4055           for (i = 0; i < k-1; ++i) {
4056             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4057             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4058             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4059           }
4060           /*   top */
4061           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4062           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4063           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4064         }
4065 
4066         /* Top Slice */
4067         /*   bottom */
4068         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4069         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4070         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4071         /*   middle */
4072         for (i = 0; i < k-1; ++i) {
4073           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4074           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4075           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4076         }
4077         /*   top */
4078         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4079         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4080         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4081 
4082         foffset = offset;
4083       }
4084       break;
4085     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4086     }
4087   }
4088   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4089   /* Check permutation */
4090   {
4091     PetscInt *check;
4092 
4093     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4094     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4095     for (i = 0; i < size; ++i) check[perm[i]] = i;
4096     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4097     ierr = PetscFree(check);CHKERRQ(ierr);
4098   }
4099   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4100   PetscFunctionReturn(0);
4101 }
4102 
4103 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4104 {
4105   PetscDS        prob;
4106   PetscInt       depth, Nf, h;
4107   DMLabel        label;
4108   PetscErrorCode ierr;
4109 
4110   PetscFunctionBeginHot;
4111   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4112   Nf      = prob->Nf;
4113   label   = dm->depthLabel;
4114   *dspace = NULL;
4115   if (field < Nf) {
4116     PetscObject disc = prob->disc[field];
4117 
4118     if (disc->classid == PETSCFE_CLASSID) {
4119       PetscDualSpace dsp;
4120 
4121       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4122       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4123       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4124       h    = depth - 1 - h;
4125       if (h) {
4126         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4127       } else {
4128         *dspace = dsp;
4129       }
4130     }
4131   }
4132   PetscFunctionReturn(0);
4133 }
4134 
4135 
4136 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4137 {
4138   PetscScalar    *array, *vArray;
4139   const PetscInt *cone, *coneO;
4140   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4141   PetscErrorCode  ierr;
4142 
4143   PetscFunctionBeginHot;
4144   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4145   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4146   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4147   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4148   if (!values || !*values) {
4149     if ((point >= pStart) && (point < pEnd)) {
4150       PetscInt dof;
4151 
4152       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4153       size += dof;
4154     }
4155     for (p = 0; p < numPoints; ++p) {
4156       const PetscInt cp = cone[p];
4157       PetscInt       dof;
4158 
4159       if ((cp < pStart) || (cp >= pEnd)) continue;
4160       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4161       size += dof;
4162     }
4163     if (!values) {
4164       if (csize) *csize = size;
4165       PetscFunctionReturn(0);
4166     }
4167     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4168   } else {
4169     array = *values;
4170   }
4171   size = 0;
4172   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4173   if ((point >= pStart) && (point < pEnd)) {
4174     PetscInt     dof, off, d;
4175     PetscScalar *varr;
4176 
4177     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4178     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4179     varr = &vArray[off];
4180     for (d = 0; d < dof; ++d, ++offset) {
4181       array[offset] = varr[d];
4182     }
4183     size += dof;
4184   }
4185   for (p = 0; p < numPoints; ++p) {
4186     const PetscInt cp = cone[p];
4187     PetscInt       o  = coneO[p];
4188     PetscInt       dof, off, d;
4189     PetscScalar   *varr;
4190 
4191     if ((cp < pStart) || (cp >= pEnd)) continue;
4192     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4193     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4194     varr = &vArray[off];
4195     if (o >= 0) {
4196       for (d = 0; d < dof; ++d, ++offset) {
4197         array[offset] = varr[d];
4198       }
4199     } else {
4200       for (d = dof-1; d >= 0; --d, ++offset) {
4201         array[offset] = varr[d];
4202       }
4203     }
4204     size += dof;
4205   }
4206   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4207   if (!*values) {
4208     if (csize) *csize = size;
4209     *values = array;
4210   } else {
4211     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4212     *csize = size;
4213   }
4214   PetscFunctionReturn(0);
4215 }
4216 
4217 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4218 {
4219   const PetscInt *cla;
4220   PetscInt       np, *pts = NULL;
4221   PetscErrorCode ierr;
4222 
4223   PetscFunctionBeginHot;
4224   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4225   if (!*clPoints) {
4226     PetscInt pStart, pEnd, p, q;
4227 
4228     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4229     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4230     /* Compress out points not in the section */
4231     for (p = 0, q = 0; p < np; p++) {
4232       PetscInt r = pts[2*p];
4233       if ((r >= pStart) && (r < pEnd)) {
4234         pts[q*2]   = r;
4235         pts[q*2+1] = pts[2*p+1];
4236         ++q;
4237       }
4238     }
4239     np = q;
4240     cla = NULL;
4241   } else {
4242     PetscInt dof, off;
4243 
4244     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4245     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4246     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4247     np   = dof/2;
4248     pts  = (PetscInt *) &cla[off];
4249   }
4250   *numPoints = np;
4251   *points    = pts;
4252   *clp       = cla;
4253 
4254   PetscFunctionReturn(0);
4255 }
4256 
4257 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4258 {
4259   PetscErrorCode ierr;
4260 
4261   PetscFunctionBeginHot;
4262   if (!*clPoints) {
4263     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4264   } else {
4265     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4266   }
4267   *numPoints = 0;
4268   *points    = NULL;
4269   *clSec     = NULL;
4270   *clPoints  = NULL;
4271   *clp       = NULL;
4272   PetscFunctionReturn(0);
4273 }
4274 
4275 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4276 {
4277   PetscInt          offset = 0, p;
4278   const PetscInt    **perms = NULL;
4279   const PetscScalar **flips = NULL;
4280   PetscErrorCode    ierr;
4281 
4282   PetscFunctionBeginHot;
4283   *size = 0;
4284   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4285   for (p = 0; p < numPoints; p++) {
4286     const PetscInt    point = points[2*p];
4287     const PetscInt    *perm = perms ? perms[p] : NULL;
4288     const PetscScalar *flip = flips ? flips[p] : NULL;
4289     PetscInt          dof, off, d;
4290     const PetscScalar *varr;
4291 
4292     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4293     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4294     varr = &vArray[off];
4295     if (clperm) {
4296       if (perm) {
4297         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4298       } else {
4299         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4300       }
4301       if (flip) {
4302         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4303       }
4304     } else {
4305       if (perm) {
4306         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4307       } else {
4308         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4309       }
4310       if (flip) {
4311         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4312       }
4313     }
4314     offset += dof;
4315   }
4316   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4317   *size = offset;
4318   PetscFunctionReturn(0);
4319 }
4320 
4321 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4322 {
4323   PetscInt          offset = 0, f;
4324   PetscErrorCode    ierr;
4325 
4326   PetscFunctionBeginHot;
4327   *size = 0;
4328   for (f = 0; f < numFields; ++f) {
4329     PetscInt          p;
4330     const PetscInt    **perms = NULL;
4331     const PetscScalar **flips = NULL;
4332 
4333     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4334     for (p = 0; p < numPoints; p++) {
4335       const PetscInt    point = points[2*p];
4336       PetscInt          fdof, foff, b;
4337       const PetscScalar *varr;
4338       const PetscInt    *perm = perms ? perms[p] : NULL;
4339       const PetscScalar *flip = flips ? flips[p] : NULL;
4340 
4341       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4342       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4343       varr = &vArray[foff];
4344       if (clperm) {
4345         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4346         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4347         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4348       } else {
4349         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4350         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4351         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4352       }
4353       offset += fdof;
4354     }
4355     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4356   }
4357   *size = offset;
4358   PetscFunctionReturn(0);
4359 }
4360 
4361 /*@C
4362   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4363 
4364   Not collective
4365 
4366   Input Parameters:
4367 + dm - The DM
4368 . section - The section describing the layout in v, or NULL to use the default section
4369 . v - The local vector
4370 . point - The point in the DM
4371 . csize - The size of the input values array, or NULL
4372 - values - An array to use for the values, or NULL to have it allocated automatically
4373 
4374   Output Parameters:
4375 + csize - The number of values in the closure
4376 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4377 
4378 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4379 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4380 $ assembly function, and a user may already have allocated storage for this operation.
4381 $
4382 $ A typical use could be
4383 $
4384 $  values = NULL;
4385 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4386 $  for (cl = 0; cl < clSize; ++cl) {
4387 $    <Compute on closure>
4388 $  }
4389 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4390 $
4391 $ or
4392 $
4393 $  PetscMalloc1(clMaxSize, &values);
4394 $  for (p = pStart; p < pEnd; ++p) {
4395 $    clSize = clMaxSize;
4396 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4397 $    for (cl = 0; cl < clSize; ++cl) {
4398 $      <Compute on closure>
4399 $    }
4400 $  }
4401 $  PetscFree(values);
4402 
4403   Fortran Notes:
4404   Since it returns an array, this routine is only available in Fortran 90, and you must
4405   include petsc.h90 in your code.
4406 
4407   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4408 
4409   Level: intermediate
4410 
4411 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4412 @*/
4413 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4414 {
4415   PetscSection       clSection;
4416   IS                 clPoints;
4417   PetscScalar       *array;
4418   const PetscScalar *vArray;
4419   PetscInt          *points = NULL;
4420   const PetscInt    *clp, *perm;
4421   PetscInt           depth, numFields, numPoints, size;
4422   PetscErrorCode     ierr;
4423 
4424   PetscFunctionBeginHot;
4425   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4426   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4427   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4428   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4429   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4430   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4431   if (depth == 1 && numFields < 2) {
4432     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4433     PetscFunctionReturn(0);
4434   }
4435   /* Get points */
4436   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4437   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4438   /* Get array */
4439   if (!values || !*values) {
4440     PetscInt asize = 0, dof, p;
4441 
4442     for (p = 0; p < numPoints*2; p += 2) {
4443       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4444       asize += dof;
4445     }
4446     if (!values) {
4447       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4448       if (csize) *csize = asize;
4449       PetscFunctionReturn(0);
4450     }
4451     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4452   } else {
4453     array = *values;
4454   }
4455   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4456   /* Get values */
4457   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4458   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4459   /* Cleanup points */
4460   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4461   /* Cleanup array */
4462   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4463   if (!*values) {
4464     if (csize) *csize = size;
4465     *values = array;
4466   } else {
4467     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4468     *csize = size;
4469   }
4470   PetscFunctionReturn(0);
4471 }
4472 
4473 /*@C
4474   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4475 
4476   Not collective
4477 
4478   Input Parameters:
4479 + dm - The DM
4480 . section - The section describing the layout in v, or NULL to use the default section
4481 . v - The local vector
4482 . point - The point in the DM
4483 . csize - The number of values in the closure, or NULL
4484 - values - The array of values, which is a borrowed array and should not be freed
4485 
4486   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4487 
4488   Fortran Notes:
4489   Since it returns an array, this routine is only available in Fortran 90, and you must
4490   include petsc.h90 in your code.
4491 
4492   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4493 
4494   Level: intermediate
4495 
4496 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4497 @*/
4498 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4499 {
4500   PetscInt       size = 0;
4501   PetscErrorCode ierr;
4502 
4503   PetscFunctionBegin;
4504   /* Should work without recalculating size */
4505   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4506   *values = NULL;
4507   PetscFunctionReturn(0);
4508 }
4509 
4510 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4511 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4512 
4513 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4514 {
4515   PetscInt        cdof;   /* The number of constraints on this point */
4516   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4517   PetscScalar    *a;
4518   PetscInt        off, cind = 0, k;
4519   PetscErrorCode  ierr;
4520 
4521   PetscFunctionBegin;
4522   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4523   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4524   a    = &array[off];
4525   if (!cdof || setBC) {
4526     if (clperm) {
4527       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4528       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4529     } else {
4530       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4531       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4532     }
4533   } else {
4534     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4535     if (clperm) {
4536       if (perm) {for (k = 0; k < dof; ++k) {
4537           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4538           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4539         }
4540       } else {
4541         for (k = 0; k < dof; ++k) {
4542           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4543           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4544         }
4545       }
4546     } else {
4547       if (perm) {
4548         for (k = 0; k < dof; ++k) {
4549           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4550           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4551         }
4552       } else {
4553         for (k = 0; k < dof; ++k) {
4554           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4555           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4556         }
4557       }
4558     }
4559   }
4560   PetscFunctionReturn(0);
4561 }
4562 
4563 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4564 {
4565   PetscInt        cdof;   /* The number of constraints on this point */
4566   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4567   PetscScalar    *a;
4568   PetscInt        off, cind = 0, k;
4569   PetscErrorCode  ierr;
4570 
4571   PetscFunctionBegin;
4572   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4573   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4574   a    = &array[off];
4575   if (cdof) {
4576     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4577     if (clperm) {
4578       if (perm) {
4579         for (k = 0; k < dof; ++k) {
4580           if ((cind < cdof) && (k == cdofs[cind])) {
4581             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4582             cind++;
4583           }
4584         }
4585       } else {
4586         for (k = 0; k < dof; ++k) {
4587           if ((cind < cdof) && (k == cdofs[cind])) {
4588             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4589             cind++;
4590           }
4591         }
4592       }
4593     } else {
4594       if (perm) {
4595         for (k = 0; k < dof; ++k) {
4596           if ((cind < cdof) && (k == cdofs[cind])) {
4597             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4598             cind++;
4599           }
4600         }
4601       } else {
4602         for (k = 0; k < dof; ++k) {
4603           if ((cind < cdof) && (k == cdofs[cind])) {
4604             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4605             cind++;
4606           }
4607         }
4608       }
4609     }
4610   }
4611   PetscFunctionReturn(0);
4612 }
4613 
4614 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4615 {
4616   PetscScalar    *a;
4617   PetscInt        fdof, foff, fcdof, foffset = *offset;
4618   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4619   PetscInt        cind = 0, b;
4620   PetscErrorCode  ierr;
4621 
4622   PetscFunctionBegin;
4623   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4624   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4625   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4626   a    = &array[foff];
4627   if (!fcdof || setBC) {
4628     if (clperm) {
4629       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4630       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4631     } else {
4632       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4633       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4634     }
4635   } else {
4636     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4637     if (clperm) {
4638       if (perm) {
4639         for (b = 0; b < fdof; b++) {
4640           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4641           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4642         }
4643       } else {
4644         for (b = 0; b < fdof; b++) {
4645           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4646           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4647         }
4648       }
4649     } else {
4650       if (perm) {
4651         for (b = 0; b < fdof; b++) {
4652           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4653           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4654         }
4655       } else {
4656         for (b = 0; b < fdof; b++) {
4657           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4658           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4659         }
4660       }
4661     }
4662   }
4663   *offset += fdof;
4664   PetscFunctionReturn(0);
4665 }
4666 
4667 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4668 {
4669   PetscScalar    *a;
4670   PetscInt        fdof, foff, fcdof, foffset = *offset;
4671   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4672   PetscInt        cind = 0, ncind = 0, b;
4673   PetscBool       ncSet, fcSet;
4674   PetscErrorCode  ierr;
4675 
4676   PetscFunctionBegin;
4677   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4678   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4679   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4680   a    = &array[foff];
4681   if (fcdof) {
4682     /* We just override fcdof and fcdofs with Ncc and comps */
4683     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4684     if (clperm) {
4685       if (perm) {
4686         if (comps) {
4687           for (b = 0; b < fdof; b++) {
4688             ncSet = fcSet = PETSC_FALSE;
4689             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4690             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4691             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4692           }
4693         } else {
4694           for (b = 0; b < fdof; b++) {
4695             if ((cind < fcdof) && (b == fcdofs[cind])) {
4696               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4697               ++cind;
4698             }
4699           }
4700         }
4701       } else {
4702         if (comps) {
4703           for (b = 0; b < fdof; b++) {
4704             ncSet = fcSet = PETSC_FALSE;
4705             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4706             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4707             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4708           }
4709         } else {
4710           for (b = 0; b < fdof; b++) {
4711             if ((cind < fcdof) && (b == fcdofs[cind])) {
4712               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4713               ++cind;
4714             }
4715           }
4716         }
4717       }
4718     } else {
4719       if (perm) {
4720         if (comps) {
4721           for (b = 0; b < fdof; b++) {
4722             ncSet = fcSet = PETSC_FALSE;
4723             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4724             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4725             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4726           }
4727         } else {
4728           for (b = 0; b < fdof; b++) {
4729             if ((cind < fcdof) && (b == fcdofs[cind])) {
4730               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4731               ++cind;
4732             }
4733           }
4734         }
4735       } else {
4736         if (comps) {
4737           for (b = 0; b < fdof; b++) {
4738             ncSet = fcSet = PETSC_FALSE;
4739             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4740             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4741             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4742           }
4743         } else {
4744           for (b = 0; b < fdof; b++) {
4745             if ((cind < fcdof) && (b == fcdofs[cind])) {
4746               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4747               ++cind;
4748             }
4749           }
4750         }
4751       }
4752     }
4753   }
4754   *offset += fdof;
4755   PetscFunctionReturn(0);
4756 }
4757 
4758 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4759 {
4760   PetscScalar    *array;
4761   const PetscInt *cone, *coneO;
4762   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4763   PetscErrorCode  ierr;
4764 
4765   PetscFunctionBeginHot;
4766   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4767   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4768   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4769   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4770   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4771   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4772     const PetscInt cp = !p ? point : cone[p-1];
4773     const PetscInt o  = !p ? 0     : coneO[p-1];
4774 
4775     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4776     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4777     /* ADD_VALUES */
4778     {
4779       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4780       PetscScalar    *a;
4781       PetscInt        cdof, coff, cind = 0, k;
4782 
4783       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4784       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4785       a    = &array[coff];
4786       if (!cdof) {
4787         if (o >= 0) {
4788           for (k = 0; k < dof; ++k) {
4789             a[k] += values[off+k];
4790           }
4791         } else {
4792           for (k = 0; k < dof; ++k) {
4793             a[k] += values[off+dof-k-1];
4794           }
4795         }
4796       } else {
4797         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4798         if (o >= 0) {
4799           for (k = 0; k < dof; ++k) {
4800             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4801             a[k] += values[off+k];
4802           }
4803         } else {
4804           for (k = 0; k < dof; ++k) {
4805             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4806             a[k] += values[off+dof-k-1];
4807           }
4808         }
4809       }
4810     }
4811   }
4812   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4813   PetscFunctionReturn(0);
4814 }
4815 
4816 /*@C
4817   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4818 
4819   Not collective
4820 
4821   Input Parameters:
4822 + dm - The DM
4823 . section - The section describing the layout in v, or NULL to use the default section
4824 . v - The local vector
4825 . point - The point in the DM
4826 . values - The array of values
4827 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4828          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4829 
4830   Fortran Notes:
4831   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4832 
4833   Level: intermediate
4834 
4835 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4836 @*/
4837 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4838 {
4839   PetscSection    clSection;
4840   IS              clPoints;
4841   PetscScalar    *array;
4842   PetscInt       *points = NULL;
4843   const PetscInt *clp, *clperm;
4844   PetscInt        depth, numFields, numPoints, p;
4845   PetscErrorCode  ierr;
4846 
4847   PetscFunctionBeginHot;
4848   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4849   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4850   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4851   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4852   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4853   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4854   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4855     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4856     PetscFunctionReturn(0);
4857   }
4858   /* Get points */
4859   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4860   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4861   /* Get array */
4862   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4863   /* Get values */
4864   if (numFields > 0) {
4865     PetscInt offset = 0, f;
4866     for (f = 0; f < numFields; ++f) {
4867       const PetscInt    **perms = NULL;
4868       const PetscScalar **flips = NULL;
4869 
4870       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4871       switch (mode) {
4872       case INSERT_VALUES:
4873         for (p = 0; p < numPoints; p++) {
4874           const PetscInt    point = points[2*p];
4875           const PetscInt    *perm = perms ? perms[p] : NULL;
4876           const PetscScalar *flip = flips ? flips[p] : NULL;
4877           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4878         } break;
4879       case INSERT_ALL_VALUES:
4880         for (p = 0; p < numPoints; p++) {
4881           const PetscInt    point = points[2*p];
4882           const PetscInt    *perm = perms ? perms[p] : NULL;
4883           const PetscScalar *flip = flips ? flips[p] : NULL;
4884           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4885         } break;
4886       case INSERT_BC_VALUES:
4887         for (p = 0; p < numPoints; p++) {
4888           const PetscInt    point = points[2*p];
4889           const PetscInt    *perm = perms ? perms[p] : NULL;
4890           const PetscScalar *flip = flips ? flips[p] : NULL;
4891           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4892         } break;
4893       case ADD_VALUES:
4894         for (p = 0; p < numPoints; p++) {
4895           const PetscInt    point = points[2*p];
4896           const PetscInt    *perm = perms ? perms[p] : NULL;
4897           const PetscScalar *flip = flips ? flips[p] : NULL;
4898           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4899         } break;
4900       case ADD_ALL_VALUES:
4901         for (p = 0; p < numPoints; p++) {
4902           const PetscInt    point = points[2*p];
4903           const PetscInt    *perm = perms ? perms[p] : NULL;
4904           const PetscScalar *flip = flips ? flips[p] : NULL;
4905           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4906         } break;
4907       case ADD_BC_VALUES:
4908         for (p = 0; p < numPoints; p++) {
4909           const PetscInt    point = points[2*p];
4910           const PetscInt    *perm = perms ? perms[p] : NULL;
4911           const PetscScalar *flip = flips ? flips[p] : NULL;
4912           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4913         } break;
4914       default:
4915         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4916       }
4917       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4918     }
4919   } else {
4920     PetscInt dof, off;
4921     const PetscInt    **perms = NULL;
4922     const PetscScalar **flips = NULL;
4923 
4924     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4925     switch (mode) {
4926     case INSERT_VALUES:
4927       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4928         const PetscInt    point = points[2*p];
4929         const PetscInt    *perm = perms ? perms[p] : NULL;
4930         const PetscScalar *flip = flips ? flips[p] : NULL;
4931         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4932         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4933       } break;
4934     case INSERT_ALL_VALUES:
4935       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4936         const PetscInt    point = points[2*p];
4937         const PetscInt    *perm = perms ? perms[p] : NULL;
4938         const PetscScalar *flip = flips ? flips[p] : NULL;
4939         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4940         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4941       } break;
4942     case INSERT_BC_VALUES:
4943       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4944         const PetscInt    point = points[2*p];
4945         const PetscInt    *perm = perms ? perms[p] : NULL;
4946         const PetscScalar *flip = flips ? flips[p] : NULL;
4947         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4948         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4949       } break;
4950     case ADD_VALUES:
4951       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4952         const PetscInt    point = points[2*p];
4953         const PetscInt    *perm = perms ? perms[p] : NULL;
4954         const PetscScalar *flip = flips ? flips[p] : NULL;
4955         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4956         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4957       } break;
4958     case ADD_ALL_VALUES:
4959       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4960         const PetscInt    point = points[2*p];
4961         const PetscInt    *perm = perms ? perms[p] : NULL;
4962         const PetscScalar *flip = flips ? flips[p] : NULL;
4963         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4964         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4965       } break;
4966     case ADD_BC_VALUES:
4967       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4968         const PetscInt    point = points[2*p];
4969         const PetscInt    *perm = perms ? perms[p] : NULL;
4970         const PetscScalar *flip = flips ? flips[p] : NULL;
4971         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4972         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4973       } break;
4974     default:
4975       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4976     }
4977     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4978   }
4979   /* Cleanup points */
4980   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4981   /* Cleanup array */
4982   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4983   PetscFunctionReturn(0);
4984 }
4985 
4986 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4987 {
4988   PetscSection      clSection;
4989   IS                clPoints;
4990   PetscScalar       *array;
4991   PetscInt          *points = NULL;
4992   const PetscInt    *clp, *clperm;
4993   PetscInt          numFields, numPoints, p;
4994   PetscInt          offset = 0, f;
4995   PetscErrorCode    ierr;
4996 
4997   PetscFunctionBeginHot;
4998   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4999   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5000   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5001   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5002   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5003   /* Get points */
5004   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5005   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5006   /* Get array */
5007   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5008   /* Get values */
5009   for (f = 0; f < numFields; ++f) {
5010     const PetscInt    **perms = NULL;
5011     const PetscScalar **flips = NULL;
5012 
5013     if (!fieldActive[f]) {
5014       for (p = 0; p < numPoints*2; p += 2) {
5015         PetscInt fdof;
5016         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5017         offset += fdof;
5018       }
5019       continue;
5020     }
5021     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5022     switch (mode) {
5023     case INSERT_VALUES:
5024       for (p = 0; p < numPoints; p++) {
5025         const PetscInt    point = points[2*p];
5026         const PetscInt    *perm = perms ? perms[p] : NULL;
5027         const PetscScalar *flip = flips ? flips[p] : NULL;
5028         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5029       } break;
5030     case INSERT_ALL_VALUES:
5031       for (p = 0; p < numPoints; p++) {
5032         const PetscInt    point = points[2*p];
5033         const PetscInt    *perm = perms ? perms[p] : NULL;
5034         const PetscScalar *flip = flips ? flips[p] : NULL;
5035         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5036         } break;
5037     case INSERT_BC_VALUES:
5038       for (p = 0; p < numPoints; p++) {
5039         const PetscInt    point = points[2*p];
5040         const PetscInt    *perm = perms ? perms[p] : NULL;
5041         const PetscScalar *flip = flips ? flips[p] : NULL;
5042         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
5043       } break;
5044     case ADD_VALUES:
5045       for (p = 0; p < numPoints; p++) {
5046         const PetscInt    point = points[2*p];
5047         const PetscInt    *perm = perms ? perms[p] : NULL;
5048         const PetscScalar *flip = flips ? flips[p] : NULL;
5049         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5050       } break;
5051     case ADD_ALL_VALUES:
5052       for (p = 0; p < numPoints; p++) {
5053         const PetscInt    point = points[2*p];
5054         const PetscInt    *perm = perms ? perms[p] : NULL;
5055         const PetscScalar *flip = flips ? flips[p] : NULL;
5056         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5057       } break;
5058     default:
5059       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5060     }
5061     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5062   }
5063   /* Cleanup points */
5064   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5065   /* Cleanup array */
5066   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5067   PetscFunctionReturn(0);
5068 }
5069 
5070 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5071 {
5072   PetscMPIInt    rank;
5073   PetscInt       i, j;
5074   PetscErrorCode ierr;
5075 
5076   PetscFunctionBegin;
5077   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5078   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5079   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5080   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5081   numCIndices = numCIndices ? numCIndices : numRIndices;
5082   for (i = 0; i < numRIndices; i++) {
5083     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5084     for (j = 0; j < numCIndices; j++) {
5085 #if defined(PETSC_USE_COMPLEX)
5086       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5087 #else
5088       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5089 #endif
5090     }
5091     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5092   }
5093   PetscFunctionReturn(0);
5094 }
5095 
5096 /*
5097   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5098 
5099   Input Parameters:
5100 + section - The section for this data layout
5101 . point   - The point contributing dofs with these indices
5102 . off     - The global offset of this point
5103 . loff    - The local offset of each field
5104 . setBC   - The flag determining whether to include indices of bounsary values
5105 . perm    - A permutation of the dofs on this point, or NULL
5106 - indperm - A permutation of the entire indices array, or NULL
5107 
5108   Output Parameter:
5109 . indices - Indices for dofs on this point
5110 
5111   Level: developer
5112 
5113   Note: The indices could be local or global, depending on the value of 'off'.
5114 */
5115 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5116 {
5117   PetscInt        dof;   /* The number of unknowns on this point */
5118   PetscInt        cdof;  /* The number of constraints on this point */
5119   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5120   PetscInt        cind = 0, k;
5121   PetscErrorCode  ierr;
5122 
5123   PetscFunctionBegin;
5124   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5125   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5126   if (!cdof || setBC) {
5127     for (k = 0; k < dof; ++k) {
5128       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5129       const PetscInt ind    = indperm ? indperm[preind] : preind;
5130 
5131       indices[ind] = off + k;
5132     }
5133   } else {
5134     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5135     for (k = 0; k < dof; ++k) {
5136       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5137       const PetscInt ind    = indperm ? indperm[preind] : preind;
5138 
5139       if ((cind < cdof) && (k == cdofs[cind])) {
5140         /* Insert check for returning constrained indices */
5141         indices[ind] = -(off+k+1);
5142         ++cind;
5143       } else {
5144         indices[ind] = off+k-cind;
5145       }
5146     }
5147   }
5148   *loff += dof;
5149   PetscFunctionReturn(0);
5150 }
5151 
5152 /*
5153   This version only believes the point offset from the globalSection
5154 
5155  . off - The global offset of this point
5156 */
5157 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5158 {
5159   PetscInt       numFields, foff, f;
5160   PetscErrorCode ierr;
5161 
5162   PetscFunctionBegin;
5163   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5164   for (f = 0, foff = 0; f < numFields; ++f) {
5165     PetscInt        fdof, cfdof;
5166     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5167     PetscInt        cind = 0, b;
5168     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5169 
5170     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5171     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5172     if (!cfdof || setBC) {
5173       for (b = 0; b < fdof; ++b) {
5174         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5175         const PetscInt ind    = indperm ? indperm[preind] : preind;
5176 
5177         indices[ind] = off+foff+b;
5178       }
5179     } else {
5180       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5181       for (b = 0; b < fdof; ++b) {
5182         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5183         const PetscInt ind    = indperm ? indperm[preind] : preind;
5184 
5185         if ((cind < cfdof) && (b == fcdofs[cind])) {
5186           indices[ind] = -(off+foff+b+1);
5187           ++cind;
5188         } else {
5189           indices[ind] = off+foff+b-cind;
5190         }
5191       }
5192     }
5193     foff     += (setBC ? fdof : (fdof - cfdof));
5194     foffs[f] += fdof;
5195   }
5196   PetscFunctionReturn(0);
5197 }
5198 
5199 /*
5200   This version believes the globalSection offsets for each field, rather than just the point offset
5201 
5202  . foffs - The offset into 'indices' for each field, since it is segregated by field
5203 */
5204 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5205 {
5206   PetscInt       numFields, foff, f;
5207   PetscErrorCode ierr;
5208 
5209   PetscFunctionBegin;
5210   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5211   for (f = 0; f < numFields; ++f) {
5212     PetscInt        fdof, cfdof;
5213     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5214     PetscInt        cind = 0, b;
5215     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5216 
5217     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5218     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5219     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5220     if (!cfdof || setBC) {
5221       for (b = 0; b < fdof; ++b) {
5222         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5223         const PetscInt ind    = indperm ? indperm[preind] : preind;
5224 
5225         indices[ind] = foff+b;
5226       }
5227     } else {
5228       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5229       for (b = 0; b < fdof; ++b) {
5230         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5231         const PetscInt ind    = indperm ? indperm[preind] : preind;
5232 
5233         if ((cind < cfdof) && (b == fcdofs[cind])) {
5234           indices[ind] = -(foff+b+1);
5235           ++cind;
5236         } else {
5237           indices[ind] = foff+b-cind;
5238         }
5239       }
5240     }
5241     foffs[f] += fdof;
5242   }
5243   PetscFunctionReturn(0);
5244 }
5245 
5246 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5247 {
5248   Mat             cMat;
5249   PetscSection    aSec, cSec;
5250   IS              aIS;
5251   PetscInt        aStart = -1, aEnd = -1;
5252   const PetscInt  *anchors;
5253   PetscInt        numFields, f, p, q, newP = 0;
5254   PetscInt        newNumPoints = 0, newNumIndices = 0;
5255   PetscInt        *newPoints, *indices, *newIndices;
5256   PetscInt        maxAnchor, maxDof;
5257   PetscInt        newOffsets[32];
5258   PetscInt        *pointMatOffsets[32];
5259   PetscInt        *newPointOffsets[32];
5260   PetscScalar     *pointMat[32];
5261   PetscScalar     *newValues=NULL,*tmpValues;
5262   PetscBool       anyConstrained = PETSC_FALSE;
5263   PetscErrorCode  ierr;
5264 
5265   PetscFunctionBegin;
5266   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5267   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5268   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5269 
5270   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5271   /* if there are point-to-point constraints */
5272   if (aSec) {
5273     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5274     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5275     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5276     /* figure out how many points are going to be in the new element matrix
5277      * (we allow double counting, because it's all just going to be summed
5278      * into the global matrix anyway) */
5279     for (p = 0; p < 2*numPoints; p+=2) {
5280       PetscInt b    = points[p];
5281       PetscInt bDof = 0, bSecDof;
5282 
5283       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5284       if (!bSecDof) {
5285         continue;
5286       }
5287       if (b >= aStart && b < aEnd) {
5288         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5289       }
5290       if (bDof) {
5291         /* this point is constrained */
5292         /* it is going to be replaced by its anchors */
5293         PetscInt bOff, q;
5294 
5295         anyConstrained = PETSC_TRUE;
5296         newNumPoints  += bDof;
5297         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5298         for (q = 0; q < bDof; q++) {
5299           PetscInt a = anchors[bOff + q];
5300           PetscInt aDof;
5301 
5302           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5303           newNumIndices += aDof;
5304           for (f = 0; f < numFields; ++f) {
5305             PetscInt fDof;
5306 
5307             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5308             newOffsets[f+1] += fDof;
5309           }
5310         }
5311       }
5312       else {
5313         /* this point is not constrained */
5314         newNumPoints++;
5315         newNumIndices += bSecDof;
5316         for (f = 0; f < numFields; ++f) {
5317           PetscInt fDof;
5318 
5319           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5320           newOffsets[f+1] += fDof;
5321         }
5322       }
5323     }
5324   }
5325   if (!anyConstrained) {
5326     if (outNumPoints)  *outNumPoints  = 0;
5327     if (outNumIndices) *outNumIndices = 0;
5328     if (outPoints)     *outPoints     = NULL;
5329     if (outValues)     *outValues     = NULL;
5330     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5331     PetscFunctionReturn(0);
5332   }
5333 
5334   if (outNumPoints)  *outNumPoints  = newNumPoints;
5335   if (outNumIndices) *outNumIndices = newNumIndices;
5336 
5337   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5338 
5339   if (!outPoints && !outValues) {
5340     if (offsets) {
5341       for (f = 0; f <= numFields; f++) {
5342         offsets[f] = newOffsets[f];
5343       }
5344     }
5345     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5346     PetscFunctionReturn(0);
5347   }
5348 
5349   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5350 
5351   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5352 
5353   /* workspaces */
5354   if (numFields) {
5355     for (f = 0; f < numFields; f++) {
5356       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5357       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5358     }
5359   }
5360   else {
5361     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5362     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5363   }
5364 
5365   /* get workspaces for the point-to-point matrices */
5366   if (numFields) {
5367     PetscInt totalOffset, totalMatOffset;
5368 
5369     for (p = 0; p < numPoints; p++) {
5370       PetscInt b    = points[2*p];
5371       PetscInt bDof = 0, bSecDof;
5372 
5373       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5374       if (!bSecDof) {
5375         for (f = 0; f < numFields; f++) {
5376           newPointOffsets[f][p + 1] = 0;
5377           pointMatOffsets[f][p + 1] = 0;
5378         }
5379         continue;
5380       }
5381       if (b >= aStart && b < aEnd) {
5382         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5383       }
5384       if (bDof) {
5385         for (f = 0; f < numFields; f++) {
5386           PetscInt fDof, q, bOff, allFDof = 0;
5387 
5388           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5389           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5390           for (q = 0; q < bDof; q++) {
5391             PetscInt a = anchors[bOff + q];
5392             PetscInt aFDof;
5393 
5394             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5395             allFDof += aFDof;
5396           }
5397           newPointOffsets[f][p+1] = allFDof;
5398           pointMatOffsets[f][p+1] = fDof * allFDof;
5399         }
5400       }
5401       else {
5402         for (f = 0; f < numFields; f++) {
5403           PetscInt fDof;
5404 
5405           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5406           newPointOffsets[f][p+1] = fDof;
5407           pointMatOffsets[f][p+1] = 0;
5408         }
5409       }
5410     }
5411     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5412       newPointOffsets[f][0] = totalOffset;
5413       pointMatOffsets[f][0] = totalMatOffset;
5414       for (p = 0; p < numPoints; p++) {
5415         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5416         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5417       }
5418       totalOffset    = newPointOffsets[f][numPoints];
5419       totalMatOffset = pointMatOffsets[f][numPoints];
5420       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5421     }
5422   }
5423   else {
5424     for (p = 0; p < numPoints; p++) {
5425       PetscInt b    = points[2*p];
5426       PetscInt bDof = 0, bSecDof;
5427 
5428       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5429       if (!bSecDof) {
5430         newPointOffsets[0][p + 1] = 0;
5431         pointMatOffsets[0][p + 1] = 0;
5432         continue;
5433       }
5434       if (b >= aStart && b < aEnd) {
5435         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5436       }
5437       if (bDof) {
5438         PetscInt bOff, q, allDof = 0;
5439 
5440         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5441         for (q = 0; q < bDof; q++) {
5442           PetscInt a = anchors[bOff + q], aDof;
5443 
5444           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5445           allDof += aDof;
5446         }
5447         newPointOffsets[0][p+1] = allDof;
5448         pointMatOffsets[0][p+1] = bSecDof * allDof;
5449       }
5450       else {
5451         newPointOffsets[0][p+1] = bSecDof;
5452         pointMatOffsets[0][p+1] = 0;
5453       }
5454     }
5455     newPointOffsets[0][0] = 0;
5456     pointMatOffsets[0][0] = 0;
5457     for (p = 0; p < numPoints; p++) {
5458       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5459       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5460     }
5461     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5462   }
5463 
5464   /* output arrays */
5465   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5466 
5467   /* get the point-to-point matrices; construct newPoints */
5468   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5469   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5470   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5471   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5472   if (numFields) {
5473     for (p = 0, newP = 0; p < numPoints; p++) {
5474       PetscInt b    = points[2*p];
5475       PetscInt o    = points[2*p+1];
5476       PetscInt bDof = 0, bSecDof;
5477 
5478       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5479       if (!bSecDof) {
5480         continue;
5481       }
5482       if (b >= aStart && b < aEnd) {
5483         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5484       }
5485       if (bDof) {
5486         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5487 
5488         fStart[0] = 0;
5489         fEnd[0]   = 0;
5490         for (f = 0; f < numFields; f++) {
5491           PetscInt fDof;
5492 
5493           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5494           fStart[f+1] = fStart[f] + fDof;
5495           fEnd[f+1]   = fStart[f+1];
5496         }
5497         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5498         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5499 
5500         fAnchorStart[0] = 0;
5501         fAnchorEnd[0]   = 0;
5502         for (f = 0; f < numFields; f++) {
5503           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5504 
5505           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5506           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5507         }
5508         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5509         for (q = 0; q < bDof; q++) {
5510           PetscInt a = anchors[bOff + q], aOff;
5511 
5512           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5513           newPoints[2*(newP + q)]     = a;
5514           newPoints[2*(newP + q) + 1] = 0;
5515           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5516           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5517         }
5518         newP += bDof;
5519 
5520         if (outValues) {
5521           /* get the point-to-point submatrix */
5522           for (f = 0; f < numFields; f++) {
5523             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5524           }
5525         }
5526       }
5527       else {
5528         newPoints[2 * newP]     = b;
5529         newPoints[2 * newP + 1] = o;
5530         newP++;
5531       }
5532     }
5533   } else {
5534     for (p = 0; p < numPoints; p++) {
5535       PetscInt b    = points[2*p];
5536       PetscInt o    = points[2*p+1];
5537       PetscInt bDof = 0, bSecDof;
5538 
5539       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5540       if (!bSecDof) {
5541         continue;
5542       }
5543       if (b >= aStart && b < aEnd) {
5544         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5545       }
5546       if (bDof) {
5547         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5548 
5549         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5550         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5551 
5552         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5553         for (q = 0; q < bDof; q++) {
5554           PetscInt a = anchors[bOff + q], aOff;
5555 
5556           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5557 
5558           newPoints[2*(newP + q)]     = a;
5559           newPoints[2*(newP + q) + 1] = 0;
5560           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5561           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5562         }
5563         newP += bDof;
5564 
5565         /* get the point-to-point submatrix */
5566         if (outValues) {
5567           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5568         }
5569       }
5570       else {
5571         newPoints[2 * newP]     = b;
5572         newPoints[2 * newP + 1] = o;
5573         newP++;
5574       }
5575     }
5576   }
5577 
5578   if (outValues) {
5579     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5580     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5581     /* multiply constraints on the right */
5582     if (numFields) {
5583       for (f = 0; f < numFields; f++) {
5584         PetscInt oldOff = offsets[f];
5585 
5586         for (p = 0; p < numPoints; p++) {
5587           PetscInt cStart = newPointOffsets[f][p];
5588           PetscInt b      = points[2 * p];
5589           PetscInt c, r, k;
5590           PetscInt dof;
5591 
5592           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5593           if (!dof) {
5594             continue;
5595           }
5596           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5597             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5598             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5599 
5600             for (r = 0; r < numIndices; r++) {
5601               for (c = 0; c < nCols; c++) {
5602                 for (k = 0; k < dof; k++) {
5603                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5604                 }
5605               }
5606             }
5607           }
5608           else {
5609             /* copy this column as is */
5610             for (r = 0; r < numIndices; r++) {
5611               for (c = 0; c < dof; c++) {
5612                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5613               }
5614             }
5615           }
5616           oldOff += dof;
5617         }
5618       }
5619     }
5620     else {
5621       PetscInt oldOff = 0;
5622       for (p = 0; p < numPoints; p++) {
5623         PetscInt cStart = newPointOffsets[0][p];
5624         PetscInt b      = points[2 * p];
5625         PetscInt c, r, k;
5626         PetscInt dof;
5627 
5628         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5629         if (!dof) {
5630           continue;
5631         }
5632         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5633           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5634           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5635 
5636           for (r = 0; r < numIndices; r++) {
5637             for (c = 0; c < nCols; c++) {
5638               for (k = 0; k < dof; k++) {
5639                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5640               }
5641             }
5642           }
5643         }
5644         else {
5645           /* copy this column as is */
5646           for (r = 0; r < numIndices; r++) {
5647             for (c = 0; c < dof; c++) {
5648               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5649             }
5650           }
5651         }
5652         oldOff += dof;
5653       }
5654     }
5655 
5656     if (multiplyLeft) {
5657       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5658       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5659       /* multiply constraints transpose on the left */
5660       if (numFields) {
5661         for (f = 0; f < numFields; f++) {
5662           PetscInt oldOff = offsets[f];
5663 
5664           for (p = 0; p < numPoints; p++) {
5665             PetscInt rStart = newPointOffsets[f][p];
5666             PetscInt b      = points[2 * p];
5667             PetscInt c, r, k;
5668             PetscInt dof;
5669 
5670             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5671             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5672               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5673               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5674 
5675               for (r = 0; r < nRows; r++) {
5676                 for (c = 0; c < newNumIndices; c++) {
5677                   for (k = 0; k < dof; k++) {
5678                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5679                   }
5680                 }
5681               }
5682             }
5683             else {
5684               /* copy this row as is */
5685               for (r = 0; r < dof; r++) {
5686                 for (c = 0; c < newNumIndices; c++) {
5687                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5688                 }
5689               }
5690             }
5691             oldOff += dof;
5692           }
5693         }
5694       }
5695       else {
5696         PetscInt oldOff = 0;
5697 
5698         for (p = 0; p < numPoints; p++) {
5699           PetscInt rStart = newPointOffsets[0][p];
5700           PetscInt b      = points[2 * p];
5701           PetscInt c, r, k;
5702           PetscInt dof;
5703 
5704           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5705           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5706             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5707             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5708 
5709             for (r = 0; r < nRows; r++) {
5710               for (c = 0; c < newNumIndices; c++) {
5711                 for (k = 0; k < dof; k++) {
5712                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5713                 }
5714               }
5715             }
5716           }
5717           else {
5718             /* copy this row as is */
5719             for (r = 0; r < dof; r++) {
5720               for (c = 0; c < newNumIndices; c++) {
5721                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5722               }
5723             }
5724           }
5725           oldOff += dof;
5726         }
5727       }
5728 
5729       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5730     }
5731     else {
5732       newValues = tmpValues;
5733     }
5734   }
5735 
5736   /* clean up */
5737   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5738   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5739 
5740   if (numFields) {
5741     for (f = 0; f < numFields; f++) {
5742       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5743       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5744       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5745     }
5746   }
5747   else {
5748     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5749     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5750     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5751   }
5752   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5753 
5754   /* output */
5755   if (outPoints) {
5756     *outPoints = newPoints;
5757   }
5758   else {
5759     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5760   }
5761   if (outValues) {
5762     *outValues = newValues;
5763   }
5764   for (f = 0; f <= numFields; f++) {
5765     offsets[f] = newOffsets[f];
5766   }
5767   PetscFunctionReturn(0);
5768 }
5769 
5770 /*@C
5771   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5772 
5773   Not collective
5774 
5775   Input Parameters:
5776 + dm - The DM
5777 . section - The section describing the layout in v, or NULL to use the default section
5778 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5779 - point - The mesh point
5780 
5781   Output parameters:
5782 + numIndices - The number of indices
5783 . indices - The indices
5784 - outOffsets - Field offset if not NULL
5785 
5786   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5787 
5788   Level: advanced
5789 
5790 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5791 @*/
5792 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5793 {
5794   PetscSection    clSection;
5795   IS              clPoints;
5796   const PetscInt *clp, *clperm;
5797   const PetscInt  **perms[32] = {NULL};
5798   PetscInt       *points = NULL, *pointsNew;
5799   PetscInt        numPoints, numPointsNew;
5800   PetscInt        offsets[32];
5801   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5802   PetscErrorCode  ierr;
5803 
5804   PetscFunctionBegin;
5805   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5806   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5807   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5808   if (numIndices) PetscValidPointer(numIndices, 4);
5809   PetscValidPointer(indices, 5);
5810   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5811   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5812   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5813   /* Get points in closure */
5814   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5815   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5816   /* Get number of indices and indices per field */
5817   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5818     PetscInt dof, fdof;
5819 
5820     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5821     for (f = 0; f < Nf; ++f) {
5822       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5823       offsets[f+1] += fdof;
5824     }
5825     Nind += dof;
5826   }
5827   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5828   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5829   if (!Nf) offsets[1] = Nind;
5830   /* Get dual space symmetries */
5831   for (f = 0; f < PetscMax(1,Nf); f++) {
5832     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5833     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5834   }
5835   /* Correct for hanging node constraints */
5836   {
5837     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5838     if (numPointsNew) {
5839       for (f = 0; f < PetscMax(1,Nf); f++) {
5840         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5841         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5842       }
5843       for (f = 0; f < PetscMax(1,Nf); f++) {
5844         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5845         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5846       }
5847       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5848       numPoints = numPointsNew;
5849       Nind      = NindNew;
5850       points    = pointsNew;
5851     }
5852   }
5853   /* Calculate indices */
5854   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5855   if (Nf) {
5856     if (outOffsets) {
5857       PetscInt f;
5858 
5859       for (f = 0; f <= Nf; f++) {
5860         outOffsets[f] = offsets[f];
5861       }
5862     }
5863     for (p = 0; p < numPoints; p++) {
5864       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5865       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);
5866     }
5867   } else {
5868     for (p = 0, off = 0; p < numPoints; p++) {
5869       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5870 
5871       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5872       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);
5873     }
5874   }
5875   /* Cleanup points */
5876   for (f = 0; f < PetscMax(1,Nf); f++) {
5877     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5878     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5879   }
5880   if (numPointsNew) {
5881     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5882   } else {
5883     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5884   }
5885   if (numIndices) *numIndices = Nind;
5886   PetscFunctionReturn(0);
5887 }
5888 
5889 /*@C
5890   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5891 
5892   Not collective
5893 
5894   Input Parameters:
5895 + dm - The DM
5896 . section - The section describing the layout in v, or NULL to use the default section
5897 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5898 . point - The mesh point
5899 . numIndices - The number of indices
5900 . indices - The indices
5901 - outOffsets - Field offset if not NULL
5902 
5903   Level: advanced
5904 
5905 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5906 @*/
5907 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5908 {
5909   PetscErrorCode ierr;
5910 
5911   PetscFunctionBegin;
5912   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5913   PetscValidPointer(indices, 5);
5914   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5915   PetscFunctionReturn(0);
5916 }
5917 
5918 /*@C
5919   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5920 
5921   Not collective
5922 
5923   Input Parameters:
5924 + dm - The DM
5925 . section - The section describing the layout in v, or NULL to use the default section
5926 . globalSection - The section describing the layout in v, or NULL to use the default global section
5927 . A - The matrix
5928 . point - The point in the DM
5929 . values - The array of values
5930 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5931 
5932   Fortran Notes:
5933   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5934 
5935   Level: intermediate
5936 
5937 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5938 @*/
5939 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5940 {
5941   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5942   PetscSection        clSection;
5943   IS                  clPoints;
5944   PetscInt           *points = NULL, *newPoints;
5945   const PetscInt     *clp, *clperm;
5946   PetscInt           *indices;
5947   PetscInt            offsets[32];
5948   const PetscInt    **perms[32] = {NULL};
5949   const PetscScalar **flips[32] = {NULL};
5950   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5951   PetscScalar        *valCopy = NULL;
5952   PetscScalar        *newValues;
5953   PetscErrorCode      ierr;
5954 
5955   PetscFunctionBegin;
5956   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5957   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5958   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5959   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5960   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5961   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5962   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5963   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5964   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5965   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5966   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5967   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5968     PetscInt fdof;
5969 
5970     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5971     for (f = 0; f < numFields; ++f) {
5972       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5973       offsets[f+1] += fdof;
5974     }
5975     numIndices += dof;
5976   }
5977   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5978 
5979   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5980   /* Get symmetries */
5981   for (f = 0; f < PetscMax(1,numFields); f++) {
5982     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5983     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5984     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5985       PetscInt foffset = offsets[f];
5986 
5987       for (p = 0; p < numPoints; p++) {
5988         PetscInt point          = points[2*p], fdof;
5989         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5990 
5991         if (!numFields) {
5992           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5993         } else {
5994           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5995         }
5996         if (flip) {
5997           PetscInt i, j, k;
5998 
5999           if (!valCopy) {
6000             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6001             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6002             values = valCopy;
6003           }
6004           for (i = 0; i < fdof; i++) {
6005             PetscScalar fval = flip[i];
6006 
6007             for (k = 0; k < numIndices; k++) {
6008               valCopy[numIndices * (foffset + i) + k] *= fval;
6009               valCopy[numIndices * k + (foffset + i)] *= fval;
6010             }
6011           }
6012         }
6013         foffset += fdof;
6014       }
6015     }
6016   }
6017   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6018   if (newNumPoints) {
6019     if (valCopy) {
6020       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6021     }
6022     for (f = 0; f < PetscMax(1,numFields); f++) {
6023       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6024       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6025     }
6026     for (f = 0; f < PetscMax(1,numFields); f++) {
6027       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6028       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6029     }
6030     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6031     numPoints  = newNumPoints;
6032     numIndices = newNumIndices;
6033     points     = newPoints;
6034     values     = newValues;
6035   }
6036   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6037   if (numFields) {
6038     PetscBool useFieldOffsets;
6039 
6040     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6041     if (useFieldOffsets) {
6042       for (p = 0; p < numPoints; p++) {
6043         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
6044       }
6045     } else {
6046       for (p = 0; p < numPoints; p++) {
6047         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6048         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
6049       }
6050     }
6051   } else {
6052     for (p = 0, off = 0; p < numPoints; p++) {
6053       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6054       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6055       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
6056     }
6057   }
6058   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6059   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
6060   if (mesh->printFEM > 1) {
6061     PetscInt i;
6062     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6063     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6064     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6065   }
6066   if (ierr) {
6067     PetscMPIInt    rank;
6068     PetscErrorCode ierr2;
6069 
6070     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6071     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6072     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6073     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6074     CHKERRQ(ierr);
6075   }
6076   for (f = 0; f < PetscMax(1,numFields); f++) {
6077     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6078     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6079   }
6080   if (newNumPoints) {
6081     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6082     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6083   }
6084   else {
6085     if (valCopy) {
6086       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6087     }
6088     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6089   }
6090   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6091   PetscFunctionReturn(0);
6092 }
6093 
6094 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6095 {
6096   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6097   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6098   PetscInt       *cpoints = NULL;
6099   PetscInt       *findices, *cindices;
6100   const PetscInt *fclperm, *cclperm;
6101   PetscInt        foffsets[32], coffsets[32];
6102   CellRefiner     cellRefiner;
6103   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6104   PetscErrorCode  ierr;
6105 
6106   PetscFunctionBegin;
6107   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6108   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6109   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6110   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6111   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6112   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6113   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6114   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6115   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6116   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6117   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6118   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6119   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6120   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6121   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6122   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6123   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6124   /* Column indices */
6125   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6126   maxFPoints = numCPoints;
6127   /* Compress out points not in the section */
6128   /*   TODO: Squeeze out points with 0 dof as well */
6129   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6130   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6131     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6132       cpoints[q*2]   = cpoints[p];
6133       cpoints[q*2+1] = cpoints[p+1];
6134       ++q;
6135     }
6136   }
6137   numCPoints = q;
6138   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6139     PetscInt fdof;
6140 
6141     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6142     if (!dof) continue;
6143     for (f = 0; f < numFields; ++f) {
6144       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6145       coffsets[f+1] += fdof;
6146     }
6147     numCIndices += dof;
6148   }
6149   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6150   /* Row indices */
6151   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6152   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6153   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6154   for (r = 0, q = 0; r < numSubcells; ++r) {
6155     /* TODO Map from coarse to fine cells */
6156     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6157     /* Compress out points not in the section */
6158     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6159     for (p = 0; p < numFPoints*2; p += 2) {
6160       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6161         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6162         if (!dof) continue;
6163         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6164         if (s < q) continue;
6165         ftotpoints[q*2]   = fpoints[p];
6166         ftotpoints[q*2+1] = fpoints[p+1];
6167         ++q;
6168       }
6169     }
6170     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6171   }
6172   numFPoints = q;
6173   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6174     PetscInt fdof;
6175 
6176     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6177     if (!dof) continue;
6178     for (f = 0; f < numFields; ++f) {
6179       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6180       foffsets[f+1] += fdof;
6181     }
6182     numFIndices += dof;
6183   }
6184   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6185 
6186   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6187   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6188   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6189   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6190   if (numFields) {
6191     const PetscInt **permsF[32] = {NULL};
6192     const PetscInt **permsC[32] = {NULL};
6193 
6194     for (f = 0; f < numFields; f++) {
6195       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6196       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6197     }
6198     for (p = 0; p < numFPoints; p++) {
6199       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6200       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6201     }
6202     for (p = 0; p < numCPoints; p++) {
6203       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6204       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6205     }
6206     for (f = 0; f < numFields; f++) {
6207       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6208       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6209     }
6210   } else {
6211     const PetscInt **permsF = NULL;
6212     const PetscInt **permsC = NULL;
6213 
6214     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6215     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6216     for (p = 0, off = 0; p < numFPoints; p++) {
6217       const PetscInt *perm = permsF ? permsF[p] : NULL;
6218 
6219       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6220       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6221     }
6222     for (p = 0, off = 0; p < numCPoints; p++) {
6223       const PetscInt *perm = permsC ? permsC[p] : NULL;
6224 
6225       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6226       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6227     }
6228     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6229     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6230   }
6231   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6232   /* TODO: flips */
6233   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6234   if (ierr) {
6235     PetscMPIInt    rank;
6236     PetscErrorCode ierr2;
6237 
6238     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6239     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6240     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6241     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6242     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6243     CHKERRQ(ierr);
6244   }
6245   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6246   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6247   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6248   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6249   PetscFunctionReturn(0);
6250 }
6251 
6252 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6253 {
6254   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6255   PetscInt      *cpoints = NULL;
6256   PetscInt       foffsets[32], coffsets[32];
6257   const PetscInt *fclperm, *cclperm;
6258   CellRefiner    cellRefiner;
6259   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6260   PetscErrorCode ierr;
6261 
6262   PetscFunctionBegin;
6263   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6264   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6265   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6266   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6267   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6268   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6269   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6270   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6271   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6272   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6273   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6274   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6275   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6276   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6277   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6278   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6279   /* Column indices */
6280   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6281   maxFPoints = numCPoints;
6282   /* Compress out points not in the section */
6283   /*   TODO: Squeeze out points with 0 dof as well */
6284   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6285   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6286     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6287       cpoints[q*2]   = cpoints[p];
6288       cpoints[q*2+1] = cpoints[p+1];
6289       ++q;
6290     }
6291   }
6292   numCPoints = q;
6293   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6294     PetscInt fdof;
6295 
6296     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6297     if (!dof) continue;
6298     for (f = 0; f < numFields; ++f) {
6299       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6300       coffsets[f+1] += fdof;
6301     }
6302     numCIndices += dof;
6303   }
6304   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6305   /* Row indices */
6306   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6307   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6308   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6309   for (r = 0, q = 0; r < numSubcells; ++r) {
6310     /* TODO Map from coarse to fine cells */
6311     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6312     /* Compress out points not in the section */
6313     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6314     for (p = 0; p < numFPoints*2; p += 2) {
6315       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6316         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6317         if (!dof) continue;
6318         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6319         if (s < q) continue;
6320         ftotpoints[q*2]   = fpoints[p];
6321         ftotpoints[q*2+1] = fpoints[p+1];
6322         ++q;
6323       }
6324     }
6325     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6326   }
6327   numFPoints = q;
6328   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6329     PetscInt fdof;
6330 
6331     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6332     if (!dof) continue;
6333     for (f = 0; f < numFields; ++f) {
6334       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6335       foffsets[f+1] += fdof;
6336     }
6337     numFIndices += dof;
6338   }
6339   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6340 
6341   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6342   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6343   if (numFields) {
6344     const PetscInt **permsF[32] = {NULL};
6345     const PetscInt **permsC[32] = {NULL};
6346 
6347     for (f = 0; f < numFields; f++) {
6348       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6349       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6350     }
6351     for (p = 0; p < numFPoints; p++) {
6352       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6353       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6354     }
6355     for (p = 0; p < numCPoints; p++) {
6356       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6357       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6358     }
6359     for (f = 0; f < numFields; f++) {
6360       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6361       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6362     }
6363   } else {
6364     const PetscInt **permsF = NULL;
6365     const PetscInt **permsC = NULL;
6366 
6367     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6368     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6369     for (p = 0, off = 0; p < numFPoints; p++) {
6370       const PetscInt *perm = permsF ? permsF[p] : NULL;
6371 
6372       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6373       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6374     }
6375     for (p = 0, off = 0; p < numCPoints; p++) {
6376       const PetscInt *perm = permsC ? permsC[p] : NULL;
6377 
6378       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6379       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6380     }
6381     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6382     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6383   }
6384   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6385   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6386   PetscFunctionReturn(0);
6387 }
6388 
6389 /*@
6390   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6391 
6392   Input Parameter:
6393 . dm - The DMPlex object
6394 
6395   Output Parameters:
6396 + cMax - The first hybrid cell
6397 . fMax - The first hybrid face
6398 . eMax - The first hybrid edge
6399 - vMax - The first hybrid vertex
6400 
6401   Level: developer
6402 
6403 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6404 @*/
6405 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6406 {
6407   DM_Plex       *mesh = (DM_Plex*) dm->data;
6408   PetscInt       dim;
6409   PetscErrorCode ierr;
6410 
6411   PetscFunctionBegin;
6412   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6413   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6414   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6415   if (cMax) *cMax = mesh->hybridPointMax[dim];
6416   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6417   if (eMax) *eMax = mesh->hybridPointMax[1];
6418   if (vMax) *vMax = mesh->hybridPointMax[0];
6419   PetscFunctionReturn(0);
6420 }
6421 
6422 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6423 {
6424   IS             is, his;
6425   PetscInt       first = 0, stride;
6426   PetscBool      isStride;
6427   PetscErrorCode ierr;
6428 
6429   PetscFunctionBegin;
6430   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6431   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6432   if (isStride) {
6433     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6434   }
6435   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6436   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6437   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6438   ierr = ISDestroy(&his);CHKERRQ(ierr);
6439   ierr = ISDestroy(&is);CHKERRQ(ierr);
6440   PetscFunctionReturn(0);
6441 }
6442 
6443 /*@
6444   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6445 
6446   Input Parameters:
6447 + dm   - The DMPlex object
6448 . cMax - The first hybrid cell
6449 . fMax - The first hybrid face
6450 . eMax - The first hybrid edge
6451 - vMax - The first hybrid vertex
6452 
6453   Level: developer
6454 
6455 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6456 @*/
6457 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6458 {
6459   DM_Plex       *mesh = (DM_Plex*) dm->data;
6460   PetscInt       dim;
6461   PetscErrorCode ierr;
6462 
6463   PetscFunctionBegin;
6464   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6465   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6466   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6467   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6468   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6469   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6470   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6471   PetscFunctionReturn(0);
6472 }
6473 
6474 /*@C
6475   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6476 
6477   Input Parameter:
6478 . dm   - The DMPlex object
6479 
6480   Output Parameter:
6481 . cellHeight - The height of a cell
6482 
6483   Level: developer
6484 
6485 .seealso DMPlexSetVTKCellHeight()
6486 @*/
6487 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6488 {
6489   DM_Plex *mesh = (DM_Plex*) dm->data;
6490 
6491   PetscFunctionBegin;
6492   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6493   PetscValidPointer(cellHeight, 2);
6494   *cellHeight = mesh->vtkCellHeight;
6495   PetscFunctionReturn(0);
6496 }
6497 
6498 /*@C
6499   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6500 
6501   Input Parameters:
6502 + dm   - The DMPlex object
6503 - cellHeight - The height of a cell
6504 
6505   Level: developer
6506 
6507 .seealso DMPlexGetVTKCellHeight()
6508 @*/
6509 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6510 {
6511   DM_Plex *mesh = (DM_Plex*) dm->data;
6512 
6513   PetscFunctionBegin;
6514   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6515   mesh->vtkCellHeight = cellHeight;
6516   PetscFunctionReturn(0);
6517 }
6518 
6519 /* We can easily have a form that takes an IS instead */
6520 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6521 {
6522   PetscSection   section, globalSection;
6523   PetscInt      *numbers, p;
6524   PetscErrorCode ierr;
6525 
6526   PetscFunctionBegin;
6527   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6528   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6529   for (p = pStart; p < pEnd; ++p) {
6530     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6531   }
6532   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6533   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6534   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6535   for (p = pStart; p < pEnd; ++p) {
6536     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6537     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6538     else                       numbers[p-pStart] += shift;
6539   }
6540   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6541   if (globalSize) {
6542     PetscLayout layout;
6543     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6544     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6545     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6546   }
6547   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6548   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6549   PetscFunctionReturn(0);
6550 }
6551 
6552 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6553 {
6554   PetscInt       cellHeight, cStart, cEnd, cMax;
6555   PetscErrorCode ierr;
6556 
6557   PetscFunctionBegin;
6558   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6559   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6560   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6561   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6562   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6563   PetscFunctionReturn(0);
6564 }
6565 
6566 /*@
6567   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6568 
6569   Input Parameter:
6570 . dm   - The DMPlex object
6571 
6572   Output Parameter:
6573 . globalCellNumbers - Global cell numbers for all cells on this process
6574 
6575   Level: developer
6576 
6577 .seealso DMPlexGetVertexNumbering()
6578 @*/
6579 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6580 {
6581   DM_Plex       *mesh = (DM_Plex*) dm->data;
6582   PetscErrorCode ierr;
6583 
6584   PetscFunctionBegin;
6585   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6586   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6587   *globalCellNumbers = mesh->globalCellNumbers;
6588   PetscFunctionReturn(0);
6589 }
6590 
6591 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6592 {
6593   PetscInt       vStart, vEnd, vMax;
6594   PetscErrorCode ierr;
6595 
6596   PetscFunctionBegin;
6597   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6598   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6599   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6600   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6601   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6602   PetscFunctionReturn(0);
6603 }
6604 
6605 /*@
6606   DMPlexGetVertexNumbering - Get a global certex numbering for all vertices on this process
6607 
6608   Input Parameter:
6609 . dm   - The DMPlex object
6610 
6611   Output Parameter:
6612 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6613 
6614   Level: developer
6615 
6616 .seealso DMPlexGetCellNumbering()
6617 @*/
6618 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6619 {
6620   DM_Plex       *mesh = (DM_Plex*) dm->data;
6621   PetscErrorCode ierr;
6622 
6623   PetscFunctionBegin;
6624   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6625   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6626   *globalVertexNumbers = mesh->globalVertexNumbers;
6627   PetscFunctionReturn(0);
6628 }
6629 
6630 /*@
6631   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6632 
6633   Input Parameter:
6634 . dm   - The DMPlex object
6635 
6636   Output Parameter:
6637 . globalPointNumbers - Global numbers for all points on this process
6638 
6639   Level: developer
6640 
6641 .seealso DMPlexGetCellNumbering()
6642 @*/
6643 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6644 {
6645   IS             nums[4];
6646   PetscInt       depths[4], gdepths[4], starts[4];
6647   PetscInt       depth, d, shift = 0;
6648   PetscErrorCode ierr;
6649 
6650   PetscFunctionBegin;
6651   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6652   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6653   /* For unstratified meshes use dim instead of depth */
6654   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6655   for (d = 0; d <= depth; ++d) {
6656     PetscInt end;
6657 
6658     depths[d] = depth-d;
6659     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6660     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6661   }
6662   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6663   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6664   for (d = 0; d <= depth; ++d) {
6665     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6666   }
6667   for (d = 0; d <= depth; ++d) {
6668     PetscInt pStart, pEnd, gsize;
6669 
6670     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6671     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6672     shift += gsize;
6673   }
6674   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6675   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6676   PetscFunctionReturn(0);
6677 }
6678 
6679 
6680 /*@
6681   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6682 
6683   Input Parameter:
6684 . dm - The DMPlex object
6685 
6686   Output Parameter:
6687 . ranks - The rank field
6688 
6689   Options Database Keys:
6690 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6691 
6692   Level: intermediate
6693 
6694 .seealso: DMView()
6695 @*/
6696 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6697 {
6698   DM             rdm;
6699   PetscFE        fe;
6700   PetscScalar   *r;
6701   PetscMPIInt    rank;
6702   PetscInt       dim, cStart, cEnd, c;
6703   PetscErrorCode ierr;
6704 
6705   PetscFunctionBeginUser;
6706   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6707   PetscValidPointer(ranks, 2);
6708   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6709   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6710   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6711   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6712   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6713   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6714   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6715   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6716   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6717   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6718   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6719   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6720   for (c = cStart; c < cEnd; ++c) {
6721     PetscScalar *lr;
6722 
6723     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6724     *lr = rank;
6725   }
6726   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6727   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6728   PetscFunctionReturn(0);
6729 }
6730 
6731 /*@
6732   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6733 
6734   Input Parameters:
6735 + dm    - The DMPlex
6736 - label - The DMLabel
6737 
6738   Output Parameter:
6739 . val - The label value field
6740 
6741   Options Database Keys:
6742 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6743 
6744   Level: intermediate
6745 
6746 .seealso: DMView()
6747 @*/
6748 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6749 {
6750   DM             rdm;
6751   PetscFE        fe;
6752   PetscScalar   *v;
6753   PetscInt       dim, cStart, cEnd, c;
6754   PetscErrorCode ierr;
6755 
6756   PetscFunctionBeginUser;
6757   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6758   PetscValidPointer(label, 2);
6759   PetscValidPointer(val, 3);
6760   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6761   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6762   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6763   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6764   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6765   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6766   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6767   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6768   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6769   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6770   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6771   for (c = cStart; c < cEnd; ++c) {
6772     PetscScalar *lv;
6773     PetscInt     cval;
6774 
6775     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6776     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6777     *lv = cval;
6778   }
6779   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6780   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6781   PetscFunctionReturn(0);
6782 }
6783 
6784 /*@
6785   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6786 
6787   Input Parameter:
6788 . dm - The DMPlex object
6789 
6790   Note: This is a useful diagnostic when creating meshes programmatically.
6791 
6792   Level: developer
6793 
6794 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6795 @*/
6796 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6797 {
6798   PetscSection    coneSection, supportSection;
6799   const PetscInt *cone, *support;
6800   PetscInt        coneSize, c, supportSize, s;
6801   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6802   PetscBool       storagecheck = PETSC_TRUE;
6803   PetscErrorCode  ierr;
6804 
6805   PetscFunctionBegin;
6806   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6807   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6808   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6809   /* Check that point p is found in the support of its cone points, and vice versa */
6810   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6811   for (p = pStart; p < pEnd; ++p) {
6812     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6813     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6814     for (c = 0; c < coneSize; ++c) {
6815       PetscBool dup = PETSC_FALSE;
6816       PetscInt  d;
6817       for (d = c-1; d >= 0; --d) {
6818         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6819       }
6820       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6821       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6822       for (s = 0; s < supportSize; ++s) {
6823         if (support[s] == p) break;
6824       }
6825       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6826         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6827         for (s = 0; s < coneSize; ++s) {
6828           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6829         }
6830         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6831         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6832         for (s = 0; s < supportSize; ++s) {
6833           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6834         }
6835         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6836         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6837         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6838       }
6839     }
6840     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6841     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6842     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6843     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6844     for (s = 0; s < supportSize; ++s) {
6845       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6846       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6847       for (c = 0; c < coneSize; ++c) {
6848         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6849         if (cone[c] != pp) { c = 0; break; }
6850         if (cone[c] == p) break;
6851       }
6852       if (c >= coneSize) {
6853         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6854         for (c = 0; c < supportSize; ++c) {
6855           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6856         }
6857         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6858         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6859         for (c = 0; c < coneSize; ++c) {
6860           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6861         }
6862         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6863         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6864       }
6865     }
6866   }
6867   if (storagecheck) {
6868     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6869     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6870     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6871   }
6872   PetscFunctionReturn(0);
6873 }
6874 
6875 /*@
6876   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6877 
6878   Input Parameters:
6879 + dm - The DMPlex object
6880 - cellHeight - Normally 0
6881 
6882   Note: This is a useful diagnostic when creating meshes programmatically.
6883   Currently applicable only to homogeneous simplex or tensor meshes.
6884 
6885   Level: developer
6886 
6887 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6888 @*/
6889 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6890 {
6891   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6892   PetscBool      isSimplex = PETSC_FALSE;
6893   PetscErrorCode ierr;
6894 
6895   PetscFunctionBegin;
6896   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6897   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6898   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6899   if (cStart < cEnd) {
6900     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6901     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6902   }
6903   switch (dim) {
6904   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6905   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6906   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6907   default:
6908     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6909   }
6910   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6911   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6912   cMax = cMax >= 0 ? cMax : cEnd;
6913   for (c = cStart; c < cMax; ++c) {
6914     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6915 
6916     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6917     for (cl = 0; cl < closureSize*2; cl += 2) {
6918       const PetscInt p = closure[cl];
6919       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6920     }
6921     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6922     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6923   }
6924   for (c = cMax; c < cEnd; ++c) {
6925     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6926 
6927     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6928     for (cl = 0; cl < closureSize*2; cl += 2) {
6929       const PetscInt p = closure[cl];
6930       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6931     }
6932     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6933     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6934   }
6935   PetscFunctionReturn(0);
6936 }
6937 
6938 /*@
6939   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6940 
6941   Input Parameters:
6942 + dm - The DMPlex object
6943 - cellHeight - Normally 0
6944 
6945   Note: This is a useful diagnostic when creating meshes programmatically.
6946 
6947   Level: developer
6948 
6949 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6950 @*/
6951 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6952 {
6953   PetscInt       pMax[4];
6954   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6955   PetscErrorCode ierr;
6956 
6957   PetscFunctionBegin;
6958   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6959   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6960   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6961   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6962   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6963   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6964     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6965     for (c = cStart; c < cEnd; ++c) {
6966       const PetscInt *cone, *ornt, *faces;
6967       PetscInt        numFaces, faceSize, coneSize,f;
6968       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6969 
6970       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6971       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6972       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6973       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6974       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6975       for (cl = 0; cl < closureSize*2; cl += 2) {
6976         const PetscInt p = closure[cl];
6977         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6978       }
6979       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6980       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6981       for (f = 0; f < numFaces; ++f) {
6982         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6983 
6984         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6985         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6986           const PetscInt p = fclosure[cl];
6987           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6988         }
6989         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6990         for (v = 0; v < fnumCorners; ++v) {
6991           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6992         }
6993         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6994       }
6995       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6996       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6997     }
6998   }
6999   PetscFunctionReturn(0);
7000 }
7001 
7002 /*@
7003   DMPlexCheckGeometry - Check the geometry of mesh cells
7004 
7005   Input Parameter:
7006 . dm - The DMPlex object
7007 
7008   Note: This is a useful diagnostic when creating meshes programmatically.
7009 
7010   Level: developer
7011 
7012 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
7013 @*/
7014 PetscErrorCode DMPlexCheckGeometry(DM dm)
7015 {
7016   PetscReal      detJ, J[9], refVol = 1.0;
7017   PetscReal      vol;
7018   PetscInt       dim, depth, d, cStart, cEnd, c, cMax;
7019   PetscErrorCode ierr;
7020 
7021   PetscFunctionBegin;
7022   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7023   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7024   for (d = 0; d < dim; ++d) refVol *= 2.0;
7025   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7026   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7027   cMax = cMax < 0 ? cEnd : cMax;
7028   for (c = cStart; c < cMax; ++c) {
7029     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7030     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7031     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7032     if (depth > 1) {
7033       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7034       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7035       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7036     }
7037   }
7038   PetscFunctionReturn(0);
7039 }
7040 
7041 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
7042 {
7043   PetscInt i,l,n;
7044   const PetscInt *cone;
7045   PetscErrorCode ierr;
7046 
7047   PetscFunctionBegin;
7048   *missingPoint = -1;
7049   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
7050   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
7051   for (i=0; i<n; i++) {
7052     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
7053     if (l < 0) {
7054       *missingPoint = cone[i];
7055       break;
7056     }
7057   }
7058   PetscFunctionReturn(0);
7059 }
7060 
7061 /*@
7062   DMPlexCheckPointSF - Check that several sufficient conditions are met for the point SF of this plex.
7063 
7064   Input Parameters:
7065 . dm - The DMPlex object
7066 
7067   Note: This is mainly intended for debugging/testing purposes.
7068 
7069   Level: developer
7070 
7071 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7072 @*/
7073 PetscErrorCode DMPlexCheckPointSF(DM dm)
7074 {
7075   PetscSF sf;
7076   PetscInt d,depth,i,nleaves,p,plo,phi,missingPoint;
7077   const PetscInt *locals;
7078   PetscErrorCode ierr;
7079 
7080   PetscFunctionBegin;
7081   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7082   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7083   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
7084   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
7085 
7086   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7087   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
7088   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
7089   for (i=0; i<nleaves; i++) {
7090     p = locals[i];
7091     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
7092   }
7093 
7094   /* 2) if some point is in interface, then all its cone points must be also in interface  */
7095   for (i=0; i<nleaves; i++) {
7096     p = locals[i];
7097     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
7098     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
7099   }
7100   PetscFunctionReturn(0);
7101 }
7102 
7103 typedef struct cell_stats
7104 {
7105   PetscReal min, max, sum, squaresum;
7106   PetscInt  count;
7107 } cell_stats_t;
7108 
7109 static void cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7110 {
7111   PetscInt i, N = *len;
7112 
7113   for (i = 0; i < N; i++) {
7114     cell_stats_t *A = (cell_stats_t *) a;
7115     cell_stats_t *B = (cell_stats_t *) b;
7116 
7117     B->min = PetscMin(A->min,B->min);
7118     B->max = PetscMax(A->max,B->max);
7119     B->sum += A->sum;
7120     B->squaresum += A->squaresum;
7121     B->count += A->count;
7122   }
7123 }
7124 
7125 /*@
7126   DMPlexCheckCellShape - Checks the Jacobian of the mapping and computes some minimal statistics.
7127 
7128   Input Parameters:
7129 + dm - The DMPlex object
7130 - output - If true, statistics will be displayed on stdout
7131 
7132   Note: This is mainly intended for debugging/testing purposes.
7133 
7134   Level: developer
7135 
7136 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7137 @*/
7138 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output)
7139 {
7140   PetscMPIInt    rank,size;
7141   PetscInt       dim, c, cStart, cEnd, cMax, count = 0;
7142   cell_stats_t   stats, globalStats;
7143   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7144   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7145   DM             dmCoarse;
7146   PetscErrorCode ierr;
7147 
7148   PetscFunctionBegin;
7149   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7150   stats.min   = PETSC_MAX_REAL;
7151   stats.max   = PETSC_MIN_REAL;
7152   stats.sum   = stats.squaresum = 0.;
7153   stats.count = 0;
7154 
7155   ierr = DMGetCoordinateDim(dm,&dim);CHKERRQ(ierr);
7156   ierr = PetscMalloc2(dim * dim, &J, dim * dim, &invJ);CHKERRQ(ierr);
7157   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7158   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7159   cMax = cMax < 0 ? cEnd : cMax;
7160   for (c = cStart; c < cMax; c++) {
7161     PetscInt  i;
7162     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7163 
7164     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7165     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7166     for (i = 0; i < dim * dim; i++) {
7167       frobJ    += J[i] * J[i];
7168       frobInvJ += invJ[i] * invJ[i];
7169     }
7170     cond2 = frobJ * frobInvJ;
7171     cond  = PetscSqrtReal(cond2);
7172 
7173     stats.min        = PetscMin(stats.min,cond);
7174     stats.max        = PetscMax(stats.max,cond);
7175     stats.sum       += cond;
7176     stats.squaresum += cond2;
7177     stats.count++;
7178   }
7179 
7180   ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
7181   if (size > 1) {
7182     PetscMPIInt   blockLengths[2] = {4,1};
7183     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7184     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7185     MPI_Op        statReduce;
7186 
7187     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7188     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7189     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7190     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7191     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7192     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7193   } else {
7194     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7195   }
7196 
7197   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
7198   if (!rank) {
7199     count = globalStats.count;
7200     min   = globalStats.min;
7201     max   = globalStats.max;
7202     mean  = globalStats.sum / globalStats.count;
7203     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7204   }
7205 
7206   if (output) {
7207     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7208   }
7209   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7210 
7211   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7212   if (dmCoarse) {
7213     PetscBool isplex;
7214 
7215     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7216     if (isplex) {
7217       ierr = DMPlexCheckCellShape(dmCoarse,output);CHKERRQ(ierr);
7218     }
7219   }
7220   PetscFunctionReturn(0);
7221 }
7222 
7223 /* Pointwise interpolation
7224      Just code FEM for now
7225      u^f = I u^c
7226      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7227      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7228      I_{ij} = psi^f_i phi^c_j
7229 */
7230 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7231 {
7232   PetscSection   gsc, gsf;
7233   PetscInt       m, n;
7234   void          *ctx;
7235   DM             cdm;
7236   PetscBool      regular, ismatis;
7237   PetscErrorCode ierr;
7238 
7239   PetscFunctionBegin;
7240   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7241   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7242   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7243   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7244 
7245   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7246   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7247   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7248   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7249   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7250 
7251   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7252   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7253   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7254   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7255   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7256   if (scaling) {
7257     /* Use naive scaling */
7258     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7259   }
7260   PetscFunctionReturn(0);
7261 }
7262 
7263 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7264 {
7265   PetscErrorCode ierr;
7266   VecScatter     ctx;
7267 
7268   PetscFunctionBegin;
7269   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7270   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7271   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7272   PetscFunctionReturn(0);
7273 }
7274 
7275 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7276 {
7277   PetscSection   gsc, gsf;
7278   PetscInt       m, n;
7279   void          *ctx;
7280   DM             cdm;
7281   PetscBool      regular;
7282   PetscErrorCode ierr;
7283 
7284   PetscFunctionBegin;
7285   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7286   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7287   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7288   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7289 
7290   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7291   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7292   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7293   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7294 
7295   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7296   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7297   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7298   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7299   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7300   PetscFunctionReturn(0);
7301 }
7302 
7303 /*@
7304   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7305 
7306   Input Parameter:
7307 . dm - The DMPlex object
7308 
7309   Output Parameter:
7310 . regular - The flag
7311 
7312   Level: intermediate
7313 
7314 .seealso: DMPlexSetRegularRefinement()
7315 @*/
7316 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7317 {
7318   PetscFunctionBegin;
7319   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7320   PetscValidPointer(regular, 2);
7321   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7322   PetscFunctionReturn(0);
7323 }
7324 
7325 /*@
7326   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7327 
7328   Input Parameters:
7329 + dm - The DMPlex object
7330 - regular - The flag
7331 
7332   Level: intermediate
7333 
7334 .seealso: DMPlexGetRegularRefinement()
7335 @*/
7336 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7337 {
7338   PetscFunctionBegin;
7339   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7340   ((DM_Plex *) dm->data)->regularRefinement = regular;
7341   PetscFunctionReturn(0);
7342 }
7343 
7344 /* anchors */
7345 /*@
7346   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7347   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7348 
7349   not collective
7350 
7351   Input Parameters:
7352 . dm - The DMPlex object
7353 
7354   Output Parameters:
7355 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7356 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7357 
7358 
7359   Level: intermediate
7360 
7361 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7362 @*/
7363 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7364 {
7365   DM_Plex *plex = (DM_Plex *)dm->data;
7366   PetscErrorCode ierr;
7367 
7368   PetscFunctionBegin;
7369   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7370   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7371   if (anchorSection) *anchorSection = plex->anchorSection;
7372   if (anchorIS) *anchorIS = plex->anchorIS;
7373   PetscFunctionReturn(0);
7374 }
7375 
7376 /*@
7377   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7378   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7379   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7380 
7381   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7382   DMGetConstraints() and filling in the entries in the constraint matrix.
7383 
7384   collective on dm
7385 
7386   Input Parameters:
7387 + dm - The DMPlex object
7388 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7389 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7390 
7391   The reference counts of anchorSection and anchorIS are incremented.
7392 
7393   Level: intermediate
7394 
7395 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7396 @*/
7397 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7398 {
7399   DM_Plex        *plex = (DM_Plex *)dm->data;
7400   PetscMPIInt    result;
7401   PetscErrorCode ierr;
7402 
7403   PetscFunctionBegin;
7404   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7405   if (anchorSection) {
7406     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7407     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7408     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7409   }
7410   if (anchorIS) {
7411     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7412     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7413     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7414   }
7415 
7416   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7417   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7418   plex->anchorSection = anchorSection;
7419 
7420   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7421   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7422   plex->anchorIS = anchorIS;
7423 
7424 #if defined(PETSC_USE_DEBUG)
7425   if (anchorIS && anchorSection) {
7426     PetscInt size, a, pStart, pEnd;
7427     const PetscInt *anchors;
7428 
7429     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7430     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7431     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7432     for (a = 0; a < size; a++) {
7433       PetscInt p;
7434 
7435       p = anchors[a];
7436       if (p >= pStart && p < pEnd) {
7437         PetscInt dof;
7438 
7439         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7440         if (dof) {
7441           PetscErrorCode ierr2;
7442 
7443           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7444           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7445         }
7446       }
7447     }
7448     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7449   }
7450 #endif
7451   /* reset the generic constraints */
7452   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7453   PetscFunctionReturn(0);
7454 }
7455 
7456 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7457 {
7458   PetscSection anchorSection;
7459   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7460   PetscErrorCode ierr;
7461 
7462   PetscFunctionBegin;
7463   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7464   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7465   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7466   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7467   if (numFields) {
7468     PetscInt f;
7469     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7470 
7471     for (f = 0; f < numFields; f++) {
7472       PetscInt numComp;
7473 
7474       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7475       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7476     }
7477   }
7478   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7479   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7480   pStart = PetscMax(pStart,sStart);
7481   pEnd   = PetscMin(pEnd,sEnd);
7482   pEnd   = PetscMax(pStart,pEnd);
7483   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7484   for (p = pStart; p < pEnd; p++) {
7485     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7486     if (dof) {
7487       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7488       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7489       for (f = 0; f < numFields; f++) {
7490         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7491         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7492       }
7493     }
7494   }
7495   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7496   PetscFunctionReturn(0);
7497 }
7498 
7499 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7500 {
7501   PetscSection aSec;
7502   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7503   const PetscInt *anchors;
7504   PetscInt numFields, f;
7505   IS aIS;
7506   PetscErrorCode ierr;
7507 
7508   PetscFunctionBegin;
7509   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7510   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7511   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7512   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7513   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7514   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7515   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7516   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7517   /* cSec will be a subset of aSec and section */
7518   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7519   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7520   i[0] = 0;
7521   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7522   for (p = pStart; p < pEnd; p++) {
7523     PetscInt rDof, rOff, r;
7524 
7525     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7526     if (!rDof) continue;
7527     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7528     if (numFields) {
7529       for (f = 0; f < numFields; f++) {
7530         annz = 0;
7531         for (r = 0; r < rDof; r++) {
7532           a = anchors[rOff + r];
7533           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7534           annz += aDof;
7535         }
7536         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7537         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7538         for (q = 0; q < dof; q++) {
7539           i[off + q + 1] = i[off + q] + annz;
7540         }
7541       }
7542     }
7543     else {
7544       annz = 0;
7545       for (q = 0; q < dof; q++) {
7546         a = anchors[off + q];
7547         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7548         annz += aDof;
7549       }
7550       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7551       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7552       for (q = 0; q < dof; q++) {
7553         i[off + q + 1] = i[off + q] + annz;
7554       }
7555     }
7556   }
7557   nnz = i[m];
7558   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7559   offset = 0;
7560   for (p = pStart; p < pEnd; p++) {
7561     if (numFields) {
7562       for (f = 0; f < numFields; f++) {
7563         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7564         for (q = 0; q < dof; q++) {
7565           PetscInt rDof, rOff, r;
7566           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7567           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7568           for (r = 0; r < rDof; r++) {
7569             PetscInt s;
7570 
7571             a = anchors[rOff + r];
7572             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7573             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7574             for (s = 0; s < aDof; s++) {
7575               j[offset++] = aOff + s;
7576             }
7577           }
7578         }
7579       }
7580     }
7581     else {
7582       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7583       for (q = 0; q < dof; q++) {
7584         PetscInt rDof, rOff, r;
7585         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7586         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7587         for (r = 0; r < rDof; r++) {
7588           PetscInt s;
7589 
7590           a = anchors[rOff + r];
7591           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7592           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7593           for (s = 0; s < aDof; s++) {
7594             j[offset++] = aOff + s;
7595           }
7596         }
7597       }
7598     }
7599   }
7600   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7601   ierr = PetscFree(i);CHKERRQ(ierr);
7602   ierr = PetscFree(j);CHKERRQ(ierr);
7603   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7604   PetscFunctionReturn(0);
7605 }
7606 
7607 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7608 {
7609   DM_Plex        *plex = (DM_Plex *)dm->data;
7610   PetscSection   anchorSection, section, cSec;
7611   Mat            cMat;
7612   PetscErrorCode ierr;
7613 
7614   PetscFunctionBegin;
7615   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7616   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7617   if (anchorSection) {
7618     PetscInt Nf;
7619 
7620     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
7621     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7622     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7623     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7624     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7625     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7626     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7627     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7628   }
7629   PetscFunctionReturn(0);
7630 }
7631 
7632 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7633 {
7634   IS             subis;
7635   PetscSection   section, subsection;
7636   PetscErrorCode ierr;
7637 
7638   PetscFunctionBegin;
7639   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
7640   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7641   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7642   /* Create subdomain */
7643   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7644   /* Create submodel */
7645   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7646   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7647   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7648   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
7649   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7650   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7651   /* Create map from submodel to global model */
7652   if (is) {
7653     PetscSection    sectionGlobal, subsectionGlobal;
7654     IS              spIS;
7655     const PetscInt *spmap;
7656     PetscInt       *subIndices;
7657     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7658     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7659 
7660     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7661     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7662     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7663     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7664     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7665     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7666     for (p = pStart; p < pEnd; ++p) {
7667       PetscInt gdof, pSubSize  = 0;
7668 
7669       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7670       if (gdof > 0) {
7671         for (f = 0; f < Nf; ++f) {
7672           PetscInt fdof, fcdof;
7673 
7674           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7675           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7676           pSubSize += fdof-fcdof;
7677         }
7678         subSize += pSubSize;
7679         if (pSubSize) {
7680           if (bs < 0) {
7681             bs = pSubSize;
7682           } else if (bs != pSubSize) {
7683             /* Layout does not admit a pointwise block size */
7684             bs = 1;
7685           }
7686         }
7687       }
7688     }
7689     /* Must have same blocksize on all procs (some might have no points) */
7690     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7691     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7692     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7693     else                            {bs = bsMinMax[0];}
7694     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7695     for (p = pStart; p < pEnd; ++p) {
7696       PetscInt gdof, goff;
7697 
7698       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7699       if (gdof > 0) {
7700         const PetscInt point = spmap[p];
7701 
7702         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7703         for (f = 0; f < Nf; ++f) {
7704           PetscInt fdof, fcdof, fc, f2, poff = 0;
7705 
7706           /* Can get rid of this loop by storing field information in the global section */
7707           for (f2 = 0; f2 < f; ++f2) {
7708             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7709             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7710             poff += fdof-fcdof;
7711           }
7712           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7713           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7714           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7715             subIndices[subOff] = goff+poff+fc;
7716           }
7717         }
7718       }
7719     }
7720     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7721     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7722     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7723     if (bs > 1) {
7724       /* We need to check that the block size does not come from non-contiguous fields */
7725       PetscInt i, j, set = 1;
7726       for (i = 0; i < subSize; i += bs) {
7727         for (j = 0; j < bs; ++j) {
7728           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7729         }
7730       }
7731       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7732     }
7733     /* Attach nullspace */
7734     for (f = 0; f < Nf; ++f) {
7735       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7736       if ((*subdm)->nullspaceConstructors[f]) break;
7737     }
7738     if (f < Nf) {
7739       MatNullSpace nullSpace;
7740 
7741       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7742       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7743       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7744     }
7745   }
7746   PetscFunctionReturn(0);
7747 }
7748