xref: /petsc/src/dm/impls/plex/plex.c (revision 7e4e2a2d5b77d05327773006136c5762506227f3)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     char         lname[PETSC_MAX_PATH_LEN];
675     PetscReal    scale         = 2.0;
676     PetscReal    tikzscale     = 1.0;
677     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
678     double       tcoords[3];
679     PetscScalar *coords;
680     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
681     PetscMPIInt  rank, size;
682     char         **names, **colors, **lcolors;
683     PetscBool    plotEdges, flg, lflg;
684     PetscBT      wp = NULL;
685     PetscInt     pEnd, pStart;
686 
687     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
688     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
689     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
690     numLabels  = PetscMax(numLabels, 10);
691     numColors  = 10;
692     numLColors = 10;
693     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
694     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
695     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
696     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
697     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
698     if (!useLabels) numLabels = 0;
699     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
700     if (!useColors) {
701       numColors = 3;
702       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
703     }
704     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
705     if (!useColors) {
706       numLColors = 4;
707       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
708     }
709     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
710     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
711     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
712     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
713     if (depth < dim) plotEdges = PETSC_FALSE;
714 
715     /* filter points with labelvalue != labeldefaultvalue */
716     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
717     if (lflg) {
718       DMLabel lbl;
719 
720       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
721       if (lbl) {
722         PetscInt val, defval;
723 
724         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
725         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
726         for (c = pStart;  c < pEnd; c++) {
727           PetscInt *closure = NULL;
728           PetscInt  closureSize;
729 
730           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
731           if (val == defval) continue;
732 
733           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
734           for (p = 0; p < closureSize*2; p += 2) {
735             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
736           }
737           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
738         }
739       }
740     }
741 
742     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
743     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
744     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
745     ierr = PetscViewerASCIIPrintf(viewer, "\
746 \\documentclass[tikz]{standalone}\n\n\
747 \\usepackage{pgflibraryshapes}\n\
748 \\usetikzlibrary{backgrounds}\n\
749 \\usetikzlibrary{arrows}\n\
750 \\begin{document}\n");CHKERRQ(ierr);
751     if (size > 1) {
752       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
753       for (p = 0; p < size; ++p) {
754         if (p > 0 && p == size-1) {
755           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
756         } else if (p > 0) {
757           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
758         }
759         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
760       }
761       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
762     }
763     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
764 
765     /* Plot vertices */
766     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
767     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
768     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
769     for (v = vStart; v < vEnd; ++v) {
770       PetscInt  off, dof, d;
771       PetscBool isLabeled = PETSC_FALSE;
772 
773       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
774       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
775       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
776       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
777       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
778       for (d = 0; d < dof; ++d) {
779         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
780         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
781       }
782       /* Rotate coordinates since PGF makes z point out of the page instead of up */
783       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
784       for (d = 0; d < dof; ++d) {
785         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
786         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
787       }
788       color = colors[rank%numColors];
789       for (l = 0; l < numLabels; ++l) {
790         PetscInt val;
791         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
792         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
793       }
794       if (useNumbers) {
795         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
796       } else {
797         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
798       }
799     }
800     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
801     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
802     /* Plot cells */
803     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
804     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
805     if (dim == 3 || !useNumbers) {
806       for (e = eStart; e < eEnd; ++e) {
807         const PetscInt *cone;
808 
809         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
810         color = colors[rank%numColors];
811         for (l = 0; l < numLabels; ++l) {
812           PetscInt val;
813           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
814           if (val >= 0) {color = lcolors[l%numLColors]; break;}
815         }
816         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
817         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
818       }
819     } else {
820       for (c = cStart; c < cEnd; ++c) {
821         PetscInt *closure = NULL;
822         PetscInt  closureSize, firstPoint = -1;
823 
824         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
825         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
826         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
827         for (p = 0; p < closureSize*2; p += 2) {
828           const PetscInt point = closure[p];
829 
830           if ((point < vStart) || (point >= vEnd)) continue;
831           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
832           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
833           if (firstPoint < 0) firstPoint = point;
834         }
835         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
836         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
837         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
838       }
839     }
840     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
841     for (c = cStart; c < cEnd; ++c) {
842       double    ccoords[3] = {0.0, 0.0, 0.0};
843       PetscBool isLabeled  = PETSC_FALSE;
844       PetscInt *closure    = NULL;
845       PetscInt  closureSize, dof, d, n = 0;
846 
847       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
848       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
849       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
850       for (p = 0; p < closureSize*2; p += 2) {
851         const PetscInt point = closure[p];
852         PetscInt       off;
853 
854         if ((point < vStart) || (point >= vEnd)) continue;
855         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
856         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
857         for (d = 0; d < dof; ++d) {
858           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
859           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
860         }
861         /* Rotate coordinates since PGF makes z point out of the page instead of up */
862         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
863         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
864         ++n;
865       }
866       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
867       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
868       for (d = 0; d < dof; ++d) {
869         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
870         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
871       }
872       color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (useNumbers) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
880       } else {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       }
883     }
884     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
885     /* Plot edges */
886     if (plotEdges) {
887       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
888       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
889       for (e = eStart; e < eEnd; ++e) {
890         const PetscInt *cone;
891         PetscInt        coneSize, offA, offB, dof, d;
892 
893         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
894         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
895         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
896         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
897         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
898         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
899         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
900         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
901         for (d = 0; d < dof; ++d) {
902           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
903           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
904         }
905         /* Rotate coordinates since PGF makes z point out of the page instead of up */
906         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
907         for (d = 0; d < dof; ++d) {
908           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
909           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
910         }
911         color = colors[rank%numColors];
912         for (l = 0; l < numLabels; ++l) {
913           PetscInt val;
914           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
915           if (val >= 0) {color = lcolors[l%numLColors]; break;}
916         }
917         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
918       }
919       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
920       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
921       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
922     }
923     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
924     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
925     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
926     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
927     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
928     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
929     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
930     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
931     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
932   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
933     Vec                    cown,acown;
934     VecScatter             sct;
935     ISLocalToGlobalMapping g2l;
936     IS                     gid,acis;
937     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
938     MPI_Group              ggroup,ngroup;
939     PetscScalar            *array,nid;
940     const PetscInt         *idxs;
941     PetscInt               *idxs2,*start,*adjacency,*work;
942     PetscInt64             lm[3],gm[3];
943     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
944     PetscMPIInt            d1,d2,rank;
945 
946     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
947     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
948 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
949     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
950 #endif
951     if (ncomm != MPI_COMM_NULL) {
952       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
953       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
954       d1   = 0;
955       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
956       nid  = d2;
957       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
958       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
959       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
960     } else nid = 0.0;
961 
962     /* Get connectivity */
963     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
964     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
965 
966     /* filter overlapped local cells */
967     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
968     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
969     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
970     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
971     for (c = cStart, cum = 0; c < cEnd; c++) {
972       if (idxs[c-cStart] < 0) continue;
973       idxs2[cum++] = idxs[c-cStart];
974     }
975     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
976     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
977     ierr = ISDestroy(&gid);CHKERRQ(ierr);
978     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
979 
980     /* support for node-aware cell locality */
981     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
982     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
983     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
984     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
985     for (c = 0; c < numVertices; c++) array[c] = nid;
986     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
987     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
988     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
989     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
990     ierr = ISDestroy(&acis);CHKERRQ(ierr);
991     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
992     ierr = VecDestroy(&cown);CHKERRQ(ierr);
993 
994     /* compute edgeCut */
995     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
996     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
997     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
998     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
999     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1000     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1001     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1002       PetscInt totl;
1003 
1004       totl = start[c+1]-start[c];
1005       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1006       for (i = 0; i < totl; i++) {
1007         if (work[i] < 0) {
1008           ect  += 1;
1009           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1010         }
1011       }
1012     }
1013     ierr  = PetscFree(work);CHKERRQ(ierr);
1014     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1015     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1016     lm[1] = -numVertices;
1017     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1018     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1019     lm[0] = ect; /* edgeCut */
1020     lm[1] = ectn; /* node-aware edgeCut */
1021     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1022     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1023     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1024 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1025     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1026 #else
1027     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1028 #endif
1029     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1030     ierr  = PetscFree(start);CHKERRQ(ierr);
1031     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1032     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1033   } else {
1034     MPI_Comm    comm;
1035     PetscInt   *sizes, *hybsizes;
1036     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
1037     PetscInt    pStart, pEnd, p;
1038     PetscInt    numLabels, l;
1039     const char *name;
1040     PetscMPIInt size;
1041 
1042     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1043     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1044     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1045     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1046     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1047     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1048     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1049     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1050     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1051     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1052     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1053     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1054     if (depth == 1) {
1055       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1056       pEnd = pEnd - pStart;
1057       pMax[0] -= pStart;
1058       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1059       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1060       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1061       for (p = 0; p < size; ++p) {
1062         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1063         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1064       }
1065       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1066       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1067       pEnd = pEnd - pStart;
1068       pMax[depth] -= pStart;
1069       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1070       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1071       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1072       for (p = 0; p < size; ++p) {
1073         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1074         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1075       }
1076       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1077     } else {
1078       PetscMPIInt rank;
1079       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1080       for (d = 0; d <= dim; d++) {
1081         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1082         pEnd    -= pStart;
1083         pMax[d] -= pStart;
1084         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1085         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1086         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1087         for (p = 0; p < size; ++p) {
1088           if (!rank) {
1089             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1090             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1091           }
1092         }
1093         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1094       }
1095     }
1096     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1097     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1098     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1099     for (l = 0; l < numLabels; ++l) {
1100       DMLabel         label;
1101       const char     *name;
1102       IS              valueIS;
1103       const PetscInt *values;
1104       PetscInt        numValues, v;
1105 
1106       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1107       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1108       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1109       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1110       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1111       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1112       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1113       for (v = 0; v < numValues; ++v) {
1114         PetscInt size;
1115 
1116         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1117         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1118         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1119       }
1120       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1122       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1123       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1124     }
1125     /* If no fields are specified, people do not want to see adjacency */
1126     if (dm->Nf) {
1127       PetscInt f;
1128 
1129       for (f = 0; f < dm->Nf; ++f) {
1130         const char *name;
1131 
1132         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1133         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1134         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1135         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1136         if (dm->fields[f].adjacency[0]) {
1137           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1138           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1139         } else {
1140           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1141           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1142         }
1143         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1144       }
1145     }
1146     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1147     if (cdm) {
1148       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1149       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1150       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1151     }
1152   }
1153   PetscFunctionReturn(0);
1154 }
1155 
1156 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1157 {
1158   PetscDraw          draw;
1159   DM                 cdm;
1160   PetscSection       coordSection;
1161   Vec                coordinates;
1162   const PetscScalar *coords;
1163   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1164   PetscBool          isnull;
1165   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1166   PetscMPIInt        rank;
1167   PetscErrorCode     ierr;
1168 
1169   PetscFunctionBegin;
1170   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1171   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1172   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1173   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1174   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1175   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1176   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1177 
1178   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1179   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1180   if (isnull) PetscFunctionReturn(0);
1181   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1182 
1183   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1184   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1185   for (c = 0; c < N; c += dim) {
1186     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1187     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1188   }
1189   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1190   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1191   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1192   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1193   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1194 
1195   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1196   for (c = cStart; c < cEnd; ++c) {
1197     PetscScalar *coords = NULL;
1198     PetscInt     numCoords,coneSize;
1199 
1200     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1201     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202     switch (coneSize) {
1203     case 3:
1204       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1205                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1206                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1207                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1208       break;
1209     case 4:
1210       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1211                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1212                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1213                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1214                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1215       break;
1216     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1217     }
1218     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1219   }
1220   for (c = cStart; c < cEnd; ++c) {
1221     PetscScalar *coords = NULL;
1222     PetscInt     numCoords,coneSize;
1223 
1224     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1225     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1226     switch (coneSize) {
1227     case 3:
1228       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1229       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1230       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1231       break;
1232     case 4:
1233       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1234       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1235       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1236       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237       break;
1238     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1239     }
1240     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1241   }
1242   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1243   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1244   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1249 {
1250   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1251   char           name[PETSC_MAX_PATH_LEN];
1252   PetscErrorCode ierr;
1253 
1254   PetscFunctionBegin;
1255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1256   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1257   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1258   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1259   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1260   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1261   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1262   if (iascii) {
1263     PetscViewerFormat format;
1264     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1265     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1266       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1267     } else {
1268       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1269     }
1270   } else if (ishdf5) {
1271 #if defined(PETSC_HAVE_HDF5)
1272     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1273 #else
1274     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1275 #endif
1276   } else if (isvtk) {
1277     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1278   } else if (isdraw) {
1279     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1280   } else if (isglvis) {
1281     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1282   } else {
1283     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1284   }
1285   /* Optionally view the partition */
1286   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1287   if (flg) {
1288     Vec ranks;
1289     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1290     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1291     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1292   }
1293   /* Optionally view a label */
1294   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1295   if (flg) {
1296     DMLabel label;
1297     Vec     val;
1298 
1299     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1300     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1301     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1302     ierr = VecView(val, viewer);CHKERRQ(ierr);
1303     ierr = VecDestroy(&val);CHKERRQ(ierr);
1304   }
1305   PetscFunctionReturn(0);
1306 }
1307 
1308 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1309 {
1310   PetscBool      ishdf5;
1311   PetscErrorCode ierr;
1312 
1313   PetscFunctionBegin;
1314   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1315   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1316   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1317   if (ishdf5) {
1318 #if defined(PETSC_HAVE_HDF5)
1319     PetscViewerFormat format;
1320     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1321     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1322       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1323     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1324       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1325     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1326 #else
1327     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1328 #endif
1329   } else {
1330     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1331   }
1332   PetscFunctionReturn(0);
1333 }
1334 
1335 PetscErrorCode DMDestroy_Plex(DM dm)
1336 {
1337   DM_Plex       *mesh = (DM_Plex*) dm->data;
1338   PetscErrorCode ierr;
1339 
1340   PetscFunctionBegin;
1341   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1342   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1343   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
1344   if (--mesh->refct > 0) PetscFunctionReturn(0);
1345   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1346   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1347   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1348   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1349   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1350   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1351   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1352   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1353   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1354   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1355   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1356   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1357   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1358   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1359   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1360   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1361   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1362   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1363   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1364   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1365   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1366   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1367   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1368   ierr = PetscFree(mesh);CHKERRQ(ierr);
1369   PetscFunctionReturn(0);
1370 }
1371 
1372 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1373 {
1374   PetscSection           sectionGlobal;
1375   PetscInt               bs = -1, mbs;
1376   PetscInt               localSize;
1377   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1378   PetscErrorCode         ierr;
1379   MatType                mtype;
1380   ISLocalToGlobalMapping ltog;
1381 
1382   PetscFunctionBegin;
1383   ierr = MatInitializePackage();CHKERRQ(ierr);
1384   mtype = dm->mattype;
1385   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1386   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1387   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1388   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1389   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1390   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1391   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1392   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1393   if (mbs > 1) bs = mbs;
1394   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1395   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1396   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1397   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1398   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1399   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1400   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1401   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1402   if (!isShell) {
1403     PetscSection subSection;
1404     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1405     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1406     PetscInt     pStart, pEnd, p, dof, cdof;
1407 
1408     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1409     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1410       PetscSection section;
1411       PetscInt     size;
1412 
1413       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1414       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1415       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1416       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1417     } else {
1418       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1419     }
1420     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1421     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1422       PetscInt bdof;
1423 
1424       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1425       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1426       dof  = dof < 0 ? -(dof+1) : dof;
1427       bdof = cdof && (dof-cdof) ? 1 : dof;
1428       if (dof) {
1429         if (bs < 0)          {bs = bdof;}
1430         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1431       }
1432       if (isMatIS) {
1433         PetscInt loff,c,off;
1434         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1435         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1436         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1437       }
1438     }
1439     /* Must have same blocksize on all procs (some might have no points) */
1440     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1441     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1442     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1443     else                            {bs = bsMinMax[0];}
1444     bs = PetscMax(1,bs);
1445     if (isMatIS) { /* Must reduce indices by blocksize */
1446       PetscInt l;
1447 
1448       lsize = lsize/bs;
1449       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1450       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1451     }
1452     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1453     if (isMatIS) {
1454       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1455     }
1456     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1457     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1458     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1459   }
1460   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1461   PetscFunctionReturn(0);
1462 }
1463 
1464 /*@
1465   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1466 
1467   Not collective
1468 
1469   Input Parameter:
1470 . mesh - The DMPlex
1471 
1472   Output Parameters:
1473 . subsection - The subdomain section
1474 
1475   Level: developer
1476 
1477 .seealso:
1478 @*/
1479 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1480 {
1481   DM_Plex       *mesh = (DM_Plex*) dm->data;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   if (!mesh->subdomainSection) {
1487     PetscSection section;
1488     PetscSF      sf;
1489 
1490     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1491     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1492     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1493     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1494   }
1495   *subsection = mesh->subdomainSection;
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 /*@
1500   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1501 
1502   Not collective
1503 
1504   Input Parameter:
1505 . mesh - The DMPlex
1506 
1507   Output Parameters:
1508 + pStart - The first mesh point
1509 - pEnd   - The upper bound for mesh points
1510 
1511   Level: beginner
1512 
1513 .seealso: DMPlexCreate(), DMPlexSetChart()
1514 @*/
1515 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1516 {
1517   DM_Plex       *mesh = (DM_Plex*) dm->data;
1518   PetscErrorCode ierr;
1519 
1520   PetscFunctionBegin;
1521   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1522   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1523   PetscFunctionReturn(0);
1524 }
1525 
1526 /*@
1527   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1528 
1529   Not collective
1530 
1531   Input Parameters:
1532 + mesh - The DMPlex
1533 . pStart - The first mesh point
1534 - pEnd   - The upper bound for mesh points
1535 
1536   Output Parameters:
1537 
1538   Level: beginner
1539 
1540 .seealso: DMPlexCreate(), DMPlexGetChart()
1541 @*/
1542 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1543 {
1544   DM_Plex       *mesh = (DM_Plex*) dm->data;
1545   PetscErrorCode ierr;
1546 
1547   PetscFunctionBegin;
1548   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1549   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1550   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1551   PetscFunctionReturn(0);
1552 }
1553 
1554 /*@
1555   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1556 
1557   Not collective
1558 
1559   Input Parameters:
1560 + mesh - The DMPlex
1561 - p - The point, which must lie in the chart set with DMPlexSetChart()
1562 
1563   Output Parameter:
1564 . size - The cone size for point p
1565 
1566   Level: beginner
1567 
1568 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1569 @*/
1570 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1571 {
1572   DM_Plex       *mesh = (DM_Plex*) dm->data;
1573   PetscErrorCode ierr;
1574 
1575   PetscFunctionBegin;
1576   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1577   PetscValidPointer(size, 3);
1578   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1579   PetscFunctionReturn(0);
1580 }
1581 
1582 /*@
1583   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1584 
1585   Not collective
1586 
1587   Input Parameters:
1588 + mesh - The DMPlex
1589 . p - The point, which must lie in the chart set with DMPlexSetChart()
1590 - size - The cone size for point p
1591 
1592   Output Parameter:
1593 
1594   Note:
1595   This should be called after DMPlexSetChart().
1596 
1597   Level: beginner
1598 
1599 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1600 @*/
1601 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1602 {
1603   DM_Plex       *mesh = (DM_Plex*) dm->data;
1604   PetscErrorCode ierr;
1605 
1606   PetscFunctionBegin;
1607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1608   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1609 
1610   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1611   PetscFunctionReturn(0);
1612 }
1613 
1614 /*@
1615   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1616 
1617   Not collective
1618 
1619   Input Parameters:
1620 + mesh - The DMPlex
1621 . p - The point, which must lie in the chart set with DMPlexSetChart()
1622 - size - The additional cone size for point p
1623 
1624   Output Parameter:
1625 
1626   Note:
1627   This should be called after DMPlexSetChart().
1628 
1629   Level: beginner
1630 
1631 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1632 @*/
1633 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1634 {
1635   DM_Plex       *mesh = (DM_Plex*) dm->data;
1636   PetscInt       csize;
1637   PetscErrorCode ierr;
1638 
1639   PetscFunctionBegin;
1640   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1641   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1642   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1643 
1644   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1645   PetscFunctionReturn(0);
1646 }
1647 
1648 /*@C
1649   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1650 
1651   Not collective
1652 
1653   Input Parameters:
1654 + dm - The DMPlex
1655 - p - The point, which must lie in the chart set with DMPlexSetChart()
1656 
1657   Output Parameter:
1658 . cone - An array of points which are on the in-edges for point p
1659 
1660   Level: beginner
1661 
1662   Fortran Notes:
1663   Since it returns an array, this routine is only available in Fortran 90, and you must
1664   include petsc.h90 in your code.
1665   You must also call DMPlexRestoreCone() after you finish using the returned array.
1666   DMPlexRestoreCone() is not needed/available in C.
1667 
1668 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1669 @*/
1670 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1671 {
1672   DM_Plex       *mesh = (DM_Plex*) dm->data;
1673   PetscInt       off;
1674   PetscErrorCode ierr;
1675 
1676   PetscFunctionBegin;
1677   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1678   PetscValidPointer(cone, 3);
1679   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1680   *cone = &mesh->cones[off];
1681   PetscFunctionReturn(0);
1682 }
1683 
1684 /*@C
1685   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1686 
1687   Not collective
1688 
1689   Input Parameters:
1690 + dm - The DMPlex
1691 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1692 
1693   Output Parameter:
1694 + pConesSection - PetscSection describing the layout of pCones
1695 - pCones - An array of points which are on the in-edges for the point set p
1696 
1697   Level: intermediate
1698 
1699 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1700 @*/
1701 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1702 {
1703   PetscSection        cs, newcs;
1704   PetscInt            *cones;
1705   PetscInt            *newarr=NULL;
1706   PetscInt            n;
1707   PetscErrorCode      ierr;
1708 
1709   PetscFunctionBegin;
1710   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1711   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1712   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1713   if (pConesSection) *pConesSection = newcs;
1714   if (pCones) {
1715     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1716     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1717   }
1718   PetscFunctionReturn(0);
1719 }
1720 
1721 /*@
1722   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1723 
1724   Not collective
1725 
1726   Input Parameters:
1727 + dm - The DMPlex
1728 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1729 
1730   Output Parameter:
1731 . expandedPoints - An array of vertices recursively expanded from input points
1732 
1733   Level: advanced
1734 
1735   Notes:
1736   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1737   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1738 
1739 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1740 @*/
1741 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1742 {
1743   IS                  *expandedPointsAll;
1744   PetscInt            depth;
1745   PetscErrorCode      ierr;
1746 
1747   PetscFunctionBegin;
1748   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1749   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1750   PetscValidPointer(expandedPoints, 3);
1751   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1752   *expandedPoints = expandedPointsAll[0];
1753   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1754   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1755   PetscFunctionReturn(0);
1756 }
1757 
1758 /*@
1759   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1760 
1761   Not collective
1762 
1763   Input Parameters:
1764 + dm - The DMPlex
1765 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1766 
1767   Output Parameter:
1768 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1769 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1770 - sections - (optional) An array of sections which describe mappings from points to their cone points
1771 
1772   Level: advanced
1773 
1774   Notes:
1775   Like DMPlexGetConeTuple() but recursive.
1776 
1777   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1778   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1779 
1780   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1781   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1782   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1783 
1784 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1785 @*/
1786 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1787 {
1788   const PetscInt      *arr0=NULL, *cone=NULL;
1789   PetscInt            *arr=NULL, *newarr=NULL;
1790   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1791   IS                  *expandedPoints_;
1792   PetscSection        *sections_;
1793   PetscErrorCode      ierr;
1794 
1795   PetscFunctionBegin;
1796   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1797   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1798   if (depth) PetscValidIntPointer(depth, 3);
1799   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1800   if (sections) PetscValidPointer(sections, 5);
1801   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1802   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1803   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1804   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1805   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1806   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1807   for (d=depth_-1; d>=0; d--) {
1808     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1809     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1810     for (i=0; i<n; i++) {
1811       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1812       if (arr[i] >= start && arr[i] < end) {
1813         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1814         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1815       } else {
1816         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1817       }
1818     }
1819     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1820     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1821     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1822     for (i=0; i<n; i++) {
1823       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1824       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1825       if (cn > 1) {
1826         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1827         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1828       } else {
1829         newarr[co] = arr[i];
1830       }
1831     }
1832     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1833     arr = newarr;
1834     n = newn;
1835   }
1836   *depth = depth_;
1837   if (expandedPoints) *expandedPoints = expandedPoints_;
1838   else {
1839     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1840     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1841   }
1842   if (sections) *sections = sections_;
1843   else {
1844     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1845     ierr = PetscFree(sections_);CHKERRQ(ierr);
1846   }
1847   PetscFunctionReturn(0);
1848 }
1849 
1850 /*@
1851   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1852 
1853   Not collective
1854 
1855   Input Parameters:
1856 + dm - The DMPlex
1857 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1858 
1859   Output Parameter:
1860 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1861 . expandedPoints - (optional) An array of recursively expanded cones
1862 - sections - (optional) An array of sections which describe mappings from points to their cone points
1863 
1864   Level: advanced
1865 
1866   Notes:
1867   See DMPlexGetConeRecursive() for details.
1868 
1869 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1870 @*/
1871 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1872 {
1873   PetscInt            d, depth_;
1874   PetscErrorCode      ierr;
1875 
1876   PetscFunctionBegin;
1877   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1878   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1879   if (depth) *depth = 0;
1880   if (expandedPoints) {
1881     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1882     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1883   }
1884   if (sections)  {
1885     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1886     ierr = PetscFree(*sections);CHKERRQ(ierr);
1887   }
1888   PetscFunctionReturn(0);
1889 }
1890 
1891 /*@
1892   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1893 
1894   Not collective
1895 
1896   Input Parameters:
1897 + mesh - The DMPlex
1898 . p - The point, which must lie in the chart set with DMPlexSetChart()
1899 - cone - An array of points which are on the in-edges for point p
1900 
1901   Output Parameter:
1902 
1903   Note:
1904   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1905 
1906   Developer Note: Why not call this DMPlexSetCover()
1907 
1908   Level: beginner
1909 
1910 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1911 @*/
1912 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1913 {
1914   DM_Plex       *mesh = (DM_Plex*) dm->data;
1915   PetscInt       pStart, pEnd;
1916   PetscInt       dof, off, c;
1917   PetscErrorCode ierr;
1918 
1919   PetscFunctionBegin;
1920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1921   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1922   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1923   if (dof) PetscValidPointer(cone, 3);
1924   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1925   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1926   for (c = 0; c < dof; ++c) {
1927     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1928     mesh->cones[off+c] = cone[c];
1929   }
1930   PetscFunctionReturn(0);
1931 }
1932 
1933 /*@C
1934   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1935 
1936   Not collective
1937 
1938   Input Parameters:
1939 + mesh - The DMPlex
1940 - p - The point, which must lie in the chart set with DMPlexSetChart()
1941 
1942   Output Parameter:
1943 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1944                     integer giving the prescription for cone traversal. If it is negative, the cone is
1945                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1946                     the index of the cone point on which to start.
1947 
1948   Level: beginner
1949 
1950   Fortran Notes:
1951   Since it returns an array, this routine is only available in Fortran 90, and you must
1952   include petsc.h90 in your code.
1953   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1954   DMPlexRestoreConeOrientation() is not needed/available in C.
1955 
1956 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1957 @*/
1958 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1959 {
1960   DM_Plex       *mesh = (DM_Plex*) dm->data;
1961   PetscInt       off;
1962   PetscErrorCode ierr;
1963 
1964   PetscFunctionBegin;
1965   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1966 #if defined(PETSC_USE_DEBUG)
1967   {
1968     PetscInt dof;
1969     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1970     if (dof) PetscValidPointer(coneOrientation, 3);
1971   }
1972 #endif
1973   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1974 
1975   *coneOrientation = &mesh->coneOrientations[off];
1976   PetscFunctionReturn(0);
1977 }
1978 
1979 /*@
1980   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1981 
1982   Not collective
1983 
1984   Input Parameters:
1985 + mesh - The DMPlex
1986 . p - The point, which must lie in the chart set with DMPlexSetChart()
1987 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1988                     integer giving the prescription for cone traversal. If it is negative, the cone is
1989                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1990                     the index of the cone point on which to start.
1991 
1992   Output Parameter:
1993 
1994   Note:
1995   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1996 
1997   Level: beginner
1998 
1999 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2000 @*/
2001 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2002 {
2003   DM_Plex       *mesh = (DM_Plex*) dm->data;
2004   PetscInt       pStart, pEnd;
2005   PetscInt       dof, off, c;
2006   PetscErrorCode ierr;
2007 
2008   PetscFunctionBegin;
2009   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2010   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2011   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2012   if (dof) PetscValidPointer(coneOrientation, 3);
2013   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2014   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2015   for (c = 0; c < dof; ++c) {
2016     PetscInt cdof, o = coneOrientation[c];
2017 
2018     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2019     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2020     mesh->coneOrientations[off+c] = o;
2021   }
2022   PetscFunctionReturn(0);
2023 }
2024 
2025 /*@
2026   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2027 
2028   Not collective
2029 
2030   Input Parameters:
2031 + mesh - The DMPlex
2032 . p - The point, which must lie in the chart set with DMPlexSetChart()
2033 . conePos - The local index in the cone where the point should be put
2034 - conePoint - The mesh point to insert
2035 
2036   Level: beginner
2037 
2038 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2039 @*/
2040 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2041 {
2042   DM_Plex       *mesh = (DM_Plex*) dm->data;
2043   PetscInt       pStart, pEnd;
2044   PetscInt       dof, off;
2045   PetscErrorCode ierr;
2046 
2047   PetscFunctionBegin;
2048   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2049   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2050   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2051   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2052   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2053   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2054   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2055   mesh->cones[off+conePos] = conePoint;
2056   PetscFunctionReturn(0);
2057 }
2058 
2059 /*@
2060   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2061 
2062   Not collective
2063 
2064   Input Parameters:
2065 + mesh - The DMPlex
2066 . p - The point, which must lie in the chart set with DMPlexSetChart()
2067 . conePos - The local index in the cone where the point should be put
2068 - coneOrientation - The point orientation to insert
2069 
2070   Level: beginner
2071 
2072 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2073 @*/
2074 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2075 {
2076   DM_Plex       *mesh = (DM_Plex*) dm->data;
2077   PetscInt       pStart, pEnd;
2078   PetscInt       dof, off;
2079   PetscErrorCode ierr;
2080 
2081   PetscFunctionBegin;
2082   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2083   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2084   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2085   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2086   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2087   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2088   mesh->coneOrientations[off+conePos] = coneOrientation;
2089   PetscFunctionReturn(0);
2090 }
2091 
2092 /*@
2093   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2094 
2095   Not collective
2096 
2097   Input Parameters:
2098 + mesh - The DMPlex
2099 - p - The point, which must lie in the chart set with DMPlexSetChart()
2100 
2101   Output Parameter:
2102 . size - The support size for point p
2103 
2104   Level: beginner
2105 
2106 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2107 @*/
2108 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2109 {
2110   DM_Plex       *mesh = (DM_Plex*) dm->data;
2111   PetscErrorCode ierr;
2112 
2113   PetscFunctionBegin;
2114   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2115   PetscValidPointer(size, 3);
2116   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2117   PetscFunctionReturn(0);
2118 }
2119 
2120 /*@
2121   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2122 
2123   Not collective
2124 
2125   Input Parameters:
2126 + mesh - The DMPlex
2127 . p - The point, which must lie in the chart set with DMPlexSetChart()
2128 - size - The support size for point p
2129 
2130   Output Parameter:
2131 
2132   Note:
2133   This should be called after DMPlexSetChart().
2134 
2135   Level: beginner
2136 
2137 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2138 @*/
2139 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2140 {
2141   DM_Plex       *mesh = (DM_Plex*) dm->data;
2142   PetscErrorCode ierr;
2143 
2144   PetscFunctionBegin;
2145   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2146   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2147 
2148   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2149   PetscFunctionReturn(0);
2150 }
2151 
2152 /*@C
2153   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2154 
2155   Not collective
2156 
2157   Input Parameters:
2158 + mesh - The DMPlex
2159 - p - The point, which must lie in the chart set with DMPlexSetChart()
2160 
2161   Output Parameter:
2162 . support - An array of points which are on the out-edges for point p
2163 
2164   Level: beginner
2165 
2166   Fortran Notes:
2167   Since it returns an array, this routine is only available in Fortran 90, and you must
2168   include petsc.h90 in your code.
2169   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2170   DMPlexRestoreSupport() is not needed/available in C.
2171 
2172 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2173 @*/
2174 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2175 {
2176   DM_Plex       *mesh = (DM_Plex*) dm->data;
2177   PetscInt       off;
2178   PetscErrorCode ierr;
2179 
2180   PetscFunctionBegin;
2181   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2182   PetscValidPointer(support, 3);
2183   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2184   *support = &mesh->supports[off];
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 /*@
2189   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2190 
2191   Not collective
2192 
2193   Input Parameters:
2194 + mesh - The DMPlex
2195 . p - The point, which must lie in the chart set with DMPlexSetChart()
2196 - support - An array of points which are on the out-edges for point p
2197 
2198   Output Parameter:
2199 
2200   Note:
2201   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2202 
2203   Level: beginner
2204 
2205 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2206 @*/
2207 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2208 {
2209   DM_Plex       *mesh = (DM_Plex*) dm->data;
2210   PetscInt       pStart, pEnd;
2211   PetscInt       dof, off, c;
2212   PetscErrorCode ierr;
2213 
2214   PetscFunctionBegin;
2215   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2216   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2217   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2218   if (dof) PetscValidPointer(support, 3);
2219   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2220   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2221   for (c = 0; c < dof; ++c) {
2222     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2223     mesh->supports[off+c] = support[c];
2224   }
2225   PetscFunctionReturn(0);
2226 }
2227 
2228 /*@
2229   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2230 
2231   Not collective
2232 
2233   Input Parameters:
2234 + mesh - The DMPlex
2235 . p - The point, which must lie in the chart set with DMPlexSetChart()
2236 . supportPos - The local index in the cone where the point should be put
2237 - supportPoint - The mesh point to insert
2238 
2239   Level: beginner
2240 
2241 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2242 @*/
2243 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2244 {
2245   DM_Plex       *mesh = (DM_Plex*) dm->data;
2246   PetscInt       pStart, pEnd;
2247   PetscInt       dof, off;
2248   PetscErrorCode ierr;
2249 
2250   PetscFunctionBegin;
2251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2252   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2253   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2254   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2255   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2256   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2257   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2258   mesh->supports[off+supportPos] = supportPoint;
2259   PetscFunctionReturn(0);
2260 }
2261 
2262 /*@C
2263   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2264 
2265   Not collective
2266 
2267   Input Parameters:
2268 + mesh - The DMPlex
2269 . p - The point, which must lie in the chart set with DMPlexSetChart()
2270 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2271 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2272 
2273   Output Parameters:
2274 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2275 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2276 
2277   Note:
2278   If using internal storage (points is NULL on input), each call overwrites the last output.
2279 
2280   Fortran Notes:
2281   Since it returns an array, this routine is only available in Fortran 90, and you must
2282   include petsc.h90 in your code.
2283 
2284   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2285 
2286   Level: beginner
2287 
2288 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2289 @*/
2290 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2291 {
2292   DM_Plex        *mesh = (DM_Plex*) dm->data;
2293   PetscInt       *closure, *fifo;
2294   const PetscInt *tmp = NULL, *tmpO = NULL;
2295   PetscInt        tmpSize, t;
2296   PetscInt        depth       = 0, maxSize;
2297   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2298   PetscErrorCode  ierr;
2299 
2300   PetscFunctionBegin;
2301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2302   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2303   /* This is only 1-level */
2304   if (useCone) {
2305     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2306     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2307     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2308   } else {
2309     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2310     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2311   }
2312   if (depth == 1) {
2313     if (*points) {
2314       closure = *points;
2315     } else {
2316       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2317       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2318     }
2319     closure[0] = p; closure[1] = 0;
2320     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2321       closure[closureSize]   = tmp[t];
2322       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2323     }
2324     if (numPoints) *numPoints = closureSize/2;
2325     if (points)    *points    = closure;
2326     PetscFunctionReturn(0);
2327   }
2328   {
2329     PetscInt c, coneSeries, s,supportSeries;
2330 
2331     c = mesh->maxConeSize;
2332     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2333     s = mesh->maxSupportSize;
2334     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2335     maxSize = 2*PetscMax(coneSeries,supportSeries);
2336   }
2337   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2338   if (*points) {
2339     closure = *points;
2340   } else {
2341     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2342   }
2343   closure[0] = p; closure[1] = 0;
2344   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2345     const PetscInt cp = tmp[t];
2346     const PetscInt co = tmpO ? tmpO[t] : 0;
2347 
2348     closure[closureSize]   = cp;
2349     closure[closureSize+1] = co;
2350     fifo[fifoSize]         = cp;
2351     fifo[fifoSize+1]       = co;
2352   }
2353   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2354   while (fifoSize - fifoStart) {
2355     const PetscInt q   = fifo[fifoStart];
2356     const PetscInt o   = fifo[fifoStart+1];
2357     const PetscInt rev = o >= 0 ? 0 : 1;
2358     const PetscInt off = rev ? -(o+1) : o;
2359 
2360     if (useCone) {
2361       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2362       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2363       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2364     } else {
2365       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2366       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2367       tmpO = NULL;
2368     }
2369     for (t = 0; t < tmpSize; ++t) {
2370       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2371       const PetscInt cp = tmp[i];
2372       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2373       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2374        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2375       PetscInt       co = tmpO ? tmpO[i] : 0;
2376       PetscInt       c;
2377 
2378       if (rev) {
2379         PetscInt childSize, coff;
2380         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2381         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2382         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2383       }
2384       /* Check for duplicate */
2385       for (c = 0; c < closureSize; c += 2) {
2386         if (closure[c] == cp) break;
2387       }
2388       if (c == closureSize) {
2389         closure[closureSize]   = cp;
2390         closure[closureSize+1] = co;
2391         fifo[fifoSize]         = cp;
2392         fifo[fifoSize+1]       = co;
2393         closureSize           += 2;
2394         fifoSize              += 2;
2395       }
2396     }
2397     fifoStart += 2;
2398   }
2399   if (numPoints) *numPoints = closureSize/2;
2400   if (points)    *points    = closure;
2401   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2402   PetscFunctionReturn(0);
2403 }
2404 
2405 /*@C
2406   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2407 
2408   Not collective
2409 
2410   Input Parameters:
2411 + mesh - The DMPlex
2412 . p - The point, which must lie in the chart set with DMPlexSetChart()
2413 . orientation - The orientation of the point
2414 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2415 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2416 
2417   Output Parameters:
2418 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2419 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2420 
2421   Note:
2422   If using internal storage (points is NULL on input), each call overwrites the last output.
2423 
2424   Fortran Notes:
2425   Since it returns an array, this routine is only available in Fortran 90, and you must
2426   include petsc.h90 in your code.
2427 
2428   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2429 
2430   Level: beginner
2431 
2432 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2433 @*/
2434 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2435 {
2436   DM_Plex        *mesh = (DM_Plex*) dm->data;
2437   PetscInt       *closure, *fifo;
2438   const PetscInt *tmp = NULL, *tmpO = NULL;
2439   PetscInt        tmpSize, t;
2440   PetscInt        depth       = 0, maxSize;
2441   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2442   PetscErrorCode  ierr;
2443 
2444   PetscFunctionBegin;
2445   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2446   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2447   /* This is only 1-level */
2448   if (useCone) {
2449     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2450     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2451     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2452   } else {
2453     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2454     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2455   }
2456   if (depth == 1) {
2457     if (*points) {
2458       closure = *points;
2459     } else {
2460       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2461       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2462     }
2463     closure[0] = p; closure[1] = ornt;
2464     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2465       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2466       closure[closureSize]   = tmp[i];
2467       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2468     }
2469     if (numPoints) *numPoints = closureSize/2;
2470     if (points)    *points    = closure;
2471     PetscFunctionReturn(0);
2472   }
2473   {
2474     PetscInt c, coneSeries, s,supportSeries;
2475 
2476     c = mesh->maxConeSize;
2477     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2478     s = mesh->maxSupportSize;
2479     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2480     maxSize = 2*PetscMax(coneSeries,supportSeries);
2481   }
2482   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2483   if (*points) {
2484     closure = *points;
2485   } else {
2486     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2487   }
2488   closure[0] = p; closure[1] = ornt;
2489   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2490     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2491     const PetscInt cp = tmp[i];
2492     PetscInt       co = tmpO ? tmpO[i] : 0;
2493 
2494     if (ornt < 0) {
2495       PetscInt childSize, coff;
2496       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2497       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2498       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2499     }
2500     closure[closureSize]   = cp;
2501     closure[closureSize+1] = co;
2502     fifo[fifoSize]         = cp;
2503     fifo[fifoSize+1]       = co;
2504   }
2505   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2506   while (fifoSize - fifoStart) {
2507     const PetscInt q   = fifo[fifoStart];
2508     const PetscInt o   = fifo[fifoStart+1];
2509     const PetscInt rev = o >= 0 ? 0 : 1;
2510     const PetscInt off = rev ? -(o+1) : o;
2511 
2512     if (useCone) {
2513       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2514       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2515       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2516     } else {
2517       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2518       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2519       tmpO = NULL;
2520     }
2521     for (t = 0; t < tmpSize; ++t) {
2522       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2523       const PetscInt cp = tmp[i];
2524       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2525       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2526        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2527       PetscInt       co = tmpO ? tmpO[i] : 0;
2528       PetscInt       c;
2529 
2530       if (rev) {
2531         PetscInt childSize, coff;
2532         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2533         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2534         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2535       }
2536       /* Check for duplicate */
2537       for (c = 0; c < closureSize; c += 2) {
2538         if (closure[c] == cp) break;
2539       }
2540       if (c == closureSize) {
2541         closure[closureSize]   = cp;
2542         closure[closureSize+1] = co;
2543         fifo[fifoSize]         = cp;
2544         fifo[fifoSize+1]       = co;
2545         closureSize           += 2;
2546         fifoSize              += 2;
2547       }
2548     }
2549     fifoStart += 2;
2550   }
2551   if (numPoints) *numPoints = closureSize/2;
2552   if (points)    *points    = closure;
2553   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2554   PetscFunctionReturn(0);
2555 }
2556 
2557 /*@C
2558   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2559 
2560   Not collective
2561 
2562   Input Parameters:
2563 + mesh - The DMPlex
2564 . p - The point, which must lie in the chart set with DMPlexSetChart()
2565 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2566 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2567 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2568 
2569   Note:
2570   If not using internal storage (points is not NULL on input), this call is unnecessary
2571 
2572   Fortran Notes:
2573   Since it returns an array, this routine is only available in Fortran 90, and you must
2574   include petsc.h90 in your code.
2575 
2576   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2577 
2578   Level: beginner
2579 
2580 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2581 @*/
2582 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2583 {
2584   PetscErrorCode ierr;
2585 
2586   PetscFunctionBegin;
2587   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2588   if (numPoints) PetscValidIntPointer(numPoints,4);
2589   if (points) PetscValidPointer(points,5);
2590   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2591   if (numPoints) *numPoints = 0;
2592   PetscFunctionReturn(0);
2593 }
2594 
2595 /*@
2596   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2597 
2598   Not collective
2599 
2600   Input Parameter:
2601 . mesh - The DMPlex
2602 
2603   Output Parameters:
2604 + maxConeSize - The maximum number of in-edges
2605 - maxSupportSize - The maximum number of out-edges
2606 
2607   Level: beginner
2608 
2609 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2610 @*/
2611 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2612 {
2613   DM_Plex *mesh = (DM_Plex*) dm->data;
2614 
2615   PetscFunctionBegin;
2616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2617   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2618   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2619   PetscFunctionReturn(0);
2620 }
2621 
2622 PetscErrorCode DMSetUp_Plex(DM dm)
2623 {
2624   DM_Plex       *mesh = (DM_Plex*) dm->data;
2625   PetscInt       size;
2626   PetscErrorCode ierr;
2627 
2628   PetscFunctionBegin;
2629   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2630   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2631   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2632   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2633   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2634   if (mesh->maxSupportSize) {
2635     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2636     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2637     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2638   }
2639   PetscFunctionReturn(0);
2640 }
2641 
2642 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2643 {
2644   PetscErrorCode ierr;
2645 
2646   PetscFunctionBegin;
2647   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2648   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2649   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2650   if (dm->useNatural && dm->sfMigration) {
2651     PetscSF        sfMigrationInv,sfNatural;
2652     PetscSection   section, sectionSeq;
2653 
2654     (*subdm)->sfMigration = dm->sfMigration;
2655     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2656     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2657     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2658     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2659     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2660 
2661     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2662     (*subdm)->sfNatural = sfNatural;
2663     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2664     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2665   }
2666   PetscFunctionReturn(0);
2667 }
2668 
2669 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2670 {
2671   PetscErrorCode ierr;
2672   PetscInt       i = 0;
2673 
2674   PetscFunctionBegin;
2675   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2676   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2677   (*superdm)->useNatural = PETSC_FALSE;
2678   for (i = 0; i < len; i++){
2679     if (dms[i]->useNatural && dms[i]->sfMigration) {
2680       PetscSF        sfMigrationInv,sfNatural;
2681       PetscSection   section, sectionSeq;
2682 
2683       (*superdm)->sfMigration = dms[i]->sfMigration;
2684       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2685       (*superdm)->useNatural = PETSC_TRUE;
2686       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2687       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2688       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2689       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2690 
2691       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2692       (*superdm)->sfNatural = sfNatural;
2693       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2694       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2695       break;
2696     }
2697   }
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 /*@
2702   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2703 
2704   Not collective
2705 
2706   Input Parameter:
2707 . mesh - The DMPlex
2708 
2709   Output Parameter:
2710 
2711   Note:
2712   This should be called after all calls to DMPlexSetCone()
2713 
2714   Level: beginner
2715 
2716 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2717 @*/
2718 PetscErrorCode DMPlexSymmetrize(DM dm)
2719 {
2720   DM_Plex       *mesh = (DM_Plex*) dm->data;
2721   PetscInt      *offsets;
2722   PetscInt       supportSize;
2723   PetscInt       pStart, pEnd, p;
2724   PetscErrorCode ierr;
2725 
2726   PetscFunctionBegin;
2727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2728   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2729   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2730   /* Calculate support sizes */
2731   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2732   for (p = pStart; p < pEnd; ++p) {
2733     PetscInt dof, off, c;
2734 
2735     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2736     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2737     for (c = off; c < off+dof; ++c) {
2738       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2739     }
2740   }
2741   for (p = pStart; p < pEnd; ++p) {
2742     PetscInt dof;
2743 
2744     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2745 
2746     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2747   }
2748   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2749   /* Calculate supports */
2750   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2751   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2752   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2753   for (p = pStart; p < pEnd; ++p) {
2754     PetscInt dof, off, c;
2755 
2756     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2757     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2758     for (c = off; c < off+dof; ++c) {
2759       const PetscInt q = mesh->cones[c];
2760       PetscInt       offS;
2761 
2762       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2763 
2764       mesh->supports[offS+offsets[q]] = p;
2765       ++offsets[q];
2766     }
2767   }
2768   ierr = PetscFree(offsets);CHKERRQ(ierr);
2769   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2770   PetscFunctionReturn(0);
2771 }
2772 
2773 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2774 {
2775   IS             stratumIS;
2776   PetscErrorCode ierr;
2777 
2778   PetscFunctionBegin;
2779   if (pStart >= pEnd) PetscFunctionReturn(0);
2780 #if defined(PETSC_USE_DEBUG)
2781   {
2782     PetscInt  qStart, qEnd, numLevels, level;
2783     PetscBool overlap = PETSC_FALSE;
2784     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2785     for (level = 0; level < numLevels; level++) {
2786       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2787       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2788     }
2789     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2790   }
2791 #endif
2792   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2793   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2794   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2795   PetscFunctionReturn(0);
2796 }
2797 
2798 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2799 
2800 /*@
2801   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2802   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2803   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2804   the DAG.
2805 
2806   Collective on dm
2807 
2808   Input Parameter:
2809 . mesh - The DMPlex
2810 
2811   Output Parameter:
2812 
2813   Notes:
2814   Concretely, DMPlexStratify() creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
2815   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
2816   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2817   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2818   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2819 
2820   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
2821   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
2822   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
2823   to interpolate only that one (e0), so that
2824   $  cone(c0) = {e0, v2}
2825   $  cone(e0) = {v0, v1}
2826   If DMPlexStratify() is run on this mesh, it will give depths
2827   $  depth 0 = {v0, v1, v2}
2828   $  depth 1 = {e0, c0}
2829   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
2830 
2831   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2832 
2833   Level: beginner
2834 
2835 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2836 @*/
2837 PetscErrorCode DMPlexStratify(DM dm)
2838 {
2839   DM_Plex       *mesh = (DM_Plex*) dm->data;
2840   DMLabel        label;
2841   PetscInt       pStart, pEnd, p;
2842   PetscInt       numRoots = 0, numLeaves = 0;
2843   PetscInt       cMax, fMax, eMax, vMax;
2844   PetscErrorCode ierr;
2845 
2846   PetscFunctionBegin;
2847   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2848   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2849 
2850   /* Create depth label */
2851   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2852   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2853   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2854 
2855   {
2856     /* Initialize roots and count leaves */
2857     PetscInt sMin = PETSC_MAX_INT;
2858     PetscInt sMax = PETSC_MIN_INT;
2859     PetscInt coneSize, supportSize;
2860 
2861     for (p = pStart; p < pEnd; ++p) {
2862       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2863       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2864       if (!coneSize && supportSize) {
2865         sMin = PetscMin(p, sMin);
2866         sMax = PetscMax(p, sMax);
2867         ++numRoots;
2868       } else if (!supportSize && coneSize) {
2869         ++numLeaves;
2870       } else if (!supportSize && !coneSize) {
2871         /* Isolated points */
2872         sMin = PetscMin(p, sMin);
2873         sMax = PetscMax(p, sMax);
2874       }
2875     }
2876     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2877   }
2878 
2879   if (numRoots + numLeaves == (pEnd - pStart)) {
2880     PetscInt sMin = PETSC_MAX_INT;
2881     PetscInt sMax = PETSC_MIN_INT;
2882     PetscInt coneSize, supportSize;
2883 
2884     for (p = pStart; p < pEnd; ++p) {
2885       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2886       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2887       if (!supportSize && coneSize) {
2888         sMin = PetscMin(p, sMin);
2889         sMax = PetscMax(p, sMax);
2890       }
2891     }
2892     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2893   } else {
2894     PetscInt level = 0;
2895     PetscInt qStart, qEnd, q;
2896 
2897     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2898     while (qEnd > qStart) {
2899       PetscInt sMin = PETSC_MAX_INT;
2900       PetscInt sMax = PETSC_MIN_INT;
2901 
2902       for (q = qStart; q < qEnd; ++q) {
2903         const PetscInt *support;
2904         PetscInt        supportSize, s;
2905 
2906         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2907         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2908         for (s = 0; s < supportSize; ++s) {
2909           sMin = PetscMin(support[s], sMin);
2910           sMax = PetscMax(support[s], sMax);
2911         }
2912       }
2913       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2914       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2915       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2916     }
2917   }
2918   { /* just in case there is an empty process */
2919     PetscInt numValues, maxValues = 0, v;
2920 
2921     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2922     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2923     for (v = numValues; v < maxValues; v++) {
2924       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2925     }
2926   }
2927   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2928 
2929   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2930   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2931     PetscInt dim;
2932     DMLabel  dimLabel;
2933 
2934     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2935     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2936     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2937     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2938     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2939     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2940     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2941   }
2942   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2943   PetscFunctionReturn(0);
2944 }
2945 
2946 /*@C
2947   DMPlexGetJoin - Get an array for the join of the set of points
2948 
2949   Not Collective
2950 
2951   Input Parameters:
2952 + dm - The DMPlex object
2953 . numPoints - The number of input points for the join
2954 - points - The input points
2955 
2956   Output Parameters:
2957 + numCoveredPoints - The number of points in the join
2958 - coveredPoints - The points in the join
2959 
2960   Level: intermediate
2961 
2962   Note: Currently, this is restricted to a single level join
2963 
2964   Fortran Notes:
2965   Since it returns an array, this routine is only available in Fortran 90, and you must
2966   include petsc.h90 in your code.
2967 
2968   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2969 
2970 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2971 @*/
2972 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2973 {
2974   DM_Plex       *mesh = (DM_Plex*) dm->data;
2975   PetscInt      *join[2];
2976   PetscInt       joinSize, i = 0;
2977   PetscInt       dof, off, p, c, m;
2978   PetscErrorCode ierr;
2979 
2980   PetscFunctionBegin;
2981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2982   PetscValidIntPointer(points, 3);
2983   PetscValidIntPointer(numCoveredPoints, 4);
2984   PetscValidPointer(coveredPoints, 5);
2985   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2986   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2987   /* Copy in support of first point */
2988   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2989   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2990   for (joinSize = 0; joinSize < dof; ++joinSize) {
2991     join[i][joinSize] = mesh->supports[off+joinSize];
2992   }
2993   /* Check each successive support */
2994   for (p = 1; p < numPoints; ++p) {
2995     PetscInt newJoinSize = 0;
2996 
2997     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2998     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2999     for (c = 0; c < dof; ++c) {
3000       const PetscInt point = mesh->supports[off+c];
3001 
3002       for (m = 0; m < joinSize; ++m) {
3003         if (point == join[i][m]) {
3004           join[1-i][newJoinSize++] = point;
3005           break;
3006         }
3007       }
3008     }
3009     joinSize = newJoinSize;
3010     i        = 1-i;
3011   }
3012   *numCoveredPoints = joinSize;
3013   *coveredPoints    = join[i];
3014   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3015   PetscFunctionReturn(0);
3016 }
3017 
3018 /*@C
3019   DMPlexRestoreJoin - Restore an array for the join of the set of points
3020 
3021   Not Collective
3022 
3023   Input Parameters:
3024 + dm - The DMPlex object
3025 . numPoints - The number of input points for the join
3026 - points - The input points
3027 
3028   Output Parameters:
3029 + numCoveredPoints - The number of points in the join
3030 - coveredPoints - The points in the join
3031 
3032   Fortran Notes:
3033   Since it returns an array, this routine is only available in Fortran 90, and you must
3034   include petsc.h90 in your code.
3035 
3036   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3037 
3038   Level: intermediate
3039 
3040 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3041 @*/
3042 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3043 {
3044   PetscErrorCode ierr;
3045 
3046   PetscFunctionBegin;
3047   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3048   if (points) PetscValidIntPointer(points,3);
3049   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3050   PetscValidPointer(coveredPoints, 5);
3051   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3052   if (numCoveredPoints) *numCoveredPoints = 0;
3053   PetscFunctionReturn(0);
3054 }
3055 
3056 /*@C
3057   DMPlexGetFullJoin - Get an array for the join of the set of points
3058 
3059   Not Collective
3060 
3061   Input Parameters:
3062 + dm - The DMPlex object
3063 . numPoints - The number of input points for the join
3064 - points - The input points
3065 
3066   Output Parameters:
3067 + numCoveredPoints - The number of points in the join
3068 - coveredPoints - The points in the join
3069 
3070   Fortran Notes:
3071   Since it returns an array, this routine is only available in Fortran 90, and you must
3072   include petsc.h90 in your code.
3073 
3074   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3075 
3076   Level: intermediate
3077 
3078 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3079 @*/
3080 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3081 {
3082   DM_Plex       *mesh = (DM_Plex*) dm->data;
3083   PetscInt      *offsets, **closures;
3084   PetscInt      *join[2];
3085   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3086   PetscInt       p, d, c, m, ms;
3087   PetscErrorCode ierr;
3088 
3089   PetscFunctionBegin;
3090   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3091   PetscValidIntPointer(points, 3);
3092   PetscValidIntPointer(numCoveredPoints, 4);
3093   PetscValidPointer(coveredPoints, 5);
3094 
3095   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3096   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3097   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3098   ms      = mesh->maxSupportSize;
3099   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3100   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3101   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3102 
3103   for (p = 0; p < numPoints; ++p) {
3104     PetscInt closureSize;
3105 
3106     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3107 
3108     offsets[p*(depth+2)+0] = 0;
3109     for (d = 0; d < depth+1; ++d) {
3110       PetscInt pStart, pEnd, i;
3111 
3112       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3113       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3114         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3115           offsets[p*(depth+2)+d+1] = i;
3116           break;
3117         }
3118       }
3119       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3120     }
3121     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3122   }
3123   for (d = 0; d < depth+1; ++d) {
3124     PetscInt dof;
3125 
3126     /* Copy in support of first point */
3127     dof = offsets[d+1] - offsets[d];
3128     for (joinSize = 0; joinSize < dof; ++joinSize) {
3129       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3130     }
3131     /* Check each successive cone */
3132     for (p = 1; p < numPoints && joinSize; ++p) {
3133       PetscInt newJoinSize = 0;
3134 
3135       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3136       for (c = 0; c < dof; ++c) {
3137         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3138 
3139         for (m = 0; m < joinSize; ++m) {
3140           if (point == join[i][m]) {
3141             join[1-i][newJoinSize++] = point;
3142             break;
3143           }
3144         }
3145       }
3146       joinSize = newJoinSize;
3147       i        = 1-i;
3148     }
3149     if (joinSize) break;
3150   }
3151   *numCoveredPoints = joinSize;
3152   *coveredPoints    = join[i];
3153   for (p = 0; p < numPoints; ++p) {
3154     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3155   }
3156   ierr = PetscFree(closures);CHKERRQ(ierr);
3157   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3158   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3159   PetscFunctionReturn(0);
3160 }
3161 
3162 /*@C
3163   DMPlexGetMeet - Get an array for the meet of the set of points
3164 
3165   Not Collective
3166 
3167   Input Parameters:
3168 + dm - The DMPlex object
3169 . numPoints - The number of input points for the meet
3170 - points - The input points
3171 
3172   Output Parameters:
3173 + numCoveredPoints - The number of points in the meet
3174 - coveredPoints - The points in the meet
3175 
3176   Level: intermediate
3177 
3178   Note: Currently, this is restricted to a single level meet
3179 
3180   Fortran Notes:
3181   Since it returns an array, this routine is only available in Fortran 90, and you must
3182   include petsc.h90 in your code.
3183 
3184   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3185 
3186 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3187 @*/
3188 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3189 {
3190   DM_Plex       *mesh = (DM_Plex*) dm->data;
3191   PetscInt      *meet[2];
3192   PetscInt       meetSize, i = 0;
3193   PetscInt       dof, off, p, c, m;
3194   PetscErrorCode ierr;
3195 
3196   PetscFunctionBegin;
3197   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3198   PetscValidPointer(points, 2);
3199   PetscValidPointer(numCoveringPoints, 3);
3200   PetscValidPointer(coveringPoints, 4);
3201   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3202   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3203   /* Copy in cone of first point */
3204   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3205   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3206   for (meetSize = 0; meetSize < dof; ++meetSize) {
3207     meet[i][meetSize] = mesh->cones[off+meetSize];
3208   }
3209   /* Check each successive cone */
3210   for (p = 1; p < numPoints; ++p) {
3211     PetscInt newMeetSize = 0;
3212 
3213     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3214     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3215     for (c = 0; c < dof; ++c) {
3216       const PetscInt point = mesh->cones[off+c];
3217 
3218       for (m = 0; m < meetSize; ++m) {
3219         if (point == meet[i][m]) {
3220           meet[1-i][newMeetSize++] = point;
3221           break;
3222         }
3223       }
3224     }
3225     meetSize = newMeetSize;
3226     i        = 1-i;
3227   }
3228   *numCoveringPoints = meetSize;
3229   *coveringPoints    = meet[i];
3230   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3231   PetscFunctionReturn(0);
3232 }
3233 
3234 /*@C
3235   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3236 
3237   Not Collective
3238 
3239   Input Parameters:
3240 + dm - The DMPlex object
3241 . numPoints - The number of input points for the meet
3242 - points - The input points
3243 
3244   Output Parameters:
3245 + numCoveredPoints - The number of points in the meet
3246 - coveredPoints - The points in the meet
3247 
3248   Level: intermediate
3249 
3250   Fortran Notes:
3251   Since it returns an array, this routine is only available in Fortran 90, and you must
3252   include petsc.h90 in your code.
3253 
3254   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3255 
3256 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3257 @*/
3258 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3259 {
3260   PetscErrorCode ierr;
3261 
3262   PetscFunctionBegin;
3263   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3264   if (points) PetscValidIntPointer(points,3);
3265   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3266   PetscValidPointer(coveredPoints,5);
3267   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3268   if (numCoveredPoints) *numCoveredPoints = 0;
3269   PetscFunctionReturn(0);
3270 }
3271 
3272 /*@C
3273   DMPlexGetFullMeet - Get an array for the meet of the set of points
3274 
3275   Not Collective
3276 
3277   Input Parameters:
3278 + dm - The DMPlex object
3279 . numPoints - The number of input points for the meet
3280 - points - The input points
3281 
3282   Output Parameters:
3283 + numCoveredPoints - The number of points in the meet
3284 - coveredPoints - The points in the meet
3285 
3286   Level: intermediate
3287 
3288   Fortran Notes:
3289   Since it returns an array, this routine is only available in Fortran 90, and you must
3290   include petsc.h90 in your code.
3291 
3292   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3293 
3294 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3295 @*/
3296 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3297 {
3298   DM_Plex       *mesh = (DM_Plex*) dm->data;
3299   PetscInt      *offsets, **closures;
3300   PetscInt      *meet[2];
3301   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3302   PetscInt       p, h, c, m, mc;
3303   PetscErrorCode ierr;
3304 
3305   PetscFunctionBegin;
3306   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3307   PetscValidPointer(points, 2);
3308   PetscValidPointer(numCoveredPoints, 3);
3309   PetscValidPointer(coveredPoints, 4);
3310 
3311   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3312   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3313   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3314   mc      = mesh->maxConeSize;
3315   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3316   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3317   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3318 
3319   for (p = 0; p < numPoints; ++p) {
3320     PetscInt closureSize;
3321 
3322     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3323 
3324     offsets[p*(height+2)+0] = 0;
3325     for (h = 0; h < height+1; ++h) {
3326       PetscInt pStart, pEnd, i;
3327 
3328       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3329       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3330         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3331           offsets[p*(height+2)+h+1] = i;
3332           break;
3333         }
3334       }
3335       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3336     }
3337     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3338   }
3339   for (h = 0; h < height+1; ++h) {
3340     PetscInt dof;
3341 
3342     /* Copy in cone of first point */
3343     dof = offsets[h+1] - offsets[h];
3344     for (meetSize = 0; meetSize < dof; ++meetSize) {
3345       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3346     }
3347     /* Check each successive cone */
3348     for (p = 1; p < numPoints && meetSize; ++p) {
3349       PetscInt newMeetSize = 0;
3350 
3351       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3352       for (c = 0; c < dof; ++c) {
3353         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3354 
3355         for (m = 0; m < meetSize; ++m) {
3356           if (point == meet[i][m]) {
3357             meet[1-i][newMeetSize++] = point;
3358             break;
3359           }
3360         }
3361       }
3362       meetSize = newMeetSize;
3363       i        = 1-i;
3364     }
3365     if (meetSize) break;
3366   }
3367   *numCoveredPoints = meetSize;
3368   *coveredPoints    = meet[i];
3369   for (p = 0; p < numPoints; ++p) {
3370     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3371   }
3372   ierr = PetscFree(closures);CHKERRQ(ierr);
3373   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3374   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3375   PetscFunctionReturn(0);
3376 }
3377 
3378 /*@C
3379   DMPlexEqual - Determine if two DMs have the same topology
3380 
3381   Not Collective
3382 
3383   Input Parameters:
3384 + dmA - A DMPlex object
3385 - dmB - A DMPlex object
3386 
3387   Output Parameters:
3388 . equal - PETSC_TRUE if the topologies are identical
3389 
3390   Level: intermediate
3391 
3392   Notes:
3393   We are not solving graph isomorphism, so we do not permutation.
3394 
3395 .seealso: DMPlexGetCone()
3396 @*/
3397 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3398 {
3399   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3400   PetscErrorCode ierr;
3401 
3402   PetscFunctionBegin;
3403   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3404   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3405   PetscValidPointer(equal, 3);
3406 
3407   *equal = PETSC_FALSE;
3408   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3409   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3410   if (depth != depthB) PetscFunctionReturn(0);
3411   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3412   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3413   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3414   for (p = pStart; p < pEnd; ++p) {
3415     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3416     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3417 
3418     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3419     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3420     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3421     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3422     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3423     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3424     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3425     for (c = 0; c < coneSize; ++c) {
3426       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3427       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3428     }
3429     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3430     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3431     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3432     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3433     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3434     for (s = 0; s < supportSize; ++s) {
3435       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3436     }
3437   }
3438   *equal = PETSC_TRUE;
3439   PetscFunctionReturn(0);
3440 }
3441 
3442 /*@C
3443   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3444 
3445   Not Collective
3446 
3447   Input Parameters:
3448 + dm         - The DMPlex
3449 . cellDim    - The cell dimension
3450 - numCorners - The number of vertices on a cell
3451 
3452   Output Parameters:
3453 . numFaceVertices - The number of vertices on a face
3454 
3455   Level: developer
3456 
3457   Notes:
3458   Of course this can only work for a restricted set of symmetric shapes
3459 
3460 .seealso: DMPlexGetCone()
3461 @*/
3462 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3463 {
3464   MPI_Comm       comm;
3465   PetscErrorCode ierr;
3466 
3467   PetscFunctionBegin;
3468   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3469   PetscValidPointer(numFaceVertices,3);
3470   switch (cellDim) {
3471   case 0:
3472     *numFaceVertices = 0;
3473     break;
3474   case 1:
3475     *numFaceVertices = 1;
3476     break;
3477   case 2:
3478     switch (numCorners) {
3479     case 3: /* triangle */
3480       *numFaceVertices = 2; /* Edge has 2 vertices */
3481       break;
3482     case 4: /* quadrilateral */
3483       *numFaceVertices = 2; /* Edge has 2 vertices */
3484       break;
3485     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3486       *numFaceVertices = 3; /* Edge has 3 vertices */
3487       break;
3488     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3489       *numFaceVertices = 3; /* Edge has 3 vertices */
3490       break;
3491     default:
3492       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3493     }
3494     break;
3495   case 3:
3496     switch (numCorners) {
3497     case 4: /* tetradehdron */
3498       *numFaceVertices = 3; /* Face has 3 vertices */
3499       break;
3500     case 6: /* tet cohesive cells */
3501       *numFaceVertices = 4; /* Face has 4 vertices */
3502       break;
3503     case 8: /* hexahedron */
3504       *numFaceVertices = 4; /* Face has 4 vertices */
3505       break;
3506     case 9: /* tet cohesive Lagrange cells */
3507       *numFaceVertices = 6; /* Face has 6 vertices */
3508       break;
3509     case 10: /* quadratic tetrahedron */
3510       *numFaceVertices = 6; /* Face has 6 vertices */
3511       break;
3512     case 12: /* hex cohesive Lagrange cells */
3513       *numFaceVertices = 6; /* Face has 6 vertices */
3514       break;
3515     case 18: /* quadratic tet cohesive Lagrange cells */
3516       *numFaceVertices = 6; /* Face has 6 vertices */
3517       break;
3518     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3519       *numFaceVertices = 9; /* Face has 9 vertices */
3520       break;
3521     default:
3522       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3523     }
3524     break;
3525   default:
3526     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3527   }
3528   PetscFunctionReturn(0);
3529 }
3530 
3531 /*@
3532   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3533 
3534   Not Collective
3535 
3536   Input Parameter:
3537 . dm    - The DMPlex object
3538 
3539   Output Parameter:
3540 . depthLabel - The DMLabel recording point depth
3541 
3542   Level: developer
3543 
3544 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3545 @*/
3546 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3547 {
3548   PetscErrorCode ierr;
3549 
3550   PetscFunctionBegin;
3551   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3552   PetscValidPointer(depthLabel, 2);
3553   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3554   *depthLabel = dm->depthLabel;
3555   PetscFunctionReturn(0);
3556 }
3557 
3558 /*@
3559   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3560 
3561   Not Collective
3562 
3563   Input Parameter:
3564 . dm    - The DMPlex object
3565 
3566   Output Parameter:
3567 . depth - The number of strata (breadth first levels) in the DAG
3568 
3569   Level: developer
3570 
3571   Notes:
3572   This returns maximum of point depths over all points, i.e. maximum value of the label returned by DMPlexGetDepthLabel().
3573   The point depth is described more in detail in DMPlexSymmetrize().
3574 
3575 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum(), DMPlexSymmetrize()
3576 @*/
3577 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3578 {
3579   DMLabel        label;
3580   PetscInt       d = 0;
3581   PetscErrorCode ierr;
3582 
3583   PetscFunctionBegin;
3584   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3585   PetscValidPointer(depth, 2);
3586   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3587   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3588   *depth = d-1;
3589   PetscFunctionReturn(0);
3590 }
3591 
3592 /*@
3593   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3594 
3595   Not Collective
3596 
3597   Input Parameters:
3598 + dm           - The DMPlex object
3599 - stratumValue - The requested depth
3600 
3601   Output Parameters:
3602 + start - The first point at this depth
3603 - end   - One beyond the last point at this depth
3604 
3605   Notes:
3606   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3607   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3608   higher dimension, e.g., "edges".
3609 
3610   Level: developer
3611 
3612 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3613 @*/
3614 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3615 {
3616   DMLabel        label;
3617   PetscInt       pStart, pEnd;
3618   PetscErrorCode ierr;
3619 
3620   PetscFunctionBegin;
3621   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3622   if (start) {PetscValidPointer(start, 3); *start = 0;}
3623   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3624   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3625   if (pStart == pEnd) PetscFunctionReturn(0);
3626   if (stratumValue < 0) {
3627     if (start) *start = pStart;
3628     if (end)   *end   = pEnd;
3629     PetscFunctionReturn(0);
3630   }
3631   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3632   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3633   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3634   PetscFunctionReturn(0);
3635 }
3636 
3637 /*@
3638   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3639 
3640   Not Collective
3641 
3642   Input Parameters:
3643 + dm           - The DMPlex object
3644 - stratumValue - The requested height
3645 
3646   Output Parameters:
3647 + start - The first point at this height
3648 - end   - One beyond the last point at this height
3649 
3650   Notes:
3651   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3652   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3653   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3654 
3655   Level: developer
3656 
3657 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3658 @*/
3659 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3660 {
3661   DMLabel        label;
3662   PetscInt       depth, pStart, pEnd;
3663   PetscErrorCode ierr;
3664 
3665   PetscFunctionBegin;
3666   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3667   if (start) {PetscValidPointer(start, 3); *start = 0;}
3668   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3669   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3670   if (pStart == pEnd) PetscFunctionReturn(0);
3671   if (stratumValue < 0) {
3672     if (start) *start = pStart;
3673     if (end)   *end   = pEnd;
3674     PetscFunctionReturn(0);
3675   }
3676   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3677   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3678   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3679   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3680   PetscFunctionReturn(0);
3681 }
3682 
3683 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3684 {
3685   PetscSection   section, s;
3686   Mat            m;
3687   PetscInt       maxHeight;
3688   PetscErrorCode ierr;
3689 
3690   PetscFunctionBegin;
3691   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3692   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3693   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3694   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3695   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3696   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3697   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3698   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3699   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3700   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3701   ierr = MatDestroy(&m);CHKERRQ(ierr);
3702 
3703   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3704   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3705   PetscFunctionReturn(0);
3706 }
3707 
3708 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3709 {
3710   Vec            coordsLocal;
3711   DM             coordsDM;
3712   PetscErrorCode ierr;
3713 
3714   PetscFunctionBegin;
3715   *field = NULL;
3716   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3717   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3718   if (coordsLocal && coordsDM) {
3719     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3720   }
3721   PetscFunctionReturn(0);
3722 }
3723 
3724 /*@C
3725   DMPlexGetConeSection - Return a section which describes the layout of cone data
3726 
3727   Not Collective
3728 
3729   Input Parameters:
3730 . dm        - The DMPlex object
3731 
3732   Output Parameter:
3733 . section - The PetscSection object
3734 
3735   Level: developer
3736 
3737 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3738 @*/
3739 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3740 {
3741   DM_Plex *mesh = (DM_Plex*) dm->data;
3742 
3743   PetscFunctionBegin;
3744   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3745   if (section) *section = mesh->coneSection;
3746   PetscFunctionReturn(0);
3747 }
3748 
3749 /*@C
3750   DMPlexGetSupportSection - Return a section which describes the layout of support data
3751 
3752   Not Collective
3753 
3754   Input Parameters:
3755 . dm        - The DMPlex object
3756 
3757   Output Parameter:
3758 . section - The PetscSection object
3759 
3760   Level: developer
3761 
3762 .seealso: DMPlexGetConeSection()
3763 @*/
3764 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3765 {
3766   DM_Plex *mesh = (DM_Plex*) dm->data;
3767 
3768   PetscFunctionBegin;
3769   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3770   if (section) *section = mesh->supportSection;
3771   PetscFunctionReturn(0);
3772 }
3773 
3774 /*@C
3775   DMPlexGetCones - Return cone data
3776 
3777   Not Collective
3778 
3779   Input Parameters:
3780 . dm        - The DMPlex object
3781 
3782   Output Parameter:
3783 . cones - The cone for each point
3784 
3785   Level: developer
3786 
3787 .seealso: DMPlexGetConeSection()
3788 @*/
3789 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3790 {
3791   DM_Plex *mesh = (DM_Plex*) dm->data;
3792 
3793   PetscFunctionBegin;
3794   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3795   if (cones) *cones = mesh->cones;
3796   PetscFunctionReturn(0);
3797 }
3798 
3799 /*@C
3800   DMPlexGetConeOrientations - Return cone orientation data
3801 
3802   Not Collective
3803 
3804   Input Parameters:
3805 . dm        - The DMPlex object
3806 
3807   Output Parameter:
3808 . coneOrientations - The cone orientation for each point
3809 
3810   Level: developer
3811 
3812 .seealso: DMPlexGetConeSection()
3813 @*/
3814 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3815 {
3816   DM_Plex *mesh = (DM_Plex*) dm->data;
3817 
3818   PetscFunctionBegin;
3819   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3820   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3821   PetscFunctionReturn(0);
3822 }
3823 
3824 /******************************** FEM Support **********************************/
3825 
3826 /*
3827  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3828  representing a line in the section.
3829 */
3830 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3831 {
3832   PetscErrorCode ierr;
3833 
3834   PetscFunctionBeginHot;
3835   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3836   if (line < 0) {
3837     *k = 0;
3838     *Nc = 0;
3839   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3840     *k = 1;
3841   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3842     /* An order k SEM disc has k-1 dofs on an edge */
3843     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3844     *k = *k / *Nc + 1;
3845   }
3846   PetscFunctionReturn(0);
3847 }
3848 
3849 /*@
3850 
3851   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3852   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3853   section provided (or the section of the DM).
3854 
3855   Input Parameters:
3856 + dm      - The DM
3857 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3858 - section - The PetscSection to reorder, or NULL for the default section
3859 
3860   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3861   degree of the basis.
3862 
3863   Example:
3864   A typical interpolated single-quad mesh might order points as
3865 .vb
3866   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3867 
3868   v4 -- e6 -- v3
3869   |           |
3870   e7    c0    e8
3871   |           |
3872   v1 -- e5 -- v2
3873 .ve
3874 
3875   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3876   dofs in the order of points, e.g.,
3877 .vb
3878     c0 -> [0,1,2,3]
3879     v1 -> [4]
3880     ...
3881     e5 -> [8, 9]
3882 .ve
3883 
3884   which corresponds to the dofs
3885 .vb
3886     6   10  11  7
3887     13  2   3   15
3888     12  0   1   14
3889     4   8   9   5
3890 .ve
3891 
3892   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3893 .vb
3894   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3895 .ve
3896 
3897   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3898 .vb
3899    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3900 .ve
3901 
3902   Level: developer
3903 
3904 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlocalSection()
3905 @*/
3906 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3907 {
3908   DMLabel        label;
3909   PetscInt      *perm;
3910   PetscInt       dim, depth = -1, eStart = -1, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3911   PetscBool      vertexchart;
3912   PetscErrorCode ierr;
3913 
3914   PetscFunctionBegin;
3915   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3916   if (dim < 1) PetscFunctionReturn(0);
3917   if (point < 0) {
3918     PetscInt sStart,sEnd;
3919 
3920     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
3921     point = sEnd-sStart ? sStart : point;
3922   }
3923   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3924   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
3925   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
3926   if (depth == 1) {eStart = point;}
3927   else if  (depth == dim) {
3928     const PetscInt *cone;
3929 
3930     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3931     if (dim == 2) eStart = cone[0];
3932     else if (dim == 3) {
3933       const PetscInt *cone2;
3934       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3935       eStart = cone2[0];
3936     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3937   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3938   {                             /* Determine whether the chart covers all points or just vertices. */
3939     PetscInt pStart,pEnd,cStart,cEnd;
3940     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3941     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3942     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3943     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3944   }
3945   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3946   for (f = 0; f < Nf; ++f) {
3947     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3948     size += PetscPowInt(k+1, dim)*Nc;
3949   }
3950   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3951   for (f = 0; f < Nf; ++f) {
3952     switch (dim) {
3953     case 1:
3954       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3955       /*
3956         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3957         We want              [ vtx0; edge of length k-1; vtx1 ]
3958       */
3959       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3960       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3961       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3962       foffset = offset;
3963       break;
3964     case 2:
3965       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3966       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3967       /* The SEM order is
3968 
3969          v_lb, {e_b}, v_rb,
3970          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3971          v_lt, reverse {e_t}, v_rt
3972       */
3973       {
3974         const PetscInt of   = 0;
3975         const PetscInt oeb  = of   + PetscSqr(k-1);
3976         const PetscInt oer  = oeb  + (k-1);
3977         const PetscInt oet  = oer  + (k-1);
3978         const PetscInt oel  = oet  + (k-1);
3979         const PetscInt ovlb = oel  + (k-1);
3980         const PetscInt ovrb = ovlb + 1;
3981         const PetscInt ovrt = ovrb + 1;
3982         const PetscInt ovlt = ovrt + 1;
3983         PetscInt       o;
3984 
3985         /* bottom */
3986         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3987         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3988         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3989         /* middle */
3990         for (i = 0; i < k-1; ++i) {
3991           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3992           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3993           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3994         }
3995         /* top */
3996         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3997         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3998         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3999         foffset = offset;
4000       }
4001       break;
4002     case 3:
4003       /* The original hex closure is
4004 
4005          {c,
4006           f_b, f_t, f_f, f_b, f_r, f_l,
4007           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
4008           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
4009       */
4010       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
4011       /* The SEM order is
4012          Bottom Slice
4013          v_blf, {e^{(k-1)-n}_bf}, v_brf,
4014          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
4015          v_blb, {e_bb}, v_brb,
4016 
4017          Middle Slice (j)
4018          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
4019          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
4020          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
4021 
4022          Top Slice
4023          v_tlf, {e_tf}, v_trf,
4024          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
4025          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
4026       */
4027       {
4028         const PetscInt oc    = 0;
4029         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4030         const PetscInt oft   = ofb   + PetscSqr(k-1);
4031         const PetscInt off   = oft   + PetscSqr(k-1);
4032         const PetscInt ofk   = off   + PetscSqr(k-1);
4033         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4034         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4035         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4036         const PetscInt oebb  = oebl  + (k-1);
4037         const PetscInt oebr  = oebb  + (k-1);
4038         const PetscInt oebf  = oebr  + (k-1);
4039         const PetscInt oetf  = oebf  + (k-1);
4040         const PetscInt oetr  = oetf  + (k-1);
4041         const PetscInt oetb  = oetr  + (k-1);
4042         const PetscInt oetl  = oetb  + (k-1);
4043         const PetscInt oerf  = oetl  + (k-1);
4044         const PetscInt oelf  = oerf  + (k-1);
4045         const PetscInt oelb  = oelf  + (k-1);
4046         const PetscInt oerb  = oelb  + (k-1);
4047         const PetscInt ovblf = oerb  + (k-1);
4048         const PetscInt ovblb = ovblf + 1;
4049         const PetscInt ovbrb = ovblb + 1;
4050         const PetscInt ovbrf = ovbrb + 1;
4051         const PetscInt ovtlf = ovbrf + 1;
4052         const PetscInt ovtrf = ovtlf + 1;
4053         const PetscInt ovtrb = ovtrf + 1;
4054         const PetscInt ovtlb = ovtrb + 1;
4055         PetscInt       o, n;
4056 
4057         /* Bottom Slice */
4058         /*   bottom */
4059         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4060         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4061         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4062         /*   middle */
4063         for (i = 0; i < k-1; ++i) {
4064           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4065           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4066           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4067         }
4068         /*   top */
4069         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4070         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4071         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4072 
4073         /* Middle Slice */
4074         for (j = 0; j < k-1; ++j) {
4075           /*   bottom */
4076           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4077           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4078           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4079           /*   middle */
4080           for (i = 0; i < k-1; ++i) {
4081             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4082             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4083             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4084           }
4085           /*   top */
4086           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4087           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4088           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4089         }
4090 
4091         /* Top Slice */
4092         /*   bottom */
4093         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4094         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4095         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4096         /*   middle */
4097         for (i = 0; i < k-1; ++i) {
4098           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4099           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4100           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4101         }
4102         /*   top */
4103         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4104         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4105         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4106 
4107         foffset = offset;
4108       }
4109       break;
4110     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4111     }
4112   }
4113   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4114   /* Check permutation */
4115   {
4116     PetscInt *check;
4117 
4118     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4119     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4120     for (i = 0; i < size; ++i) check[perm[i]] = i;
4121     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4122     ierr = PetscFree(check);CHKERRQ(ierr);
4123   }
4124   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4125   PetscFunctionReturn(0);
4126 }
4127 
4128 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4129 {
4130   PetscDS        prob;
4131   PetscInt       depth, Nf, h;
4132   DMLabel        label;
4133   PetscErrorCode ierr;
4134 
4135   PetscFunctionBeginHot;
4136   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4137   Nf      = prob->Nf;
4138   label   = dm->depthLabel;
4139   *dspace = NULL;
4140   if (field < Nf) {
4141     PetscObject disc = prob->disc[field];
4142 
4143     if (disc->classid == PETSCFE_CLASSID) {
4144       PetscDualSpace dsp;
4145 
4146       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4147       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4148       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4149       h    = depth - 1 - h;
4150       if (h) {
4151         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4152       } else {
4153         *dspace = dsp;
4154       }
4155     }
4156   }
4157   PetscFunctionReturn(0);
4158 }
4159 
4160 
4161 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4162 {
4163   PetscScalar    *array, *vArray;
4164   const PetscInt *cone, *coneO;
4165   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4166   PetscErrorCode  ierr;
4167 
4168   PetscFunctionBeginHot;
4169   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4170   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4171   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4172   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4173   if (!values || !*values) {
4174     if ((point >= pStart) && (point < pEnd)) {
4175       PetscInt dof;
4176 
4177       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4178       size += dof;
4179     }
4180     for (p = 0; p < numPoints; ++p) {
4181       const PetscInt cp = cone[p];
4182       PetscInt       dof;
4183 
4184       if ((cp < pStart) || (cp >= pEnd)) continue;
4185       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4186       size += dof;
4187     }
4188     if (!values) {
4189       if (csize) *csize = size;
4190       PetscFunctionReturn(0);
4191     }
4192     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4193   } else {
4194     array = *values;
4195   }
4196   size = 0;
4197   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4198   if ((point >= pStart) && (point < pEnd)) {
4199     PetscInt     dof, off, d;
4200     PetscScalar *varr;
4201 
4202     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4203     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4204     varr = &vArray[off];
4205     for (d = 0; d < dof; ++d, ++offset) {
4206       array[offset] = varr[d];
4207     }
4208     size += dof;
4209   }
4210   for (p = 0; p < numPoints; ++p) {
4211     const PetscInt cp = cone[p];
4212     PetscInt       o  = coneO[p];
4213     PetscInt       dof, off, d;
4214     PetscScalar   *varr;
4215 
4216     if ((cp < pStart) || (cp >= pEnd)) continue;
4217     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4218     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4219     varr = &vArray[off];
4220     if (o >= 0) {
4221       for (d = 0; d < dof; ++d, ++offset) {
4222         array[offset] = varr[d];
4223       }
4224     } else {
4225       for (d = dof-1; d >= 0; --d, ++offset) {
4226         array[offset] = varr[d];
4227       }
4228     }
4229     size += dof;
4230   }
4231   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4232   if (!*values) {
4233     if (csize) *csize = size;
4234     *values = array;
4235   } else {
4236     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4237     *csize = size;
4238   }
4239   PetscFunctionReturn(0);
4240 }
4241 
4242 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4243 {
4244   const PetscInt *cla;
4245   PetscInt       np, *pts = NULL;
4246   PetscErrorCode ierr;
4247 
4248   PetscFunctionBeginHot;
4249   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4250   if (!*clPoints) {
4251     PetscInt pStart, pEnd, p, q;
4252 
4253     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4254     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4255     /* Compress out points not in the section */
4256     for (p = 0, q = 0; p < np; p++) {
4257       PetscInt r = pts[2*p];
4258       if ((r >= pStart) && (r < pEnd)) {
4259         pts[q*2]   = r;
4260         pts[q*2+1] = pts[2*p+1];
4261         ++q;
4262       }
4263     }
4264     np = q;
4265     cla = NULL;
4266   } else {
4267     PetscInt dof, off;
4268 
4269     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4270     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4271     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4272     np   = dof/2;
4273     pts  = (PetscInt *) &cla[off];
4274   }
4275   *numPoints = np;
4276   *points    = pts;
4277   *clp       = cla;
4278 
4279   PetscFunctionReturn(0);
4280 }
4281 
4282 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4283 {
4284   PetscErrorCode ierr;
4285 
4286   PetscFunctionBeginHot;
4287   if (!*clPoints) {
4288     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4289   } else {
4290     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4291   }
4292   *numPoints = 0;
4293   *points    = NULL;
4294   *clSec     = NULL;
4295   *clPoints  = NULL;
4296   *clp       = NULL;
4297   PetscFunctionReturn(0);
4298 }
4299 
4300 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4301 {
4302   PetscInt          offset = 0, p;
4303   const PetscInt    **perms = NULL;
4304   const PetscScalar **flips = NULL;
4305   PetscErrorCode    ierr;
4306 
4307   PetscFunctionBeginHot;
4308   *size = 0;
4309   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4310   for (p = 0; p < numPoints; p++) {
4311     const PetscInt    point = points[2*p];
4312     const PetscInt    *perm = perms ? perms[p] : NULL;
4313     const PetscScalar *flip = flips ? flips[p] : NULL;
4314     PetscInt          dof, off, d;
4315     const PetscScalar *varr;
4316 
4317     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4318     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4319     varr = &vArray[off];
4320     if (clperm) {
4321       if (perm) {
4322         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4323       } else {
4324         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4325       }
4326       if (flip) {
4327         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4328       }
4329     } else {
4330       if (perm) {
4331         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4332       } else {
4333         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4334       }
4335       if (flip) {
4336         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4337       }
4338     }
4339     offset += dof;
4340   }
4341   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4342   *size = offset;
4343   PetscFunctionReturn(0);
4344 }
4345 
4346 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4347 {
4348   PetscInt          offset = 0, f;
4349   PetscErrorCode    ierr;
4350 
4351   PetscFunctionBeginHot;
4352   *size = 0;
4353   for (f = 0; f < numFields; ++f) {
4354     PetscInt          p;
4355     const PetscInt    **perms = NULL;
4356     const PetscScalar **flips = NULL;
4357 
4358     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4359     for (p = 0; p < numPoints; p++) {
4360       const PetscInt    point = points[2*p];
4361       PetscInt          fdof, foff, b;
4362       const PetscScalar *varr;
4363       const PetscInt    *perm = perms ? perms[p] : NULL;
4364       const PetscScalar *flip = flips ? flips[p] : NULL;
4365 
4366       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4367       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4368       varr = &vArray[foff];
4369       if (clperm) {
4370         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4371         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4372         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4373       } else {
4374         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4375         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4376         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4377       }
4378       offset += fdof;
4379     }
4380     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4381   }
4382   *size = offset;
4383   PetscFunctionReturn(0);
4384 }
4385 
4386 /*@C
4387   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4388 
4389   Not collective
4390 
4391   Input Parameters:
4392 + dm - The DM
4393 . section - The section describing the layout in v, or NULL to use the default section
4394 . v - The local vector
4395 . point - The point in the DM
4396 . csize - The size of the input values array, or NULL
4397 - values - An array to use for the values, or NULL to have it allocated automatically
4398 
4399   Output Parameters:
4400 + csize - The number of values in the closure
4401 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4402 
4403 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4404 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4405 $ assembly function, and a user may already have allocated storage for this operation.
4406 $
4407 $ A typical use could be
4408 $
4409 $  values = NULL;
4410 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4411 $  for (cl = 0; cl < clSize; ++cl) {
4412 $    <Compute on closure>
4413 $  }
4414 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4415 $
4416 $ or
4417 $
4418 $  PetscMalloc1(clMaxSize, &values);
4419 $  for (p = pStart; p < pEnd; ++p) {
4420 $    clSize = clMaxSize;
4421 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4422 $    for (cl = 0; cl < clSize; ++cl) {
4423 $      <Compute on closure>
4424 $    }
4425 $  }
4426 $  PetscFree(values);
4427 
4428   Fortran Notes:
4429   Since it returns an array, this routine is only available in Fortran 90, and you must
4430   include petsc.h90 in your code.
4431 
4432   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4433 
4434   Level: intermediate
4435 
4436 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4437 @*/
4438 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4439 {
4440   PetscSection       clSection;
4441   IS                 clPoints;
4442   PetscScalar       *array;
4443   const PetscScalar *vArray;
4444   PetscInt          *points = NULL;
4445   const PetscInt    *clp, *perm;
4446   PetscInt           depth, numFields, numPoints, size;
4447   PetscErrorCode     ierr;
4448 
4449   PetscFunctionBeginHot;
4450   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4451   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4452   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4453   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4454   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4455   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4456   if (depth == 1 && numFields < 2) {
4457     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4458     PetscFunctionReturn(0);
4459   }
4460   /* Get points */
4461   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4462   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4463   /* Get array */
4464   if (!values || !*values) {
4465     PetscInt asize = 0, dof, p;
4466 
4467     for (p = 0; p < numPoints*2; p += 2) {
4468       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4469       asize += dof;
4470     }
4471     if (!values) {
4472       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4473       if (csize) *csize = asize;
4474       PetscFunctionReturn(0);
4475     }
4476     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4477   } else {
4478     array = *values;
4479   }
4480   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4481   /* Get values */
4482   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4483   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4484   /* Cleanup points */
4485   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4486   /* Cleanup array */
4487   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4488   if (!*values) {
4489     if (csize) *csize = size;
4490     *values = array;
4491   } else {
4492     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4493     *csize = size;
4494   }
4495   PetscFunctionReturn(0);
4496 }
4497 
4498 /*@C
4499   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4500 
4501   Not collective
4502 
4503   Input Parameters:
4504 + dm - The DM
4505 . section - The section describing the layout in v, or NULL to use the default section
4506 . v - The local vector
4507 . point - The point in the DM
4508 . csize - The number of values in the closure, or NULL
4509 - values - The array of values, which is a borrowed array and should not be freed
4510 
4511   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4512 
4513   Fortran Notes:
4514   Since it returns an array, this routine is only available in Fortran 90, and you must
4515   include petsc.h90 in your code.
4516 
4517   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4518 
4519   Level: intermediate
4520 
4521 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4522 @*/
4523 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4524 {
4525   PetscInt       size = 0;
4526   PetscErrorCode ierr;
4527 
4528   PetscFunctionBegin;
4529   /* Should work without recalculating size */
4530   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4531   *values = NULL;
4532   PetscFunctionReturn(0);
4533 }
4534 
4535 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4536 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4537 
4538 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4539 {
4540   PetscInt        cdof;   /* The number of constraints on this point */
4541   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4542   PetscScalar    *a;
4543   PetscInt        off, cind = 0, k;
4544   PetscErrorCode  ierr;
4545 
4546   PetscFunctionBegin;
4547   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4548   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4549   a    = &array[off];
4550   if (!cdof || setBC) {
4551     if (clperm) {
4552       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4553       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4554     } else {
4555       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4556       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4557     }
4558   } else {
4559     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4560     if (clperm) {
4561       if (perm) {for (k = 0; k < dof; ++k) {
4562           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4563           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4564         }
4565       } else {
4566         for (k = 0; k < dof; ++k) {
4567           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4568           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4569         }
4570       }
4571     } else {
4572       if (perm) {
4573         for (k = 0; k < dof; ++k) {
4574           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4575           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4576         }
4577       } else {
4578         for (k = 0; k < dof; ++k) {
4579           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4580           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4581         }
4582       }
4583     }
4584   }
4585   PetscFunctionReturn(0);
4586 }
4587 
4588 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4589 {
4590   PetscInt        cdof;   /* The number of constraints on this point */
4591   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4592   PetscScalar    *a;
4593   PetscInt        off, cind = 0, k;
4594   PetscErrorCode  ierr;
4595 
4596   PetscFunctionBegin;
4597   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4598   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4599   a    = &array[off];
4600   if (cdof) {
4601     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4602     if (clperm) {
4603       if (perm) {
4604         for (k = 0; k < dof; ++k) {
4605           if ((cind < cdof) && (k == cdofs[cind])) {
4606             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4607             cind++;
4608           }
4609         }
4610       } else {
4611         for (k = 0; k < dof; ++k) {
4612           if ((cind < cdof) && (k == cdofs[cind])) {
4613             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4614             cind++;
4615           }
4616         }
4617       }
4618     } else {
4619       if (perm) {
4620         for (k = 0; k < dof; ++k) {
4621           if ((cind < cdof) && (k == cdofs[cind])) {
4622             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4623             cind++;
4624           }
4625         }
4626       } else {
4627         for (k = 0; k < dof; ++k) {
4628           if ((cind < cdof) && (k == cdofs[cind])) {
4629             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4630             cind++;
4631           }
4632         }
4633       }
4634     }
4635   }
4636   PetscFunctionReturn(0);
4637 }
4638 
4639 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4640 {
4641   PetscScalar    *a;
4642   PetscInt        fdof, foff, fcdof, foffset = *offset;
4643   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4644   PetscInt        cind = 0, b;
4645   PetscErrorCode  ierr;
4646 
4647   PetscFunctionBegin;
4648   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4649   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4650   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4651   a    = &array[foff];
4652   if (!fcdof || setBC) {
4653     if (clperm) {
4654       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4655       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4656     } else {
4657       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4658       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4659     }
4660   } else {
4661     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4662     if (clperm) {
4663       if (perm) {
4664         for (b = 0; b < fdof; b++) {
4665           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4666           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4667         }
4668       } else {
4669         for (b = 0; b < fdof; b++) {
4670           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4671           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4672         }
4673       }
4674     } else {
4675       if (perm) {
4676         for (b = 0; b < fdof; b++) {
4677           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4678           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4679         }
4680       } else {
4681         for (b = 0; b < fdof; b++) {
4682           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4683           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4684         }
4685       }
4686     }
4687   }
4688   *offset += fdof;
4689   PetscFunctionReturn(0);
4690 }
4691 
4692 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4693 {
4694   PetscScalar    *a;
4695   PetscInt        fdof, foff, fcdof, foffset = *offset;
4696   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4697   PetscInt        cind = 0, ncind = 0, b;
4698   PetscBool       ncSet, fcSet;
4699   PetscErrorCode  ierr;
4700 
4701   PetscFunctionBegin;
4702   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4703   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4704   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4705   a    = &array[foff];
4706   if (fcdof) {
4707     /* We just override fcdof and fcdofs with Ncc and comps */
4708     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4709     if (clperm) {
4710       if (perm) {
4711         if (comps) {
4712           for (b = 0; b < fdof; b++) {
4713             ncSet = fcSet = PETSC_FALSE;
4714             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4715             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4716             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4717           }
4718         } else {
4719           for (b = 0; b < fdof; b++) {
4720             if ((cind < fcdof) && (b == fcdofs[cind])) {
4721               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4722               ++cind;
4723             }
4724           }
4725         }
4726       } else {
4727         if (comps) {
4728           for (b = 0; b < fdof; b++) {
4729             ncSet = fcSet = PETSC_FALSE;
4730             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4731             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4732             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4733           }
4734         } else {
4735           for (b = 0; b < fdof; b++) {
4736             if ((cind < fcdof) && (b == fcdofs[cind])) {
4737               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4738               ++cind;
4739             }
4740           }
4741         }
4742       }
4743     } else {
4744       if (perm) {
4745         if (comps) {
4746           for (b = 0; b < fdof; b++) {
4747             ncSet = fcSet = PETSC_FALSE;
4748             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4749             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4750             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4751           }
4752         } else {
4753           for (b = 0; b < fdof; b++) {
4754             if ((cind < fcdof) && (b == fcdofs[cind])) {
4755               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4756               ++cind;
4757             }
4758           }
4759         }
4760       } else {
4761         if (comps) {
4762           for (b = 0; b < fdof; b++) {
4763             ncSet = fcSet = PETSC_FALSE;
4764             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4765             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4766             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4767           }
4768         } else {
4769           for (b = 0; b < fdof; b++) {
4770             if ((cind < fcdof) && (b == fcdofs[cind])) {
4771               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4772               ++cind;
4773             }
4774           }
4775         }
4776       }
4777     }
4778   }
4779   *offset += fdof;
4780   PetscFunctionReturn(0);
4781 }
4782 
4783 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4784 {
4785   PetscScalar    *array;
4786   const PetscInt *cone, *coneO;
4787   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4788   PetscErrorCode  ierr;
4789 
4790   PetscFunctionBeginHot;
4791   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4792   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4793   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4794   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4795   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4796   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4797     const PetscInt cp = !p ? point : cone[p-1];
4798     const PetscInt o  = !p ? 0     : coneO[p-1];
4799 
4800     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4801     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4802     /* ADD_VALUES */
4803     {
4804       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4805       PetscScalar    *a;
4806       PetscInt        cdof, coff, cind = 0, k;
4807 
4808       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4809       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4810       a    = &array[coff];
4811       if (!cdof) {
4812         if (o >= 0) {
4813           for (k = 0; k < dof; ++k) {
4814             a[k] += values[off+k];
4815           }
4816         } else {
4817           for (k = 0; k < dof; ++k) {
4818             a[k] += values[off+dof-k-1];
4819           }
4820         }
4821       } else {
4822         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4823         if (o >= 0) {
4824           for (k = 0; k < dof; ++k) {
4825             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4826             a[k] += values[off+k];
4827           }
4828         } else {
4829           for (k = 0; k < dof; ++k) {
4830             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4831             a[k] += values[off+dof-k-1];
4832           }
4833         }
4834       }
4835     }
4836   }
4837   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4838   PetscFunctionReturn(0);
4839 }
4840 
4841 /*@C
4842   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4843 
4844   Not collective
4845 
4846   Input Parameters:
4847 + dm - The DM
4848 . section - The section describing the layout in v, or NULL to use the default section
4849 . v - The local vector
4850 . point - The point in the DM
4851 . values - The array of values
4852 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4853          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4854 
4855   Fortran Notes:
4856   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4857 
4858   Level: intermediate
4859 
4860 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4861 @*/
4862 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4863 {
4864   PetscSection    clSection;
4865   IS              clPoints;
4866   PetscScalar    *array;
4867   PetscInt       *points = NULL;
4868   const PetscInt *clp, *clperm;
4869   PetscInt        depth, numFields, numPoints, p;
4870   PetscErrorCode  ierr;
4871 
4872   PetscFunctionBeginHot;
4873   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4874   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4875   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4876   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4877   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4878   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4879   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4880     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4881     PetscFunctionReturn(0);
4882   }
4883   /* Get points */
4884   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4885   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4886   /* Get array */
4887   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4888   /* Get values */
4889   if (numFields > 0) {
4890     PetscInt offset = 0, f;
4891     for (f = 0; f < numFields; ++f) {
4892       const PetscInt    **perms = NULL;
4893       const PetscScalar **flips = NULL;
4894 
4895       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4896       switch (mode) {
4897       case INSERT_VALUES:
4898         for (p = 0; p < numPoints; p++) {
4899           const PetscInt    point = points[2*p];
4900           const PetscInt    *perm = perms ? perms[p] : NULL;
4901           const PetscScalar *flip = flips ? flips[p] : NULL;
4902           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4903         } break;
4904       case INSERT_ALL_VALUES:
4905         for (p = 0; p < numPoints; p++) {
4906           const PetscInt    point = points[2*p];
4907           const PetscInt    *perm = perms ? perms[p] : NULL;
4908           const PetscScalar *flip = flips ? flips[p] : NULL;
4909           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4910         } break;
4911       case INSERT_BC_VALUES:
4912         for (p = 0; p < numPoints; p++) {
4913           const PetscInt    point = points[2*p];
4914           const PetscInt    *perm = perms ? perms[p] : NULL;
4915           const PetscScalar *flip = flips ? flips[p] : NULL;
4916           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4917         } break;
4918       case ADD_VALUES:
4919         for (p = 0; p < numPoints; p++) {
4920           const PetscInt    point = points[2*p];
4921           const PetscInt    *perm = perms ? perms[p] : NULL;
4922           const PetscScalar *flip = flips ? flips[p] : NULL;
4923           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4924         } break;
4925       case ADD_ALL_VALUES:
4926         for (p = 0; p < numPoints; p++) {
4927           const PetscInt    point = points[2*p];
4928           const PetscInt    *perm = perms ? perms[p] : NULL;
4929           const PetscScalar *flip = flips ? flips[p] : NULL;
4930           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4931         } break;
4932       case ADD_BC_VALUES:
4933         for (p = 0; p < numPoints; p++) {
4934           const PetscInt    point = points[2*p];
4935           const PetscInt    *perm = perms ? perms[p] : NULL;
4936           const PetscScalar *flip = flips ? flips[p] : NULL;
4937           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4938         } break;
4939       default:
4940         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4941       }
4942       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4943     }
4944   } else {
4945     PetscInt dof, off;
4946     const PetscInt    **perms = NULL;
4947     const PetscScalar **flips = NULL;
4948 
4949     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4950     switch (mode) {
4951     case INSERT_VALUES:
4952       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4953         const PetscInt    point = points[2*p];
4954         const PetscInt    *perm = perms ? perms[p] : NULL;
4955         const PetscScalar *flip = flips ? flips[p] : NULL;
4956         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4957         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4958       } break;
4959     case INSERT_ALL_VALUES:
4960       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4961         const PetscInt    point = points[2*p];
4962         const PetscInt    *perm = perms ? perms[p] : NULL;
4963         const PetscScalar *flip = flips ? flips[p] : NULL;
4964         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4965         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4966       } break;
4967     case INSERT_BC_VALUES:
4968       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4969         const PetscInt    point = points[2*p];
4970         const PetscInt    *perm = perms ? perms[p] : NULL;
4971         const PetscScalar *flip = flips ? flips[p] : NULL;
4972         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4973         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4974       } break;
4975     case ADD_VALUES:
4976       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4977         const PetscInt    point = points[2*p];
4978         const PetscInt    *perm = perms ? perms[p] : NULL;
4979         const PetscScalar *flip = flips ? flips[p] : NULL;
4980         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4981         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4982       } break;
4983     case ADD_ALL_VALUES:
4984       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4985         const PetscInt    point = points[2*p];
4986         const PetscInt    *perm = perms ? perms[p] : NULL;
4987         const PetscScalar *flip = flips ? flips[p] : NULL;
4988         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4989         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4990       } break;
4991     case ADD_BC_VALUES:
4992       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4993         const PetscInt    point = points[2*p];
4994         const PetscInt    *perm = perms ? perms[p] : NULL;
4995         const PetscScalar *flip = flips ? flips[p] : NULL;
4996         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4997         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4998       } break;
4999     default:
5000       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5001     }
5002     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5003   }
5004   /* Cleanup points */
5005   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5006   /* Cleanup array */
5007   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5008   PetscFunctionReturn(0);
5009 }
5010 
5011 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
5012 {
5013   PetscSection      clSection;
5014   IS                clPoints;
5015   PetscScalar       *array;
5016   PetscInt          *points = NULL;
5017   const PetscInt    *clp, *clperm;
5018   PetscInt          numFields, numPoints, p;
5019   PetscInt          offset = 0, f;
5020   PetscErrorCode    ierr;
5021 
5022   PetscFunctionBeginHot;
5023   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5024   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5025   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5026   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5027   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5028   /* Get points */
5029   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5030   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5031   /* Get array */
5032   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5033   /* Get values */
5034   for (f = 0; f < numFields; ++f) {
5035     const PetscInt    **perms = NULL;
5036     const PetscScalar **flips = NULL;
5037 
5038     if (!fieldActive[f]) {
5039       for (p = 0; p < numPoints*2; p += 2) {
5040         PetscInt fdof;
5041         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5042         offset += fdof;
5043       }
5044       continue;
5045     }
5046     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5047     switch (mode) {
5048     case INSERT_VALUES:
5049       for (p = 0; p < numPoints; p++) {
5050         const PetscInt    point = points[2*p];
5051         const PetscInt    *perm = perms ? perms[p] : NULL;
5052         const PetscScalar *flip = flips ? flips[p] : NULL;
5053         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5054       } break;
5055     case INSERT_ALL_VALUES:
5056       for (p = 0; p < numPoints; p++) {
5057         const PetscInt    point = points[2*p];
5058         const PetscInt    *perm = perms ? perms[p] : NULL;
5059         const PetscScalar *flip = flips ? flips[p] : NULL;
5060         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5061         } break;
5062     case INSERT_BC_VALUES:
5063       for (p = 0; p < numPoints; p++) {
5064         const PetscInt    point = points[2*p];
5065         const PetscInt    *perm = perms ? perms[p] : NULL;
5066         const PetscScalar *flip = flips ? flips[p] : NULL;
5067         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
5068       } break;
5069     case ADD_VALUES:
5070       for (p = 0; p < numPoints; p++) {
5071         const PetscInt    point = points[2*p];
5072         const PetscInt    *perm = perms ? perms[p] : NULL;
5073         const PetscScalar *flip = flips ? flips[p] : NULL;
5074         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5075       } break;
5076     case ADD_ALL_VALUES:
5077       for (p = 0; p < numPoints; p++) {
5078         const PetscInt    point = points[2*p];
5079         const PetscInt    *perm = perms ? perms[p] : NULL;
5080         const PetscScalar *flip = flips ? flips[p] : NULL;
5081         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5082       } break;
5083     default:
5084       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5085     }
5086     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5087   }
5088   /* Cleanup points */
5089   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5090   /* Cleanup array */
5091   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5092   PetscFunctionReturn(0);
5093 }
5094 
5095 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5096 {
5097   PetscMPIInt    rank;
5098   PetscInt       i, j;
5099   PetscErrorCode ierr;
5100 
5101   PetscFunctionBegin;
5102   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5103   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5104   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5105   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5106   numCIndices = numCIndices ? numCIndices : numRIndices;
5107   for (i = 0; i < numRIndices; i++) {
5108     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5109     for (j = 0; j < numCIndices; j++) {
5110 #if defined(PETSC_USE_COMPLEX)
5111       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5112 #else
5113       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5114 #endif
5115     }
5116     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5117   }
5118   PetscFunctionReturn(0);
5119 }
5120 
5121 /*
5122   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5123 
5124   Input Parameters:
5125 + section - The section for this data layout
5126 . point   - The point contributing dofs with these indices
5127 . off     - The global offset of this point
5128 . loff    - The local offset of each field
5129 . setBC   - The flag determining whether to include indices of bounsary values
5130 . perm    - A permutation of the dofs on this point, or NULL
5131 - indperm - A permutation of the entire indices array, or NULL
5132 
5133   Output Parameter:
5134 . indices - Indices for dofs on this point
5135 
5136   Level: developer
5137 
5138   Note: The indices could be local or global, depending on the value of 'off'.
5139 */
5140 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5141 {
5142   PetscInt        dof;   /* The number of unknowns on this point */
5143   PetscInt        cdof;  /* The number of constraints on this point */
5144   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5145   PetscInt        cind = 0, k;
5146   PetscErrorCode  ierr;
5147 
5148   PetscFunctionBegin;
5149   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5150   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5151   if (!cdof || setBC) {
5152     for (k = 0; k < dof; ++k) {
5153       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5154       const PetscInt ind    = indperm ? indperm[preind] : preind;
5155 
5156       indices[ind] = off + k;
5157     }
5158   } else {
5159     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5160     for (k = 0; k < dof; ++k) {
5161       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5162       const PetscInt ind    = indperm ? indperm[preind] : preind;
5163 
5164       if ((cind < cdof) && (k == cdofs[cind])) {
5165         /* Insert check for returning constrained indices */
5166         indices[ind] = -(off+k+1);
5167         ++cind;
5168       } else {
5169         indices[ind] = off+k-cind;
5170       }
5171     }
5172   }
5173   *loff += dof;
5174   PetscFunctionReturn(0);
5175 }
5176 
5177 /*
5178   This version only believes the point offset from the globalSection
5179 
5180  . off - The global offset of this point
5181 */
5182 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5183 {
5184   PetscInt       numFields, foff, f;
5185   PetscErrorCode ierr;
5186 
5187   PetscFunctionBegin;
5188   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5189   for (f = 0, foff = 0; f < numFields; ++f) {
5190     PetscInt        fdof, cfdof;
5191     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5192     PetscInt        cind = 0, b;
5193     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5194 
5195     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5196     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5197     if (!cfdof || setBC) {
5198       for (b = 0; b < fdof; ++b) {
5199         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5200         const PetscInt ind    = indperm ? indperm[preind] : preind;
5201 
5202         indices[ind] = off+foff+b;
5203       }
5204     } else {
5205       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5206       for (b = 0; b < fdof; ++b) {
5207         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5208         const PetscInt ind    = indperm ? indperm[preind] : preind;
5209 
5210         if ((cind < cfdof) && (b == fcdofs[cind])) {
5211           indices[ind] = -(off+foff+b+1);
5212           ++cind;
5213         } else {
5214           indices[ind] = off+foff+b-cind;
5215         }
5216       }
5217     }
5218     foff     += (setBC ? fdof : (fdof - cfdof));
5219     foffs[f] += fdof;
5220   }
5221   PetscFunctionReturn(0);
5222 }
5223 
5224 /*
5225   This version believes the globalSection offsets for each field, rather than just the point offset
5226 
5227  . foffs - The offset into 'indices' for each field, since it is segregated by field
5228 */
5229 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5230 {
5231   PetscInt       numFields, foff, f;
5232   PetscErrorCode ierr;
5233 
5234   PetscFunctionBegin;
5235   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5236   for (f = 0; f < numFields; ++f) {
5237     PetscInt        fdof, cfdof;
5238     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5239     PetscInt        cind = 0, b;
5240     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5241 
5242     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5243     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5244     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5245     if (!cfdof || setBC) {
5246       for (b = 0; b < fdof; ++b) {
5247         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5248         const PetscInt ind    = indperm ? indperm[preind] : preind;
5249 
5250         indices[ind] = foff+b;
5251       }
5252     } else {
5253       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5254       for (b = 0; b < fdof; ++b) {
5255         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5256         const PetscInt ind    = indperm ? indperm[preind] : preind;
5257 
5258         if ((cind < cfdof) && (b == fcdofs[cind])) {
5259           indices[ind] = -(foff+b+1);
5260           ++cind;
5261         } else {
5262           indices[ind] = foff+b-cind;
5263         }
5264       }
5265     }
5266     foffs[f] += fdof;
5267   }
5268   PetscFunctionReturn(0);
5269 }
5270 
5271 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5272 {
5273   Mat             cMat;
5274   PetscSection    aSec, cSec;
5275   IS              aIS;
5276   PetscInt        aStart = -1, aEnd = -1;
5277   const PetscInt  *anchors;
5278   PetscInt        numFields, f, p, q, newP = 0;
5279   PetscInt        newNumPoints = 0, newNumIndices = 0;
5280   PetscInt        *newPoints, *indices, *newIndices;
5281   PetscInt        maxAnchor, maxDof;
5282   PetscInt        newOffsets[32];
5283   PetscInt        *pointMatOffsets[32];
5284   PetscInt        *newPointOffsets[32];
5285   PetscScalar     *pointMat[32];
5286   PetscScalar     *newValues=NULL,*tmpValues;
5287   PetscBool       anyConstrained = PETSC_FALSE;
5288   PetscErrorCode  ierr;
5289 
5290   PetscFunctionBegin;
5291   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5292   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5293   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5294 
5295   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5296   /* if there are point-to-point constraints */
5297   if (aSec) {
5298     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5299     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5300     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5301     /* figure out how many points are going to be in the new element matrix
5302      * (we allow double counting, because it's all just going to be summed
5303      * into the global matrix anyway) */
5304     for (p = 0; p < 2*numPoints; p+=2) {
5305       PetscInt b    = points[p];
5306       PetscInt bDof = 0, bSecDof;
5307 
5308       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5309       if (!bSecDof) {
5310         continue;
5311       }
5312       if (b >= aStart && b < aEnd) {
5313         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5314       }
5315       if (bDof) {
5316         /* this point is constrained */
5317         /* it is going to be replaced by its anchors */
5318         PetscInt bOff, q;
5319 
5320         anyConstrained = PETSC_TRUE;
5321         newNumPoints  += bDof;
5322         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5323         for (q = 0; q < bDof; q++) {
5324           PetscInt a = anchors[bOff + q];
5325           PetscInt aDof;
5326 
5327           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5328           newNumIndices += aDof;
5329           for (f = 0; f < numFields; ++f) {
5330             PetscInt fDof;
5331 
5332             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5333             newOffsets[f+1] += fDof;
5334           }
5335         }
5336       }
5337       else {
5338         /* this point is not constrained */
5339         newNumPoints++;
5340         newNumIndices += bSecDof;
5341         for (f = 0; f < numFields; ++f) {
5342           PetscInt fDof;
5343 
5344           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5345           newOffsets[f+1] += fDof;
5346         }
5347       }
5348     }
5349   }
5350   if (!anyConstrained) {
5351     if (outNumPoints)  *outNumPoints  = 0;
5352     if (outNumIndices) *outNumIndices = 0;
5353     if (outPoints)     *outPoints     = NULL;
5354     if (outValues)     *outValues     = NULL;
5355     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5356     PetscFunctionReturn(0);
5357   }
5358 
5359   if (outNumPoints)  *outNumPoints  = newNumPoints;
5360   if (outNumIndices) *outNumIndices = newNumIndices;
5361 
5362   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5363 
5364   if (!outPoints && !outValues) {
5365     if (offsets) {
5366       for (f = 0; f <= numFields; f++) {
5367         offsets[f] = newOffsets[f];
5368       }
5369     }
5370     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5371     PetscFunctionReturn(0);
5372   }
5373 
5374   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5375 
5376   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5377 
5378   /* workspaces */
5379   if (numFields) {
5380     for (f = 0; f < numFields; f++) {
5381       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5382       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5383     }
5384   }
5385   else {
5386     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5387     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5388   }
5389 
5390   /* get workspaces for the point-to-point matrices */
5391   if (numFields) {
5392     PetscInt totalOffset, totalMatOffset;
5393 
5394     for (p = 0; p < numPoints; p++) {
5395       PetscInt b    = points[2*p];
5396       PetscInt bDof = 0, bSecDof;
5397 
5398       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5399       if (!bSecDof) {
5400         for (f = 0; f < numFields; f++) {
5401           newPointOffsets[f][p + 1] = 0;
5402           pointMatOffsets[f][p + 1] = 0;
5403         }
5404         continue;
5405       }
5406       if (b >= aStart && b < aEnd) {
5407         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5408       }
5409       if (bDof) {
5410         for (f = 0; f < numFields; f++) {
5411           PetscInt fDof, q, bOff, allFDof = 0;
5412 
5413           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5414           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5415           for (q = 0; q < bDof; q++) {
5416             PetscInt a = anchors[bOff + q];
5417             PetscInt aFDof;
5418 
5419             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5420             allFDof += aFDof;
5421           }
5422           newPointOffsets[f][p+1] = allFDof;
5423           pointMatOffsets[f][p+1] = fDof * allFDof;
5424         }
5425       }
5426       else {
5427         for (f = 0; f < numFields; f++) {
5428           PetscInt fDof;
5429 
5430           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5431           newPointOffsets[f][p+1] = fDof;
5432           pointMatOffsets[f][p+1] = 0;
5433         }
5434       }
5435     }
5436     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5437       newPointOffsets[f][0] = totalOffset;
5438       pointMatOffsets[f][0] = totalMatOffset;
5439       for (p = 0; p < numPoints; p++) {
5440         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5441         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5442       }
5443       totalOffset    = newPointOffsets[f][numPoints];
5444       totalMatOffset = pointMatOffsets[f][numPoints];
5445       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5446     }
5447   }
5448   else {
5449     for (p = 0; p < numPoints; p++) {
5450       PetscInt b    = points[2*p];
5451       PetscInt bDof = 0, bSecDof;
5452 
5453       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5454       if (!bSecDof) {
5455         newPointOffsets[0][p + 1] = 0;
5456         pointMatOffsets[0][p + 1] = 0;
5457         continue;
5458       }
5459       if (b >= aStart && b < aEnd) {
5460         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5461       }
5462       if (bDof) {
5463         PetscInt bOff, q, allDof = 0;
5464 
5465         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5466         for (q = 0; q < bDof; q++) {
5467           PetscInt a = anchors[bOff + q], aDof;
5468 
5469           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5470           allDof += aDof;
5471         }
5472         newPointOffsets[0][p+1] = allDof;
5473         pointMatOffsets[0][p+1] = bSecDof * allDof;
5474       }
5475       else {
5476         newPointOffsets[0][p+1] = bSecDof;
5477         pointMatOffsets[0][p+1] = 0;
5478       }
5479     }
5480     newPointOffsets[0][0] = 0;
5481     pointMatOffsets[0][0] = 0;
5482     for (p = 0; p < numPoints; p++) {
5483       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5484       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5485     }
5486     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5487   }
5488 
5489   /* output arrays */
5490   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5491 
5492   /* get the point-to-point matrices; construct newPoints */
5493   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5494   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5495   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5496   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5497   if (numFields) {
5498     for (p = 0, newP = 0; p < numPoints; p++) {
5499       PetscInt b    = points[2*p];
5500       PetscInt o    = points[2*p+1];
5501       PetscInt bDof = 0, bSecDof;
5502 
5503       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5504       if (!bSecDof) {
5505         continue;
5506       }
5507       if (b >= aStart && b < aEnd) {
5508         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5509       }
5510       if (bDof) {
5511         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5512 
5513         fStart[0] = 0;
5514         fEnd[0]   = 0;
5515         for (f = 0; f < numFields; f++) {
5516           PetscInt fDof;
5517 
5518           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5519           fStart[f+1] = fStart[f] + fDof;
5520           fEnd[f+1]   = fStart[f+1];
5521         }
5522         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5523         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5524 
5525         fAnchorStart[0] = 0;
5526         fAnchorEnd[0]   = 0;
5527         for (f = 0; f < numFields; f++) {
5528           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5529 
5530           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5531           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5532         }
5533         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5534         for (q = 0; q < bDof; q++) {
5535           PetscInt a = anchors[bOff + q], aOff;
5536 
5537           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5538           newPoints[2*(newP + q)]     = a;
5539           newPoints[2*(newP + q) + 1] = 0;
5540           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5541           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5542         }
5543         newP += bDof;
5544 
5545         if (outValues) {
5546           /* get the point-to-point submatrix */
5547           for (f = 0; f < numFields; f++) {
5548             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5549           }
5550         }
5551       }
5552       else {
5553         newPoints[2 * newP]     = b;
5554         newPoints[2 * newP + 1] = o;
5555         newP++;
5556       }
5557     }
5558   } else {
5559     for (p = 0; p < numPoints; p++) {
5560       PetscInt b    = points[2*p];
5561       PetscInt o    = points[2*p+1];
5562       PetscInt bDof = 0, bSecDof;
5563 
5564       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5565       if (!bSecDof) {
5566         continue;
5567       }
5568       if (b >= aStart && b < aEnd) {
5569         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5570       }
5571       if (bDof) {
5572         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5573 
5574         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5575         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5576 
5577         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5578         for (q = 0; q < bDof; q++) {
5579           PetscInt a = anchors[bOff + q], aOff;
5580 
5581           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5582 
5583           newPoints[2*(newP + q)]     = a;
5584           newPoints[2*(newP + q) + 1] = 0;
5585           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5586           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5587         }
5588         newP += bDof;
5589 
5590         /* get the point-to-point submatrix */
5591         if (outValues) {
5592           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5593         }
5594       }
5595       else {
5596         newPoints[2 * newP]     = b;
5597         newPoints[2 * newP + 1] = o;
5598         newP++;
5599       }
5600     }
5601   }
5602 
5603   if (outValues) {
5604     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5605     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5606     /* multiply constraints on the right */
5607     if (numFields) {
5608       for (f = 0; f < numFields; f++) {
5609         PetscInt oldOff = offsets[f];
5610 
5611         for (p = 0; p < numPoints; p++) {
5612           PetscInt cStart = newPointOffsets[f][p];
5613           PetscInt b      = points[2 * p];
5614           PetscInt c, r, k;
5615           PetscInt dof;
5616 
5617           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5618           if (!dof) {
5619             continue;
5620           }
5621           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5622             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5623             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5624 
5625             for (r = 0; r < numIndices; r++) {
5626               for (c = 0; c < nCols; c++) {
5627                 for (k = 0; k < dof; k++) {
5628                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5629                 }
5630               }
5631             }
5632           }
5633           else {
5634             /* copy this column as is */
5635             for (r = 0; r < numIndices; r++) {
5636               for (c = 0; c < dof; c++) {
5637                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5638               }
5639             }
5640           }
5641           oldOff += dof;
5642         }
5643       }
5644     }
5645     else {
5646       PetscInt oldOff = 0;
5647       for (p = 0; p < numPoints; p++) {
5648         PetscInt cStart = newPointOffsets[0][p];
5649         PetscInt b      = points[2 * p];
5650         PetscInt c, r, k;
5651         PetscInt dof;
5652 
5653         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5654         if (!dof) {
5655           continue;
5656         }
5657         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5658           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5659           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5660 
5661           for (r = 0; r < numIndices; r++) {
5662             for (c = 0; c < nCols; c++) {
5663               for (k = 0; k < dof; k++) {
5664                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5665               }
5666             }
5667           }
5668         }
5669         else {
5670           /* copy this column as is */
5671           for (r = 0; r < numIndices; r++) {
5672             for (c = 0; c < dof; c++) {
5673               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5674             }
5675           }
5676         }
5677         oldOff += dof;
5678       }
5679     }
5680 
5681     if (multiplyLeft) {
5682       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5683       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5684       /* multiply constraints transpose on the left */
5685       if (numFields) {
5686         for (f = 0; f < numFields; f++) {
5687           PetscInt oldOff = offsets[f];
5688 
5689           for (p = 0; p < numPoints; p++) {
5690             PetscInt rStart = newPointOffsets[f][p];
5691             PetscInt b      = points[2 * p];
5692             PetscInt c, r, k;
5693             PetscInt dof;
5694 
5695             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5696             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5697               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5698               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5699 
5700               for (r = 0; r < nRows; r++) {
5701                 for (c = 0; c < newNumIndices; c++) {
5702                   for (k = 0; k < dof; k++) {
5703                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5704                   }
5705                 }
5706               }
5707             }
5708             else {
5709               /* copy this row as is */
5710               for (r = 0; r < dof; r++) {
5711                 for (c = 0; c < newNumIndices; c++) {
5712                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5713                 }
5714               }
5715             }
5716             oldOff += dof;
5717           }
5718         }
5719       }
5720       else {
5721         PetscInt oldOff = 0;
5722 
5723         for (p = 0; p < numPoints; p++) {
5724           PetscInt rStart = newPointOffsets[0][p];
5725           PetscInt b      = points[2 * p];
5726           PetscInt c, r, k;
5727           PetscInt dof;
5728 
5729           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5730           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5731             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5732             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5733 
5734             for (r = 0; r < nRows; r++) {
5735               for (c = 0; c < newNumIndices; c++) {
5736                 for (k = 0; k < dof; k++) {
5737                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5738                 }
5739               }
5740             }
5741           }
5742           else {
5743             /* copy this row as is */
5744             for (r = 0; r < dof; r++) {
5745               for (c = 0; c < newNumIndices; c++) {
5746                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5747               }
5748             }
5749           }
5750           oldOff += dof;
5751         }
5752       }
5753 
5754       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5755     }
5756     else {
5757       newValues = tmpValues;
5758     }
5759   }
5760 
5761   /* clean up */
5762   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5763   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5764 
5765   if (numFields) {
5766     for (f = 0; f < numFields; f++) {
5767       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5768       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5769       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5770     }
5771   }
5772   else {
5773     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5774     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5775     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5776   }
5777   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5778 
5779   /* output */
5780   if (outPoints) {
5781     *outPoints = newPoints;
5782   }
5783   else {
5784     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5785   }
5786   if (outValues) {
5787     *outValues = newValues;
5788   }
5789   for (f = 0; f <= numFields; f++) {
5790     offsets[f] = newOffsets[f];
5791   }
5792   PetscFunctionReturn(0);
5793 }
5794 
5795 /*@C
5796   DMPlexGetClosureIndices - Get the global indices for all local points in the closure of the given point
5797 
5798   Not collective
5799 
5800   Input Parameters:
5801 + dm - The DM
5802 . section - The section describing the local layout
5803 . globalSection - The section describing the parallel layout
5804 - point - The mesh point
5805 
5806   Output parameters:
5807 + numIndices - The number of indices
5808 . indices - The indices
5809 - outOffsets - Field offset if not NULL
5810 
5811   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5812 
5813   Level: advanced
5814 
5815 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5816 @*/
5817 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5818 {
5819   PetscSection    clSection;
5820   IS              clPoints;
5821   const PetscInt *clp, *clperm;
5822   const PetscInt  **perms[32] = {NULL};
5823   PetscInt       *points = NULL, *pointsNew;
5824   PetscInt        numPoints, numPointsNew;
5825   PetscInt        offsets[32];
5826   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5827   PetscErrorCode  ierr;
5828 
5829   PetscFunctionBegin;
5830   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5831   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5832   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5833   if (numIndices) PetscValidPointer(numIndices, 4);
5834   PetscValidPointer(indices, 5);
5835   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5836   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5837   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5838   /* Get points in closure */
5839   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5840   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5841   /* Get number of indices and indices per field */
5842   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5843     PetscInt dof, fdof;
5844 
5845     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5846     for (f = 0; f < Nf; ++f) {
5847       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5848       offsets[f+1] += fdof;
5849     }
5850     Nind += dof;
5851   }
5852   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5853   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5854   if (!Nf) offsets[1] = Nind;
5855   /* Get dual space symmetries */
5856   for (f = 0; f < PetscMax(1,Nf); f++) {
5857     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5858     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5859   }
5860   /* Correct for hanging node constraints */
5861   {
5862     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5863     if (numPointsNew) {
5864       for (f = 0; f < PetscMax(1,Nf); f++) {
5865         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5866         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5867       }
5868       for (f = 0; f < PetscMax(1,Nf); f++) {
5869         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5870         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5871       }
5872       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5873       numPoints = numPointsNew;
5874       Nind      = NindNew;
5875       points    = pointsNew;
5876     }
5877   }
5878   /* Calculate indices */
5879   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5880   if (Nf) {
5881     if (outOffsets) {
5882       PetscInt f;
5883 
5884       for (f = 0; f <= Nf; f++) {
5885         outOffsets[f] = offsets[f];
5886       }
5887     }
5888     for (p = 0; p < numPoints; p++) {
5889       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5890       ierr = DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);CHKERRQ(ierr);
5891     }
5892   } else {
5893     for (p = 0, off = 0; p < numPoints; p++) {
5894       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5895 
5896       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5897       ierr = DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);CHKERRQ(ierr);
5898     }
5899   }
5900   /* Cleanup points */
5901   for (f = 0; f < PetscMax(1,Nf); f++) {
5902     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5903     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5904   }
5905   if (numPointsNew) {
5906     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5907   } else {
5908     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5909   }
5910   if (numIndices) *numIndices = Nind;
5911   PetscFunctionReturn(0);
5912 }
5913 
5914 /*@C
5915   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5916 
5917   Not collective
5918 
5919   Input Parameters:
5920 + dm - The DM
5921 . section - The section describing the layout in v, or NULL to use the default section
5922 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5923 . point - The mesh point
5924 . numIndices - The number of indices
5925 . indices - The indices
5926 - outOffsets - Field offset if not NULL
5927 
5928   Level: advanced
5929 
5930 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5931 @*/
5932 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5933 {
5934   PetscErrorCode ierr;
5935 
5936   PetscFunctionBegin;
5937   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5938   PetscValidPointer(indices, 5);
5939   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5940   PetscFunctionReturn(0);
5941 }
5942 
5943 /*@C
5944   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5945 
5946   Not collective
5947 
5948   Input Parameters:
5949 + dm - The DM
5950 . section - The section describing the layout in v, or NULL to use the default section
5951 . globalSection - The section describing the layout in v, or NULL to use the default global section
5952 . A - The matrix
5953 . point - The point in the DM
5954 . values - The array of values
5955 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5956 
5957   Fortran Notes:
5958   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5959 
5960   Level: intermediate
5961 
5962 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5963 @*/
5964 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5965 {
5966   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5967   PetscSection        clSection;
5968   IS                  clPoints;
5969   PetscInt           *points = NULL, *newPoints;
5970   const PetscInt     *clp, *clperm;
5971   PetscInt           *indices;
5972   PetscInt            offsets[32];
5973   const PetscInt    **perms[32] = {NULL};
5974   const PetscScalar **flips[32] = {NULL};
5975   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5976   PetscScalar        *valCopy = NULL;
5977   PetscScalar        *newValues;
5978   PetscErrorCode      ierr;
5979 
5980   PetscFunctionBegin;
5981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5982   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5983   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5984   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5985   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5986   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5987   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5988   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5989   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5990   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5991   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5992   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5993     PetscInt fdof;
5994 
5995     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5996     for (f = 0; f < numFields; ++f) {
5997       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5998       offsets[f+1] += fdof;
5999     }
6000     numIndices += dof;
6001   }
6002   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6003 
6004   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
6005   /* Get symmetries */
6006   for (f = 0; f < PetscMax(1,numFields); f++) {
6007     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6008     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6009     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
6010       PetscInt foffset = offsets[f];
6011 
6012       for (p = 0; p < numPoints; p++) {
6013         PetscInt point          = points[2*p], fdof;
6014         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
6015 
6016         if (!numFields) {
6017           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
6018         } else {
6019           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
6020         }
6021         if (flip) {
6022           PetscInt i, j, k;
6023 
6024           if (!valCopy) {
6025             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6026             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6027             values = valCopy;
6028           }
6029           for (i = 0; i < fdof; i++) {
6030             PetscScalar fval = flip[i];
6031 
6032             for (k = 0; k < numIndices; k++) {
6033               valCopy[numIndices * (foffset + i) + k] *= fval;
6034               valCopy[numIndices * k + (foffset + i)] *= fval;
6035             }
6036           }
6037         }
6038         foffset += fdof;
6039       }
6040     }
6041   }
6042   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6043   if (newNumPoints) {
6044     if (valCopy) {
6045       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6046     }
6047     for (f = 0; f < PetscMax(1,numFields); f++) {
6048       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6049       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6050     }
6051     for (f = 0; f < PetscMax(1,numFields); f++) {
6052       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6053       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6054     }
6055     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6056     numPoints  = newNumPoints;
6057     numIndices = newNumIndices;
6058     points     = newPoints;
6059     values     = newValues;
6060   }
6061   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6062   if (numFields) {
6063     PetscBool useFieldOffsets;
6064 
6065     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6066     if (useFieldOffsets) {
6067       for (p = 0; p < numPoints; p++) {
6068         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
6069       }
6070     } else {
6071       for (p = 0; p < numPoints; p++) {
6072         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6073         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
6074       }
6075     }
6076   } else {
6077     for (p = 0, off = 0; p < numPoints; p++) {
6078       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6079       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6080       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
6081     }
6082   }
6083   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6084   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
6085   if (mesh->printFEM > 1) {
6086     PetscInt i;
6087     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6088     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6089     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6090   }
6091   if (ierr) {
6092     PetscMPIInt    rank;
6093     PetscErrorCode ierr2;
6094 
6095     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6096     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6097     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6098     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6099     CHKERRQ(ierr);
6100   }
6101   for (f = 0; f < PetscMax(1,numFields); f++) {
6102     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6103     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6104   }
6105   if (newNumPoints) {
6106     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6107     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6108   }
6109   else {
6110     if (valCopy) {
6111       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6112     }
6113     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6114   }
6115   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6116   PetscFunctionReturn(0);
6117 }
6118 
6119 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6120 {
6121   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6122   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6123   PetscInt       *cpoints = NULL;
6124   PetscInt       *findices, *cindices;
6125   const PetscInt *fclperm, *cclperm;
6126   PetscInt        foffsets[32], coffsets[32];
6127   CellRefiner     cellRefiner;
6128   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6129   PetscErrorCode  ierr;
6130 
6131   PetscFunctionBegin;
6132   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6133   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6134   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6135   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6136   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6137   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6138   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6139   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6140   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6141   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6142   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6143   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6144   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6145   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6146   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6147   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6148   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6149   /* Column indices */
6150   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6151   maxFPoints = numCPoints;
6152   /* Compress out points not in the section */
6153   /*   TODO: Squeeze out points with 0 dof as well */
6154   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6155   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6156     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6157       cpoints[q*2]   = cpoints[p];
6158       cpoints[q*2+1] = cpoints[p+1];
6159       ++q;
6160     }
6161   }
6162   numCPoints = q;
6163   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6164     PetscInt fdof;
6165 
6166     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6167     if (!dof) continue;
6168     for (f = 0; f < numFields; ++f) {
6169       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6170       coffsets[f+1] += fdof;
6171     }
6172     numCIndices += dof;
6173   }
6174   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6175   /* Row indices */
6176   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6177   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6178   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6179   for (r = 0, q = 0; r < numSubcells; ++r) {
6180     /* TODO Map from coarse to fine cells */
6181     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6182     /* Compress out points not in the section */
6183     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6184     for (p = 0; p < numFPoints*2; p += 2) {
6185       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6186         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6187         if (!dof) continue;
6188         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6189         if (s < q) continue;
6190         ftotpoints[q*2]   = fpoints[p];
6191         ftotpoints[q*2+1] = fpoints[p+1];
6192         ++q;
6193       }
6194     }
6195     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6196   }
6197   numFPoints = q;
6198   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6199     PetscInt fdof;
6200 
6201     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6202     if (!dof) continue;
6203     for (f = 0; f < numFields; ++f) {
6204       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6205       foffsets[f+1] += fdof;
6206     }
6207     numFIndices += dof;
6208   }
6209   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6210 
6211   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6212   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6213   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6214   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6215   if (numFields) {
6216     const PetscInt **permsF[32] = {NULL};
6217     const PetscInt **permsC[32] = {NULL};
6218 
6219     for (f = 0; f < numFields; f++) {
6220       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6221       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6222     }
6223     for (p = 0; p < numFPoints; p++) {
6224       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6225       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6226     }
6227     for (p = 0; p < numCPoints; p++) {
6228       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6229       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6230     }
6231     for (f = 0; f < numFields; f++) {
6232       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6233       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6234     }
6235   } else {
6236     const PetscInt **permsF = NULL;
6237     const PetscInt **permsC = NULL;
6238 
6239     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6240     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6241     for (p = 0, off = 0; p < numFPoints; p++) {
6242       const PetscInt *perm = permsF ? permsF[p] : NULL;
6243 
6244       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6245       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6246     }
6247     for (p = 0, off = 0; p < numCPoints; p++) {
6248       const PetscInt *perm = permsC ? permsC[p] : NULL;
6249 
6250       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6251       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6252     }
6253     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6254     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6255   }
6256   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6257   /* TODO: flips */
6258   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6259   if (ierr) {
6260     PetscMPIInt    rank;
6261     PetscErrorCode ierr2;
6262 
6263     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6264     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6265     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6266     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6267     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6268     CHKERRQ(ierr);
6269   }
6270   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6271   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6272   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6273   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6274   PetscFunctionReturn(0);
6275 }
6276 
6277 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6278 {
6279   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6280   PetscInt      *cpoints = NULL;
6281   PetscInt       foffsets[32], coffsets[32];
6282   const PetscInt *fclperm, *cclperm;
6283   CellRefiner    cellRefiner;
6284   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6285   PetscErrorCode ierr;
6286 
6287   PetscFunctionBegin;
6288   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6289   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6290   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6291   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6292   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6293   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6294   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6295   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6296   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6297   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6298   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6299   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6300   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6301   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6302   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6303   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6304   /* Column indices */
6305   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6306   maxFPoints = numCPoints;
6307   /* Compress out points not in the section */
6308   /*   TODO: Squeeze out points with 0 dof as well */
6309   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6310   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6311     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6312       cpoints[q*2]   = cpoints[p];
6313       cpoints[q*2+1] = cpoints[p+1];
6314       ++q;
6315     }
6316   }
6317   numCPoints = q;
6318   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6319     PetscInt fdof;
6320 
6321     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6322     if (!dof) continue;
6323     for (f = 0; f < numFields; ++f) {
6324       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6325       coffsets[f+1] += fdof;
6326     }
6327     numCIndices += dof;
6328   }
6329   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6330   /* Row indices */
6331   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6332   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6333   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6334   for (r = 0, q = 0; r < numSubcells; ++r) {
6335     /* TODO Map from coarse to fine cells */
6336     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6337     /* Compress out points not in the section */
6338     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6339     for (p = 0; p < numFPoints*2; p += 2) {
6340       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6341         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6342         if (!dof) continue;
6343         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6344         if (s < q) continue;
6345         ftotpoints[q*2]   = fpoints[p];
6346         ftotpoints[q*2+1] = fpoints[p+1];
6347         ++q;
6348       }
6349     }
6350     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6351   }
6352   numFPoints = q;
6353   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6354     PetscInt fdof;
6355 
6356     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6357     if (!dof) continue;
6358     for (f = 0; f < numFields; ++f) {
6359       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6360       foffsets[f+1] += fdof;
6361     }
6362     numFIndices += dof;
6363   }
6364   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6365 
6366   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6367   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6368   if (numFields) {
6369     const PetscInt **permsF[32] = {NULL};
6370     const PetscInt **permsC[32] = {NULL};
6371 
6372     for (f = 0; f < numFields; f++) {
6373       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6374       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6375     }
6376     for (p = 0; p < numFPoints; p++) {
6377       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6378       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6379     }
6380     for (p = 0; p < numCPoints; p++) {
6381       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6382       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6383     }
6384     for (f = 0; f < numFields; f++) {
6385       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6386       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6387     }
6388   } else {
6389     const PetscInt **permsF = NULL;
6390     const PetscInt **permsC = NULL;
6391 
6392     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6393     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6394     for (p = 0, off = 0; p < numFPoints; p++) {
6395       const PetscInt *perm = permsF ? permsF[p] : NULL;
6396 
6397       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6398       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6399     }
6400     for (p = 0, off = 0; p < numCPoints; p++) {
6401       const PetscInt *perm = permsC ? permsC[p] : NULL;
6402 
6403       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6404       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6405     }
6406     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6407     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6408   }
6409   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6410   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6411   PetscFunctionReturn(0);
6412 }
6413 
6414 /*@
6415   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6416 
6417   Input Parameter:
6418 . dm - The DMPlex object
6419 
6420   Output Parameters:
6421 + cMax - The first hybrid cell
6422 . fMax - The first hybrid face
6423 . eMax - The first hybrid edge
6424 - vMax - The first hybrid vertex
6425 
6426   Level: developer
6427 
6428 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6429 @*/
6430 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6431 {
6432   DM_Plex       *mesh = (DM_Plex*) dm->data;
6433   PetscInt       dim;
6434   PetscErrorCode ierr;
6435 
6436   PetscFunctionBegin;
6437   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6438   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6439   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6440   if (cMax) *cMax = mesh->hybridPointMax[dim];
6441   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6442   if (eMax) *eMax = mesh->hybridPointMax[1];
6443   if (vMax) *vMax = mesh->hybridPointMax[0];
6444   PetscFunctionReturn(0);
6445 }
6446 
6447 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6448 {
6449   IS             is, his;
6450   PetscInt       first = 0, stride;
6451   PetscBool      isStride;
6452   PetscErrorCode ierr;
6453 
6454   PetscFunctionBegin;
6455   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6456   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6457   if (isStride) {
6458     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6459   }
6460   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6461   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6462   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6463   ierr = ISDestroy(&his);CHKERRQ(ierr);
6464   ierr = ISDestroy(&is);CHKERRQ(ierr);
6465   PetscFunctionReturn(0);
6466 }
6467 
6468 /*@
6469   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6470 
6471   Input Parameters:
6472 + dm   - The DMPlex object
6473 . cMax - The first hybrid cell
6474 . fMax - The first hybrid face
6475 . eMax - The first hybrid edge
6476 - vMax - The first hybrid vertex
6477 
6478   Level: developer
6479 
6480 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6481 @*/
6482 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6483 {
6484   DM_Plex       *mesh = (DM_Plex*) dm->data;
6485   PetscInt       dim;
6486   PetscErrorCode ierr;
6487 
6488   PetscFunctionBegin;
6489   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6490   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6491   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6492   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6493   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6494   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6495   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6496   PetscFunctionReturn(0);
6497 }
6498 
6499 /*@C
6500   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6501 
6502   Input Parameter:
6503 . dm   - The DMPlex object
6504 
6505   Output Parameter:
6506 . cellHeight - The height of a cell
6507 
6508   Level: developer
6509 
6510 .seealso DMPlexSetVTKCellHeight()
6511 @*/
6512 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6513 {
6514   DM_Plex *mesh = (DM_Plex*) dm->data;
6515 
6516   PetscFunctionBegin;
6517   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6518   PetscValidPointer(cellHeight, 2);
6519   *cellHeight = mesh->vtkCellHeight;
6520   PetscFunctionReturn(0);
6521 }
6522 
6523 /*@C
6524   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6525 
6526   Input Parameters:
6527 + dm   - The DMPlex object
6528 - cellHeight - The height of a cell
6529 
6530   Level: developer
6531 
6532 .seealso DMPlexGetVTKCellHeight()
6533 @*/
6534 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6535 {
6536   DM_Plex *mesh = (DM_Plex*) dm->data;
6537 
6538   PetscFunctionBegin;
6539   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6540   mesh->vtkCellHeight = cellHeight;
6541   PetscFunctionReturn(0);
6542 }
6543 
6544 /* We can easily have a form that takes an IS instead */
6545 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6546 {
6547   PetscSection   section, globalSection;
6548   PetscInt      *numbers, p;
6549   PetscErrorCode ierr;
6550 
6551   PetscFunctionBegin;
6552   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6553   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6554   for (p = pStart; p < pEnd; ++p) {
6555     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6556   }
6557   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6558   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6559   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6560   for (p = pStart; p < pEnd; ++p) {
6561     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6562     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6563     else                       numbers[p-pStart] += shift;
6564   }
6565   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6566   if (globalSize) {
6567     PetscLayout layout;
6568     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6569     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6570     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6571   }
6572   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6573   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6574   PetscFunctionReturn(0);
6575 }
6576 
6577 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6578 {
6579   PetscInt       cellHeight, cStart, cEnd, cMax;
6580   PetscErrorCode ierr;
6581 
6582   PetscFunctionBegin;
6583   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6584   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6585   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6586   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6587   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6588   PetscFunctionReturn(0);
6589 }
6590 
6591 /*@
6592   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6593 
6594   Input Parameter:
6595 . dm   - The DMPlex object
6596 
6597   Output Parameter:
6598 . globalCellNumbers - Global cell numbers for all cells on this process
6599 
6600   Level: developer
6601 
6602 .seealso DMPlexGetVertexNumbering()
6603 @*/
6604 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6605 {
6606   DM_Plex       *mesh = (DM_Plex*) dm->data;
6607   PetscErrorCode ierr;
6608 
6609   PetscFunctionBegin;
6610   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6611   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6612   *globalCellNumbers = mesh->globalCellNumbers;
6613   PetscFunctionReturn(0);
6614 }
6615 
6616 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6617 {
6618   PetscInt       vStart, vEnd, vMax;
6619   PetscErrorCode ierr;
6620 
6621   PetscFunctionBegin;
6622   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6623   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6624   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6625   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6626   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6627   PetscFunctionReturn(0);
6628 }
6629 
6630 /*@
6631   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
6632 
6633   Input Parameter:
6634 . dm   - The DMPlex object
6635 
6636   Output Parameter:
6637 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6638 
6639   Level: developer
6640 
6641 .seealso DMPlexGetCellNumbering()
6642 @*/
6643 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6644 {
6645   DM_Plex       *mesh = (DM_Plex*) dm->data;
6646   PetscErrorCode ierr;
6647 
6648   PetscFunctionBegin;
6649   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6650   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6651   *globalVertexNumbers = mesh->globalVertexNumbers;
6652   PetscFunctionReturn(0);
6653 }
6654 
6655 /*@
6656   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6657 
6658   Input Parameter:
6659 . dm   - The DMPlex object
6660 
6661   Output Parameter:
6662 . globalPointNumbers - Global numbers for all points on this process
6663 
6664   Level: developer
6665 
6666 .seealso DMPlexGetCellNumbering()
6667 @*/
6668 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6669 {
6670   IS             nums[4];
6671   PetscInt       depths[4], gdepths[4], starts[4];
6672   PetscInt       depth, d, shift = 0;
6673   PetscErrorCode ierr;
6674 
6675   PetscFunctionBegin;
6676   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6677   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6678   /* For unstratified meshes use dim instead of depth */
6679   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6680   for (d = 0; d <= depth; ++d) {
6681     PetscInt end;
6682 
6683     depths[d] = depth-d;
6684     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6685     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6686   }
6687   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6688   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6689   for (d = 0; d <= depth; ++d) {
6690     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6691   }
6692   for (d = 0; d <= depth; ++d) {
6693     PetscInt pStart, pEnd, gsize;
6694 
6695     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6696     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6697     shift += gsize;
6698   }
6699   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6700   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6701   PetscFunctionReturn(0);
6702 }
6703 
6704 
6705 /*@
6706   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6707 
6708   Input Parameter:
6709 . dm - The DMPlex object
6710 
6711   Output Parameter:
6712 . ranks - The rank field
6713 
6714   Options Database Keys:
6715 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6716 
6717   Level: intermediate
6718 
6719 .seealso: DMView()
6720 @*/
6721 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6722 {
6723   DM             rdm;
6724   PetscFE        fe;
6725   PetscScalar   *r;
6726   PetscMPIInt    rank;
6727   PetscInt       dim, cStart, cEnd, c;
6728   PetscErrorCode ierr;
6729 
6730   PetscFunctionBeginUser;
6731   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6732   PetscValidPointer(ranks, 2);
6733   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6734   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6735   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6736   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6737   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6738   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6739   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6740   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6741   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6742   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6743   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6744   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6745   for (c = cStart; c < cEnd; ++c) {
6746     PetscScalar *lr;
6747 
6748     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6749     *lr = rank;
6750   }
6751   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6752   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6753   PetscFunctionReturn(0);
6754 }
6755 
6756 /*@
6757   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6758 
6759   Input Parameters:
6760 + dm    - The DMPlex
6761 - label - The DMLabel
6762 
6763   Output Parameter:
6764 . val - The label value field
6765 
6766   Options Database Keys:
6767 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6768 
6769   Level: intermediate
6770 
6771 .seealso: DMView()
6772 @*/
6773 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6774 {
6775   DM             rdm;
6776   PetscFE        fe;
6777   PetscScalar   *v;
6778   PetscInt       dim, cStart, cEnd, c;
6779   PetscErrorCode ierr;
6780 
6781   PetscFunctionBeginUser;
6782   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6783   PetscValidPointer(label, 2);
6784   PetscValidPointer(val, 3);
6785   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6786   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6787   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6788   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6789   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6790   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6791   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6792   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6793   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6794   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6795   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6796   for (c = cStart; c < cEnd; ++c) {
6797     PetscScalar *lv;
6798     PetscInt     cval;
6799 
6800     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6801     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6802     *lv = cval;
6803   }
6804   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6805   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6806   PetscFunctionReturn(0);
6807 }
6808 
6809 /*@
6810   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6811 
6812   Input Parameter:
6813 . dm - The DMPlex object
6814 
6815   Note: This is a useful diagnostic when creating meshes programmatically.
6816 
6817   Level: developer
6818 
6819 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6820 @*/
6821 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6822 {
6823   PetscSection    coneSection, supportSection;
6824   const PetscInt *cone, *support;
6825   PetscInt        coneSize, c, supportSize, s;
6826   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6827   PetscBool       storagecheck = PETSC_TRUE;
6828   PetscErrorCode  ierr;
6829 
6830   PetscFunctionBegin;
6831   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6832   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6833   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6834   /* Check that point p is found in the support of its cone points, and vice versa */
6835   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6836   for (p = pStart; p < pEnd; ++p) {
6837     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6838     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6839     for (c = 0; c < coneSize; ++c) {
6840       PetscBool dup = PETSC_FALSE;
6841       PetscInt  d;
6842       for (d = c-1; d >= 0; --d) {
6843         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6844       }
6845       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6846       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6847       for (s = 0; s < supportSize; ++s) {
6848         if (support[s] == p) break;
6849       }
6850       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6851         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6852         for (s = 0; s < coneSize; ++s) {
6853           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6854         }
6855         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6856         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6857         for (s = 0; s < supportSize; ++s) {
6858           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6859         }
6860         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6861         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6862         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6863       }
6864     }
6865     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6866     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6867     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6868     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6869     for (s = 0; s < supportSize; ++s) {
6870       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6871       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6872       for (c = 0; c < coneSize; ++c) {
6873         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6874         if (cone[c] != pp) { c = 0; break; }
6875         if (cone[c] == p) break;
6876       }
6877       if (c >= coneSize) {
6878         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6879         for (c = 0; c < supportSize; ++c) {
6880           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6881         }
6882         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6883         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6884         for (c = 0; c < coneSize; ++c) {
6885           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6886         }
6887         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6888         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6889       }
6890     }
6891   }
6892   if (storagecheck) {
6893     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6894     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6895     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6896   }
6897   PetscFunctionReturn(0);
6898 }
6899 
6900 /*@
6901   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6902 
6903   Input Parameters:
6904 + dm - The DMPlex object
6905 - cellHeight - Normally 0
6906 
6907   Note: This is a useful diagnostic when creating meshes programmatically.
6908   Currently applicable only to homogeneous simplex or tensor meshes.
6909 
6910   Level: developer
6911 
6912 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6913 @*/
6914 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6915 {
6916   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6917   PetscBool      isSimplex = PETSC_FALSE;
6918   PetscErrorCode ierr;
6919 
6920   PetscFunctionBegin;
6921   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6922   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6923   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6924   if (cStart < cEnd) {
6925     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6926     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6927   }
6928   switch (dim) {
6929   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6930   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6931   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6932   default:
6933     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6934   }
6935   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6936   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6937   cMax = cMax >= 0 ? cMax : cEnd;
6938   for (c = cStart; c < cMax; ++c) {
6939     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6940 
6941     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6942     for (cl = 0; cl < closureSize*2; cl += 2) {
6943       const PetscInt p = closure[cl];
6944       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6945     }
6946     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6947     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6948   }
6949   for (c = cMax; c < cEnd; ++c) {
6950     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6951 
6952     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6953     for (cl = 0; cl < closureSize*2; cl += 2) {
6954       const PetscInt p = closure[cl];
6955       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6956     }
6957     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6958     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6959   }
6960   PetscFunctionReturn(0);
6961 }
6962 
6963 /*@
6964   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6965 
6966   Input Parameters:
6967 + dm - The DMPlex object
6968 - cellHeight - Normally 0
6969 
6970   Note: This is a useful diagnostic when creating meshes programmatically.
6971 
6972   Level: developer
6973 
6974 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6975 @*/
6976 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6977 {
6978   PetscInt       pMax[4];
6979   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6980   PetscErrorCode ierr;
6981 
6982   PetscFunctionBegin;
6983   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6984   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6985   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6986   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6987   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6988   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6989     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6990     for (c = cStart; c < cEnd; ++c) {
6991       const PetscInt *cone, *ornt, *faces;
6992       PetscInt        numFaces, faceSize, coneSize,f;
6993       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6994 
6995       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6996       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6997       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6998       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6999       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7000       for (cl = 0; cl < closureSize*2; cl += 2) {
7001         const PetscInt p = closure[cl];
7002         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
7003       }
7004       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
7005       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
7006       for (f = 0; f < numFaces; ++f) {
7007         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
7008 
7009         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7010         for (cl = 0; cl < fclosureSize*2; cl += 2) {
7011           const PetscInt p = fclosure[cl];
7012           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
7013         }
7014         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
7015         for (v = 0; v < fnumCorners; ++v) {
7016           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
7017         }
7018         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7019       }
7020       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
7021       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7022     }
7023   }
7024   PetscFunctionReturn(0);
7025 }
7026 
7027 /*@
7028   DMPlexCheckGeometry - Check the geometry of mesh cells
7029 
7030   Input Parameter:
7031 . dm - The DMPlex object
7032 
7033   Note: This is a useful diagnostic when creating meshes programmatically.
7034 
7035   Level: developer
7036 
7037 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
7038 @*/
7039 PetscErrorCode DMPlexCheckGeometry(DM dm)
7040 {
7041   PetscReal      detJ, J[9], refVol = 1.0;
7042   PetscReal      vol;
7043   PetscInt       dim, depth, d, cStart, cEnd, c, cMax;
7044   PetscErrorCode ierr;
7045 
7046   PetscFunctionBegin;
7047   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7048   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7049   for (d = 0; d < dim; ++d) refVol *= 2.0;
7050   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7051   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7052   cMax = cMax < 0 ? cEnd : cMax;
7053   for (c = cStart; c < cMax; ++c) {
7054     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7055     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7056     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7057     if (depth > 1) {
7058       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7059       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7060       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7061     }
7062   }
7063   PetscFunctionReturn(0);
7064 }
7065 
7066 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
7067 {
7068   PetscInt i,l,n;
7069   const PetscInt *cone;
7070   PetscErrorCode ierr;
7071 
7072   PetscFunctionBegin;
7073   *missingPoint = -1;
7074   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
7075   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
7076   for (i=0; i<n; i++) {
7077     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
7078     if (l < 0) {
7079       *missingPoint = cone[i];
7080       break;
7081     }
7082   }
7083   PetscFunctionReturn(0);
7084 }
7085 
7086 /*@
7087   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
7088 
7089   Input Parameters:
7090 . dm - The DMPlex object
7091 
7092   Notes:
7093   This is mainly intended for debugging/testing purposes.
7094   It currently checks only meshes with no partition overlapping.
7095 
7096   Level: developer
7097 
7098 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7099 @*/
7100 PetscErrorCode DMPlexCheckPointSF(DM dm)
7101 {
7102   PetscSF         sf;
7103   PetscInt        d,depth,i,nleaves,p,plo,phi,missingPoint;
7104   const PetscInt *locals;
7105   PetscErrorCode  ierr;
7106 
7107   PetscFunctionBegin;
7108   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7109   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7110   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
7111   ierr = DMPlexGetOverlap(dm, &d);CHKERRQ(ierr);
7112   if (d) {
7113     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");
7114     PetscFunctionReturn(0);
7115   }
7116   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
7117 
7118   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7119   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
7120   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
7121   for (i=0; i<nleaves; i++) {
7122     p = locals[i];
7123     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
7124   }
7125 
7126   /* 2) if some point is in interface, then all its cone points must be also in interface  */
7127   for (i=0; i<nleaves; i++) {
7128     p = locals[i];
7129     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
7130     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
7131   }
7132   PetscFunctionReturn(0);
7133 }
7134 
7135 typedef struct cell_stats
7136 {
7137   PetscReal min, max, sum, squaresum;
7138   PetscInt  count;
7139 } cell_stats_t;
7140 
7141 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7142 {
7143   PetscInt i, N = *len;
7144 
7145   for (i = 0; i < N; i++) {
7146     cell_stats_t *A = (cell_stats_t *) a;
7147     cell_stats_t *B = (cell_stats_t *) b;
7148 
7149     B->min = PetscMin(A->min,B->min);
7150     B->max = PetscMax(A->max,B->max);
7151     B->sum += A->sum;
7152     B->squaresum += A->squaresum;
7153     B->count += A->count;
7154   }
7155 }
7156 
7157 /*@
7158   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
7159 
7160   Collective on dm
7161 
7162   Input Parameters:
7163 + dm        - The DMPlex object
7164 . output    - If true, statistics will be displayed on stdout
7165 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
7166 
7167   Note: This is mainly intended for debugging/testing purposes.
7168 
7169   Level: developer
7170 
7171 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7172 @*/
7173 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
7174 {
7175   DM             dmCoarse;
7176   cell_stats_t   stats, globalStats;
7177   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7178   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7179   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
7180   PetscInt       cdim, cStart, cEnd, cMax, c, eStart, eEnd, count = 0;
7181   PetscMPIInt    rank,size;
7182   PetscErrorCode ierr;
7183 
7184   PetscFunctionBegin;
7185   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7186   stats.min   = PETSC_MAX_REAL;
7187   stats.max   = PETSC_MIN_REAL;
7188   stats.sum   = stats.squaresum = 0.;
7189   stats.count = 0;
7190 
7191   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
7192   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7193   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
7194   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
7195   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7196   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
7197   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7198   cMax = cMax < 0 ? cEnd : cMax;
7199   for (c = cStart; c < cMax; c++) {
7200     PetscInt  i;
7201     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7202 
7203     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7204     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7205     for (i = 0; i < PetscSqr(cdim); ++i) {
7206       frobJ    += J[i] * J[i];
7207       frobInvJ += invJ[i] * invJ[i];
7208     }
7209     cond2 = frobJ * frobInvJ;
7210     cond  = PetscSqrtReal(cond2);
7211 
7212     stats.min        = PetscMin(stats.min,cond);
7213     stats.max        = PetscMax(stats.max,cond);
7214     stats.sum       += cond;
7215     stats.squaresum += cond2;
7216     stats.count++;
7217     if (output && cond > limit) {
7218       PetscSection coordSection;
7219       Vec          coordsLocal;
7220       PetscScalar *coords = NULL;
7221       PetscInt     Nv, d, clSize, cl, *closure = NULL;
7222 
7223       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
7224       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
7225       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7226       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
7227       for (i = 0; i < Nv/cdim; ++i) {
7228         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
7229         for (d = 0; d < cdim; ++d) {
7230           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
7231           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
7232         }
7233         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
7234       }
7235       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7236       for (cl = 0; cl < clSize*2; cl += 2) {
7237         const PetscInt edge = closure[cl];
7238 
7239         if ((edge >= eStart) && (edge < eEnd)) {
7240           PetscReal len;
7241 
7242           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
7243           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
7244         }
7245       }
7246       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7247       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7248     }
7249   }
7250   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
7251 
7252   if (size > 1) {
7253     PetscMPIInt   blockLengths[2] = {4,1};
7254     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7255     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7256     MPI_Op        statReduce;
7257 
7258     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7259     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7260     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7261     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7262     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7263     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7264   } else {
7265     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7266   }
7267   if (!rank) {
7268     count = globalStats.count;
7269     min   = globalStats.min;
7270     max   = globalStats.max;
7271     mean  = globalStats.sum / globalStats.count;
7272     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7273   }
7274 
7275   if (output) {
7276     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7277   }
7278   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7279 
7280   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7281   if (dmCoarse) {
7282     PetscBool isplex;
7283 
7284     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7285     if (isplex) {
7286       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
7287     }
7288   }
7289   PetscFunctionReturn(0);
7290 }
7291 
7292 /* Pointwise interpolation
7293      Just code FEM for now
7294      u^f = I u^c
7295      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7296      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7297      I_{ij} = psi^f_i phi^c_j
7298 */
7299 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7300 {
7301   PetscSection   gsc, gsf;
7302   PetscInt       m, n;
7303   void          *ctx;
7304   DM             cdm;
7305   PetscBool      regular, ismatis;
7306   PetscErrorCode ierr;
7307 
7308   PetscFunctionBegin;
7309   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7310   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7311   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7312   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7313 
7314   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7315   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7316   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7317   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7318   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7319 
7320   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7321   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7322   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7323   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7324   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7325   if (scaling) {
7326     /* Use naive scaling */
7327     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7328   }
7329   PetscFunctionReturn(0);
7330 }
7331 
7332 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7333 {
7334   PetscErrorCode ierr;
7335   VecScatter     ctx;
7336 
7337   PetscFunctionBegin;
7338   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7339   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7340   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7341   PetscFunctionReturn(0);
7342 }
7343 
7344 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7345 {
7346   PetscSection   gsc, gsf;
7347   PetscInt       m, n;
7348   void          *ctx;
7349   DM             cdm;
7350   PetscBool      regular;
7351   PetscErrorCode ierr;
7352 
7353   PetscFunctionBegin;
7354   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7355   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7356   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7357   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7358 
7359   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7360   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7361   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7362   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7363 
7364   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7365   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7366   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7367   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7368   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7369   PetscFunctionReturn(0);
7370 }
7371 
7372 /*@
7373   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7374 
7375   Input Parameter:
7376 . dm - The DMPlex object
7377 
7378   Output Parameter:
7379 . regular - The flag
7380 
7381   Level: intermediate
7382 
7383 .seealso: DMPlexSetRegularRefinement()
7384 @*/
7385 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7386 {
7387   PetscFunctionBegin;
7388   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7389   PetscValidPointer(regular, 2);
7390   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7391   PetscFunctionReturn(0);
7392 }
7393 
7394 /*@
7395   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7396 
7397   Input Parameters:
7398 + dm - The DMPlex object
7399 - regular - The flag
7400 
7401   Level: intermediate
7402 
7403 .seealso: DMPlexGetRegularRefinement()
7404 @*/
7405 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7406 {
7407   PetscFunctionBegin;
7408   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7409   ((DM_Plex *) dm->data)->regularRefinement = regular;
7410   PetscFunctionReturn(0);
7411 }
7412 
7413 /* anchors */
7414 /*@
7415   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7416   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7417 
7418   not collective
7419 
7420   Input Parameters:
7421 . dm - The DMPlex object
7422 
7423   Output Parameters:
7424 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7425 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7426 
7427 
7428   Level: intermediate
7429 
7430 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7431 @*/
7432 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7433 {
7434   DM_Plex *plex = (DM_Plex *)dm->data;
7435   PetscErrorCode ierr;
7436 
7437   PetscFunctionBegin;
7438   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7439   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7440   if (anchorSection) *anchorSection = plex->anchorSection;
7441   if (anchorIS) *anchorIS = plex->anchorIS;
7442   PetscFunctionReturn(0);
7443 }
7444 
7445 /*@
7446   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7447   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7448   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7449 
7450   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7451   DMGetConstraints() and filling in the entries in the constraint matrix.
7452 
7453   collective on dm
7454 
7455   Input Parameters:
7456 + dm - The DMPlex object
7457 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7458 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7459 
7460   The reference counts of anchorSection and anchorIS are incremented.
7461 
7462   Level: intermediate
7463 
7464 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7465 @*/
7466 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7467 {
7468   DM_Plex        *plex = (DM_Plex *)dm->data;
7469   PetscMPIInt    result;
7470   PetscErrorCode ierr;
7471 
7472   PetscFunctionBegin;
7473   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7474   if (anchorSection) {
7475     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7476     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7477     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7478   }
7479   if (anchorIS) {
7480     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7481     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7482     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7483   }
7484 
7485   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7486   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7487   plex->anchorSection = anchorSection;
7488 
7489   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7490   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7491   plex->anchorIS = anchorIS;
7492 
7493 #if defined(PETSC_USE_DEBUG)
7494   if (anchorIS && anchorSection) {
7495     PetscInt size, a, pStart, pEnd;
7496     const PetscInt *anchors;
7497 
7498     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7499     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7500     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7501     for (a = 0; a < size; a++) {
7502       PetscInt p;
7503 
7504       p = anchors[a];
7505       if (p >= pStart && p < pEnd) {
7506         PetscInt dof;
7507 
7508         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7509         if (dof) {
7510           PetscErrorCode ierr2;
7511 
7512           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7513           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7514         }
7515       }
7516     }
7517     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7518   }
7519 #endif
7520   /* reset the generic constraints */
7521   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7522   PetscFunctionReturn(0);
7523 }
7524 
7525 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7526 {
7527   PetscSection anchorSection;
7528   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7529   PetscErrorCode ierr;
7530 
7531   PetscFunctionBegin;
7532   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7533   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7534   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7535   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7536   if (numFields) {
7537     PetscInt f;
7538     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7539 
7540     for (f = 0; f < numFields; f++) {
7541       PetscInt numComp;
7542 
7543       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7544       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7545     }
7546   }
7547   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7548   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7549   pStart = PetscMax(pStart,sStart);
7550   pEnd   = PetscMin(pEnd,sEnd);
7551   pEnd   = PetscMax(pStart,pEnd);
7552   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7553   for (p = pStart; p < pEnd; p++) {
7554     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7555     if (dof) {
7556       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7557       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7558       for (f = 0; f < numFields; f++) {
7559         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7560         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7561       }
7562     }
7563   }
7564   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7565   PetscFunctionReturn(0);
7566 }
7567 
7568 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7569 {
7570   PetscSection aSec;
7571   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7572   const PetscInt *anchors;
7573   PetscInt numFields, f;
7574   IS aIS;
7575   PetscErrorCode ierr;
7576 
7577   PetscFunctionBegin;
7578   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7579   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7580   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7581   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7582   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7583   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7584   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7585   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7586   /* cSec will be a subset of aSec and section */
7587   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7588   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7589   i[0] = 0;
7590   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7591   for (p = pStart; p < pEnd; p++) {
7592     PetscInt rDof, rOff, r;
7593 
7594     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7595     if (!rDof) continue;
7596     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7597     if (numFields) {
7598       for (f = 0; f < numFields; f++) {
7599         annz = 0;
7600         for (r = 0; r < rDof; r++) {
7601           a = anchors[rOff + r];
7602           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7603           annz += aDof;
7604         }
7605         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7606         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7607         for (q = 0; q < dof; q++) {
7608           i[off + q + 1] = i[off + q] + annz;
7609         }
7610       }
7611     }
7612     else {
7613       annz = 0;
7614       for (q = 0; q < dof; q++) {
7615         a = anchors[off + q];
7616         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7617         annz += aDof;
7618       }
7619       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7620       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7621       for (q = 0; q < dof; q++) {
7622         i[off + q + 1] = i[off + q] + annz;
7623       }
7624     }
7625   }
7626   nnz = i[m];
7627   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7628   offset = 0;
7629   for (p = pStart; p < pEnd; p++) {
7630     if (numFields) {
7631       for (f = 0; f < numFields; f++) {
7632         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7633         for (q = 0; q < dof; q++) {
7634           PetscInt rDof, rOff, r;
7635           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7636           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7637           for (r = 0; r < rDof; r++) {
7638             PetscInt s;
7639 
7640             a = anchors[rOff + r];
7641             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7642             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7643             for (s = 0; s < aDof; s++) {
7644               j[offset++] = aOff + s;
7645             }
7646           }
7647         }
7648       }
7649     }
7650     else {
7651       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7652       for (q = 0; q < dof; q++) {
7653         PetscInt rDof, rOff, r;
7654         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7655         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7656         for (r = 0; r < rDof; r++) {
7657           PetscInt s;
7658 
7659           a = anchors[rOff + r];
7660           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7661           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7662           for (s = 0; s < aDof; s++) {
7663             j[offset++] = aOff + s;
7664           }
7665         }
7666       }
7667     }
7668   }
7669   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7670   ierr = PetscFree(i);CHKERRQ(ierr);
7671   ierr = PetscFree(j);CHKERRQ(ierr);
7672   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7673   PetscFunctionReturn(0);
7674 }
7675 
7676 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7677 {
7678   DM_Plex        *plex = (DM_Plex *)dm->data;
7679   PetscSection   anchorSection, section, cSec;
7680   Mat            cMat;
7681   PetscErrorCode ierr;
7682 
7683   PetscFunctionBegin;
7684   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7685   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7686   if (anchorSection) {
7687     PetscInt Nf;
7688 
7689     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
7690     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7691     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7692     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7693     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7694     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7695     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7696     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7697   }
7698   PetscFunctionReturn(0);
7699 }
7700 
7701 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7702 {
7703   IS             subis;
7704   PetscSection   section, subsection;
7705   PetscErrorCode ierr;
7706 
7707   PetscFunctionBegin;
7708   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
7709   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7710   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7711   /* Create subdomain */
7712   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7713   /* Create submodel */
7714   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7715   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7716   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7717   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
7718   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7719   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7720   /* Create map from submodel to global model */
7721   if (is) {
7722     PetscSection    sectionGlobal, subsectionGlobal;
7723     IS              spIS;
7724     const PetscInt *spmap;
7725     PetscInt       *subIndices;
7726     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7727     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7728 
7729     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7730     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7731     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7732     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7733     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7734     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7735     for (p = pStart; p < pEnd; ++p) {
7736       PetscInt gdof, pSubSize  = 0;
7737 
7738       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7739       if (gdof > 0) {
7740         for (f = 0; f < Nf; ++f) {
7741           PetscInt fdof, fcdof;
7742 
7743           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7744           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7745           pSubSize += fdof-fcdof;
7746         }
7747         subSize += pSubSize;
7748         if (pSubSize) {
7749           if (bs < 0) {
7750             bs = pSubSize;
7751           } else if (bs != pSubSize) {
7752             /* Layout does not admit a pointwise block size */
7753             bs = 1;
7754           }
7755         }
7756       }
7757     }
7758     /* Must have same blocksize on all procs (some might have no points) */
7759     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7760     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7761     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7762     else                            {bs = bsMinMax[0];}
7763     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7764     for (p = pStart; p < pEnd; ++p) {
7765       PetscInt gdof, goff;
7766 
7767       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7768       if (gdof > 0) {
7769         const PetscInt point = spmap[p];
7770 
7771         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7772         for (f = 0; f < Nf; ++f) {
7773           PetscInt fdof, fcdof, fc, f2, poff = 0;
7774 
7775           /* Can get rid of this loop by storing field information in the global section */
7776           for (f2 = 0; f2 < f; ++f2) {
7777             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7778             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7779             poff += fdof-fcdof;
7780           }
7781           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7782           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7783           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7784             subIndices[subOff] = goff+poff+fc;
7785           }
7786         }
7787       }
7788     }
7789     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7790     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7791     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7792     if (bs > 1) {
7793       /* We need to check that the block size does not come from non-contiguous fields */
7794       PetscInt i, j, set = 1;
7795       for (i = 0; i < subSize; i += bs) {
7796         for (j = 0; j < bs; ++j) {
7797           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7798         }
7799       }
7800       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7801     }
7802     /* Attach nullspace */
7803     for (f = 0; f < Nf; ++f) {
7804       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7805       if ((*subdm)->nullspaceConstructors[f]) break;
7806     }
7807     if (f < Nf) {
7808       MatNullSpace nullSpace;
7809 
7810       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7811       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7812       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7813     }
7814   }
7815   PetscFunctionReturn(0);
7816 }
7817