xref: /petsc/src/dm/impls/plex/plex.c (revision b72fa766f129fc2bd79867a41d61fdbe3fc875c4)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     PetscReal    scale         = 2.0;
675     PetscReal    tikzscale     = 1.0;
676     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
677     double       tcoords[3];
678     PetscScalar *coords;
679     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
680     PetscMPIInt  rank, size;
681     char         **names, **colors, **lcolors;
682     PetscBool    plotEdges, flg;
683 
684     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
685     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
686     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
687     numLabels  = PetscMax(numLabels, 10);
688     numColors  = 10;
689     numLColors = 10;
690     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
691     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
692     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
693     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
694     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
695     if (!useLabels) numLabels = 0;
696     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
697     if (!useColors) {
698       numColors = 3;
699       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
700     }
701     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
702     if (!useColors) {
703       numLColors = 4;
704       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
705     }
706     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
707     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
708     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
709     if (depth < dim) plotEdges = PETSC_FALSE;
710     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
711     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
712     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
713     ierr = PetscViewerASCIIPrintf(viewer, "\
714 \\documentclass[tikz]{standalone}\n\n\
715 \\usepackage{pgflibraryshapes}\n\
716 \\usetikzlibrary{backgrounds}\n\
717 \\usetikzlibrary{arrows}\n\
718 \\begin{document}\n");CHKERRQ(ierr);
719     if (size > 1) {
720       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
721       for (p = 0; p < size; ++p) {
722         if (p > 0 && p == size-1) {
723           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
724         } else if (p > 0) {
725           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
726         }
727         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
728       }
729       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
730     }
731     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", tikzscale);CHKERRQ(ierr);
732     /* Plot vertices */
733     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
734     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
735     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
736     for (v = vStart; v < vEnd; ++v) {
737       PetscInt  off, dof, d;
738       PetscBool isLabeled = PETSC_FALSE;
739 
740       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
741       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
742       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
743       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
744       for (d = 0; d < dof; ++d) {
745         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
746         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
747       }
748       /* Rotate coordinates since PGF makes z point out of the page instead of up */
749       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
750       for (d = 0; d < dof; ++d) {
751         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
752         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", tcoords[d]);CHKERRQ(ierr);
753       }
754       color = colors[rank%numColors];
755       for (l = 0; l < numLabels; ++l) {
756         PetscInt val;
757         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
758         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
759       }
760       if (useNumbers) {
761         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
762       } else {
763         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
764       }
765     }
766     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
767     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
768     /* Plot cells */
769     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
770     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
771     if (dim == 3 || !useNumbers) {
772       for (e = eStart; e < eEnd; ++e) {
773         const PetscInt *cone;
774 
775         color = colors[rank%numColors];
776         for (l = 0; l < numLabels; ++l) {
777           PetscInt val;
778           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
779           if (val >= 0) {color = lcolors[l%numLColors]; break;}
780         }
781         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
782         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
783       }
784     } else {
785       for (c = cStart; c < cEnd; ++c) {
786         PetscInt *closure = NULL;
787         PetscInt  closureSize, firstPoint = -1;
788 
789         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
790         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
791         for (p = 0; p < closureSize*2; p += 2) {
792           const PetscInt point = closure[p];
793 
794           if ((point < vStart) || (point >= vEnd)) continue;
795           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
796           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
797           if (firstPoint < 0) firstPoint = point;
798         }
799         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
800         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
801         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
802       }
803     }
804     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
805     for (c = cStart; c < cEnd; ++c) {
806       double    ccoords[3] = {0.0, 0.0, 0.0};
807       PetscBool isLabeled  = PETSC_FALSE;
808       PetscInt *closure    = NULL;
809       PetscInt  closureSize, dof, d, n = 0;
810 
811       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
812       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
813       for (p = 0; p < closureSize*2; p += 2) {
814         const PetscInt point = closure[p];
815         PetscInt       off;
816 
817         if ((point < vStart) || (point >= vEnd)) continue;
818         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
819         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
820         for (d = 0; d < dof; ++d) {
821           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
822           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
823         }
824         /* Rotate coordinates since PGF makes z point out of the page instead of up */
825         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
826         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
827         ++n;
828       }
829       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
830       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
831       for (d = 0; d < dof; ++d) {
832         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
833         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", ccoords[d]);CHKERRQ(ierr);
834       }
835       color = colors[rank%numColors];
836       for (l = 0; l < numLabels; ++l) {
837         PetscInt val;
838         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
839         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
840       }
841       if (useNumbers) {
842         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
843       } else {
844         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
845       }
846     }
847     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
848     /* Plot edges */
849     if (plotEdges) {
850       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
851       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
852       for (e = eStart; e < eEnd; ++e) {
853         const PetscInt *cone;
854         PetscInt        coneSize, offA, offB, dof, d;
855 
856         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
857         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
858         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
859         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
860         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
861         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
862         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
863         for (d = 0; d < dof; ++d) {
864           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
865           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
866         }
867         /* Rotate coordinates since PGF makes z point out of the page instead of up */
868         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
869         for (d = 0; d < dof; ++d) {
870           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
871           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
872         }
873         color = colors[rank%numColors];
874         for (l = 0; l < numLabels; ++l) {
875           PetscInt val;
876           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
877           if (val >= 0) {color = lcolors[l%numLColors]; break;}
878         }
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
880       }
881       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
882       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
883       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
884     }
885     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
886     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
887     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
888     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
889     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
890     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
891     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
892     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
893   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
894     Vec                    cown,acown;
895     VecScatter             sct;
896     ISLocalToGlobalMapping g2l;
897     IS                     gid,acis;
898     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
899     MPI_Group              ggroup,ngroup;
900     PetscScalar            *array,nid;
901     const PetscInt         *idxs;
902     PetscInt               *idxs2,*start,*adjacency,*work;
903     PetscInt64             lm[3],gm[3];
904     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
905     PetscMPIInt            d1,d2,rank;
906 
907     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
908     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
909 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
910     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
911 #endif
912     if (ncomm != MPI_COMM_NULL) {
913       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
914       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
915       d1   = 0;
916       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
917       nid  = d2;
918       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
919       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
920       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
921     } else nid = 0.0;
922 
923     /* Get connectivity */
924     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
925     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
926 
927     /* filter overlapped local cells */
928     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
929     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
930     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
931     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
932     for (c = cStart, cum = 0; c < cEnd; c++) {
933       if (idxs[c-cStart] < 0) continue;
934       idxs2[cum++] = idxs[c-cStart];
935     }
936     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
937     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
938     ierr = ISDestroy(&gid);CHKERRQ(ierr);
939     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
940 
941     /* support for node-aware cell locality */
942     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
943     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
944     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
945     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
946     for (c = 0; c < numVertices; c++) array[c] = nid;
947     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
948     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
949     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
950     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
951     ierr = ISDestroy(&acis);CHKERRQ(ierr);
952     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
953     ierr = VecDestroy(&cown);CHKERRQ(ierr);
954 
955     /* compute edgeCut */
956     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
957     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
958     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
959     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
960     ierr = ISDestroy(&gid);CHKERRQ(ierr);
961     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
962     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
963       PetscInt totl;
964 
965       totl = start[c+1]-start[c];
966       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
967       for (i = 0; i < totl; i++) {
968         if (work[i] < 0) {
969           ect  += 1;
970           ectn += (array[i + start[c]] != nid) ? 0 : 1;
971         }
972       }
973     }
974     ierr  = PetscFree(work);CHKERRQ(ierr);
975     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
976     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
977     lm[1] = -numVertices;
978     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
979     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
980     lm[0] = ect; /* edgeCut */
981     lm[1] = ectn; /* node-aware edgeCut */
982     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
983     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
984     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
985 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
986     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
987 #else
988     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
989 #endif
990     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
991     ierr  = PetscFree(start);CHKERRQ(ierr);
992     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
993     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
994   } else {
995     MPI_Comm    comm;
996     PetscInt   *sizes, *hybsizes;
997     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
998     PetscInt    pStart, pEnd, p;
999     PetscInt    numLabels, l;
1000     const char *name;
1001     PetscMPIInt size;
1002 
1003     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1004     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1005     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1006     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1007     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1008     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1009     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1010     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1011     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1012     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1013     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1014     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1015     if (depth == 1) {
1016       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1017       pEnd = pEnd - pStart;
1018       pMax[0] -= pStart;
1019       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1020       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1021       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1022       for (p = 0; p < size; ++p) {
1023         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1024         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1025       }
1026       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1027       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1028       pEnd = pEnd - pStart;
1029       pMax[depth] -= pStart;
1030       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1031       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1032       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1033       for (p = 0; p < size; ++p) {
1034         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1035         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1036       }
1037       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1038     } else {
1039       PetscMPIInt rank;
1040       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1041       for (d = 0; d <= dim; d++) {
1042         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1043         pEnd    -= pStart;
1044         pMax[d] -= pStart;
1045         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1046         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1047         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1048         for (p = 0; p < size; ++p) {
1049           if (!rank) {
1050             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1051             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1052           }
1053         }
1054         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1055       }
1056     }
1057     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1058     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1059     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1060     for (l = 0; l < numLabels; ++l) {
1061       DMLabel         label;
1062       const char     *name;
1063       IS              valueIS;
1064       const PetscInt *values;
1065       PetscInt        numValues, v;
1066 
1067       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1068       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1069       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1070       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1071       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1072       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1073       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1074       for (v = 0; v < numValues; ++v) {
1075         PetscInt size;
1076 
1077         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1078         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1079         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1080       }
1081       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1082       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1083       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1084       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1085     }
1086     /* If no fields are specified, people do not want to see adjacency */
1087     if (dm->Nf) {
1088       PetscInt f;
1089 
1090       for (f = 0; f < dm->Nf; ++f) {
1091         const char *name;
1092 
1093         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1094         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1095         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1096         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1097         if (dm->fields[f].adjacency[0]) {
1098           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1099           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1100         } else {
1101           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1102           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1103         }
1104         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1105       }
1106     }
1107     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1108     if (cdm) {
1109       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1110       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1111       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1112     }
1113   }
1114   PetscFunctionReturn(0);
1115 }
1116 
1117 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1118 {
1119   PetscDraw          draw;
1120   DM                 cdm;
1121   PetscSection       coordSection;
1122   Vec                coordinates;
1123   const PetscScalar *coords;
1124   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1125   PetscBool          isnull;
1126   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1127   PetscMPIInt        rank;
1128   PetscErrorCode     ierr;
1129 
1130   PetscFunctionBegin;
1131   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1132   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1133   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1134   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
1135   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1136   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1137   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1138 
1139   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1140   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1141   if (isnull) PetscFunctionReturn(0);
1142   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1143 
1144   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1145   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1146   for (c = 0; c < N; c += dim) {
1147     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1148     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1149   }
1150   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1151   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1152   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1153   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1154   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1155 
1156   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1157   for (c = cStart; c < cEnd; ++c) {
1158     PetscScalar *coords = NULL;
1159     PetscInt     numCoords,coneSize;
1160 
1161     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1162     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1163     switch (coneSize) {
1164     case 3:
1165       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1166                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1167                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1168                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1169       break;
1170     case 4:
1171       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1172                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1173                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1174                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1175                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1176       break;
1177     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1178     }
1179     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1180   }
1181   for (c = cStart; c < cEnd; ++c) {
1182     PetscScalar *coords = NULL;
1183     PetscInt     numCoords,coneSize;
1184 
1185     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1186     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1187     switch (coneSize) {
1188     case 3:
1189       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1190       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1191       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1192       break;
1193     case 4:
1194       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1195       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1196       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1197       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1198       break;
1199     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1200     }
1201     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202   }
1203   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1204   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1205   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1206   PetscFunctionReturn(0);
1207 }
1208 
1209 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1210 {
1211   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1212   char           name[PETSC_MAX_PATH_LEN];
1213   PetscErrorCode ierr;
1214 
1215   PetscFunctionBegin;
1216   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1217   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1219   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1220   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1221   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1222   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1223   if (iascii) {
1224     PetscViewerFormat format;
1225     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1226     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1227       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1228     } else {
1229       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1230     }
1231   } else if (ishdf5) {
1232 #if defined(PETSC_HAVE_HDF5)
1233     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1234 #else
1235     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1236 #endif
1237   } else if (isvtk) {
1238     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1239   } else if (isdraw) {
1240     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1241   } else if (isglvis) {
1242     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1243   } else {
1244     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1245   }
1246   /* Optionally view the partition */
1247   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1248   if (flg) {
1249     Vec ranks;
1250     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1251     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1252     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1253   }
1254   /* Optionally view a label */
1255   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1256   if (flg) {
1257     DMLabel label;
1258     Vec     val;
1259 
1260     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1261     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1262     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1263     ierr = VecView(val, viewer);CHKERRQ(ierr);
1264     ierr = VecDestroy(&val);CHKERRQ(ierr);
1265   }
1266   PetscFunctionReturn(0);
1267 }
1268 
1269 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1270 {
1271   PetscBool      ishdf5;
1272   PetscErrorCode ierr;
1273 
1274   PetscFunctionBegin;
1275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1276   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1277   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1278   if (ishdf5) {
1279 #if defined(PETSC_HAVE_HDF5)
1280     PetscViewerFormat format;
1281     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1282     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1283       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1284     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1285       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1286     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1287 #else
1288     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1289 #endif
1290   } else {
1291     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1292   }
1293   PetscFunctionReturn(0);
1294 }
1295 
1296 PetscErrorCode DMDestroy_Plex(DM dm)
1297 {
1298   DM_Plex       *mesh = (DM_Plex*) dm->data;
1299   PetscErrorCode ierr;
1300 
1301   PetscFunctionBegin;
1302   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1303   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1304   if (--mesh->refct > 0) PetscFunctionReturn(0);
1305   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1306   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1307   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1308   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1309   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1310   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1311   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1312   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1313   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1314   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1315   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1316   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1317   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1318   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1319   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1320   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1321   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1322   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1323   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1324   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1325   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1326   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1327   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1328   ierr = PetscFree(mesh);CHKERRQ(ierr);
1329   PetscFunctionReturn(0);
1330 }
1331 
1332 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1333 {
1334   PetscSection           sectionGlobal;
1335   PetscInt               bs = -1, mbs;
1336   PetscInt               localSize;
1337   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1338   PetscErrorCode         ierr;
1339   MatType                mtype;
1340   ISLocalToGlobalMapping ltog;
1341 
1342   PetscFunctionBegin;
1343   ierr = MatInitializePackage();CHKERRQ(ierr);
1344   mtype = dm->mattype;
1345   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1346   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1347   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1348   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1349   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1350   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1351   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1352   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1353   if (mbs > 1) bs = mbs;
1354   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1355   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1356   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1357   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1358   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1359   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1360   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1361   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1362   if (!isShell) {
1363     PetscSection subSection;
1364     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1365     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1366     PetscInt     pStart, pEnd, p, dof, cdof;
1367 
1368     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1369     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1370       PetscSection section;
1371       PetscInt     size;
1372 
1373       ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
1374       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1375       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1376       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1377     } else {
1378       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1379     }
1380     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1381     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1382       PetscInt bdof;
1383 
1384       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1385       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1386       dof  = dof < 0 ? -(dof+1) : dof;
1387       bdof = cdof && (dof-cdof) ? 1 : dof;
1388       if (dof) {
1389         if (bs < 0)          {bs = bdof;}
1390         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1391       }
1392       if (isMatIS) {
1393         PetscInt loff,c,off;
1394         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1395         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1396         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1397       }
1398     }
1399     /* Must have same blocksize on all procs (some might have no points) */
1400     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1401     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1402     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1403     else                            {bs = bsMinMax[0];}
1404     bs = bs < 0 ? 1 : bs;
1405     if (isMatIS) {
1406       PetscInt l;
1407       /* Must reduce indices by blocksize */
1408       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] /= bs;
1409       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1410     }
1411     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1412     if (isMatIS) {
1413       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1414     }
1415     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1416     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1417     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1418   }
1419   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1420   PetscFunctionReturn(0);
1421 }
1422 
1423 /*@
1424   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1425 
1426   Not collective
1427 
1428   Input Parameter:
1429 . mesh - The DMPlex
1430 
1431   Output Parameters:
1432 . subsection - The subdomain section
1433 
1434   Level: developer
1435 
1436 .seealso:
1437 @*/
1438 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1439 {
1440   DM_Plex       *mesh = (DM_Plex*) dm->data;
1441   PetscErrorCode ierr;
1442 
1443   PetscFunctionBegin;
1444   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1445   if (!mesh->subdomainSection) {
1446     PetscSection section;
1447     PetscSF      sf;
1448 
1449     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1450     ierr = DMGetSection(dm,&section);CHKERRQ(ierr);
1451     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1452     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1453   }
1454   *subsection = mesh->subdomainSection;
1455   PetscFunctionReturn(0);
1456 }
1457 
1458 /*@
1459   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1460 
1461   Not collective
1462 
1463   Input Parameter:
1464 . mesh - The DMPlex
1465 
1466   Output Parameters:
1467 + pStart - The first mesh point
1468 - pEnd   - The upper bound for mesh points
1469 
1470   Level: beginner
1471 
1472 .seealso: DMPlexCreate(), DMPlexSetChart()
1473 @*/
1474 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1475 {
1476   DM_Plex       *mesh = (DM_Plex*) dm->data;
1477   PetscErrorCode ierr;
1478 
1479   PetscFunctionBegin;
1480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1481   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1482   PetscFunctionReturn(0);
1483 }
1484 
1485 /*@
1486   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1487 
1488   Not collective
1489 
1490   Input Parameters:
1491 + mesh - The DMPlex
1492 . pStart - The first mesh point
1493 - pEnd   - The upper bound for mesh points
1494 
1495   Output Parameters:
1496 
1497   Level: beginner
1498 
1499 .seealso: DMPlexCreate(), DMPlexGetChart()
1500 @*/
1501 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1502 {
1503   DM_Plex       *mesh = (DM_Plex*) dm->data;
1504   PetscErrorCode ierr;
1505 
1506   PetscFunctionBegin;
1507   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1508   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1509   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1510   PetscFunctionReturn(0);
1511 }
1512 
1513 /*@
1514   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1515 
1516   Not collective
1517 
1518   Input Parameters:
1519 + mesh - The DMPlex
1520 - p - The point, which must lie in the chart set with DMPlexSetChart()
1521 
1522   Output Parameter:
1523 . size - The cone size for point p
1524 
1525   Level: beginner
1526 
1527 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1528 @*/
1529 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1530 {
1531   DM_Plex       *mesh = (DM_Plex*) dm->data;
1532   PetscErrorCode ierr;
1533 
1534   PetscFunctionBegin;
1535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1536   PetscValidPointer(size, 3);
1537   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1538   PetscFunctionReturn(0);
1539 }
1540 
1541 /*@
1542   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1543 
1544   Not collective
1545 
1546   Input Parameters:
1547 + mesh - The DMPlex
1548 . p - The point, which must lie in the chart set with DMPlexSetChart()
1549 - size - The cone size for point p
1550 
1551   Output Parameter:
1552 
1553   Note:
1554   This should be called after DMPlexSetChart().
1555 
1556   Level: beginner
1557 
1558 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1559 @*/
1560 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1561 {
1562   DM_Plex       *mesh = (DM_Plex*) dm->data;
1563   PetscErrorCode ierr;
1564 
1565   PetscFunctionBegin;
1566   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1567   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1568 
1569   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1570   PetscFunctionReturn(0);
1571 }
1572 
1573 /*@
1574   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1575 
1576   Not collective
1577 
1578   Input Parameters:
1579 + mesh - The DMPlex
1580 . p - The point, which must lie in the chart set with DMPlexSetChart()
1581 - size - The additional cone size for point p
1582 
1583   Output Parameter:
1584 
1585   Note:
1586   This should be called after DMPlexSetChart().
1587 
1588   Level: beginner
1589 
1590 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1591 @*/
1592 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1593 {
1594   DM_Plex       *mesh = (DM_Plex*) dm->data;
1595   PetscInt       csize;
1596   PetscErrorCode ierr;
1597 
1598   PetscFunctionBegin;
1599   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1600   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1601   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1602 
1603   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1604   PetscFunctionReturn(0);
1605 }
1606 
1607 /*@C
1608   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1609 
1610   Not collective
1611 
1612   Input Parameters:
1613 + dm - The DMPlex
1614 - p - The point, which must lie in the chart set with DMPlexSetChart()
1615 
1616   Output Parameter:
1617 . cone - An array of points which are on the in-edges for point p
1618 
1619   Level: beginner
1620 
1621   Fortran Notes:
1622   Since it returns an array, this routine is only available in Fortran 90, and you must
1623   include petsc.h90 in your code.
1624 
1625 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1626 @*/
1627 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1628 {
1629   DM_Plex       *mesh = (DM_Plex*) dm->data;
1630   PetscInt       off;
1631   PetscErrorCode ierr;
1632 
1633   PetscFunctionBegin;
1634   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1635   PetscValidPointer(cone, 3);
1636   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1637   *cone = &mesh->cones[off];
1638   PetscFunctionReturn(0);
1639 }
1640 
1641 /*@C
1642   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1643 
1644   Not collective
1645 
1646   Input Parameters:
1647 + dm - The DMPlex
1648 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1649 
1650   Output Parameter:
1651 + pConesSection - PetscSection describing the layout of pCones
1652 - pCones - An array of points which are on the in-edges for the point set p
1653 
1654   Level: intermediate
1655 
1656 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1657 @*/
1658 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1659 {
1660   PetscSection        cs, newcs;
1661   PetscInt            *cones;
1662   PetscInt            *newarr=NULL;
1663   PetscInt            n;
1664   PetscErrorCode      ierr;
1665 
1666   PetscFunctionBegin;
1667   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1668   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1669   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1670   if (pConesSection) *pConesSection = newcs;
1671   if (pCones) {
1672     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1673     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1674   }
1675   PetscFunctionReturn(0);
1676 }
1677 
1678 static PetscErrorCode DMPlexGetConeRecursive_Private(DM dm, PetscInt *n_inout, const PetscInt points[], PetscInt *offset_inout, PetscInt buf[])
1679 {
1680   PetscInt p, n, cn, i;
1681   const PetscInt *cone;
1682   PetscErrorCode ierr;
1683 
1684   PetscFunctionBegin;
1685   n = *n_inout;
1686   *n_inout = 0;
1687   for (i=0; i<n; i++) {
1688     p = points[i];
1689     ierr = DMPlexGetConeSize(dm, p, &cn);CHKERRQ(ierr);
1690     if (!cn) {
1691       cn = 1;
1692       if (buf) {
1693         buf[*offset_inout] = p;
1694         ++(*offset_inout);
1695       }
1696     } else {
1697       ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1698       ierr = DMPlexGetConeRecursive_Private(dm, &cn, cone, offset_inout, buf);CHKERRQ(ierr);
1699     }
1700     *n_inout += cn;
1701   }
1702   PetscFunctionReturn(0);
1703 }
1704 
1705 /*@C
1706   DMPlexGetConeRecursive - Like DMPlexGetConeTuple() but recursive, i.e. each cone point is expanded into a set of its own cone points until a vertex (DAG point with no cone) is reached.
1707 
1708   Not collective
1709 
1710   Input Parameters:
1711 + dm - The DMPlex
1712 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1713 
1714   Output Parameter:
1715 . pCones - An array of recursively expanded cones, i.e. containing only vertices, and each of them can be present multiple times
1716 
1717   Level: advanced
1718 
1719 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple()
1720 @*/
1721 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS p, IS *pCones)
1722 {
1723   const PetscInt      *arr=NULL;
1724   PetscInt            *cpoints=NULL;
1725   PetscInt            n, cn;
1726   PetscInt            zero;
1727   PetscErrorCode      ierr;
1728 
1729   PetscFunctionBegin;
1730   ierr = ISGetLocalSize(p, &n);CHKERRQ(ierr);
1731   ierr = ISGetIndices(p, &arr);CHKERRQ(ierr);
1732   zero = 0;
1733   /* first figure out the total number of returned points */
1734   cn = n;
1735   ierr = DMPlexGetConeRecursive_Private(dm, &cn, arr, &zero, NULL);CHKERRQ(ierr);
1736   ierr = PetscMalloc1(cn, &cpoints);CHKERRQ(ierr);
1737   /* now get recursive cones themselves */
1738   ierr = DMPlexGetConeRecursive_Private(dm, &n, arr, &zero, cpoints);CHKERRQ(ierr);
1739   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, cpoints, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1740   ierr = ISRestoreIndices(p, &arr);CHKERRQ(ierr);
1741   PetscFunctionReturn(0);
1742 }
1743 
1744 /*@
1745   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1746 
1747   Not collective
1748 
1749   Input Parameters:
1750 + mesh - The DMPlex
1751 . p - The point, which must lie in the chart set with DMPlexSetChart()
1752 - cone - An array of points which are on the in-edges for point p
1753 
1754   Output Parameter:
1755 
1756   Note:
1757   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1758 
1759   Developer Note: Why not call this DMPlexSetCover()
1760 
1761   Level: beginner
1762 
1763 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1764 @*/
1765 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1766 {
1767   DM_Plex       *mesh = (DM_Plex*) dm->data;
1768   PetscInt       pStart, pEnd;
1769   PetscInt       dof, off, c;
1770   PetscErrorCode ierr;
1771 
1772   PetscFunctionBegin;
1773   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1774   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1775   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1776   if (dof) PetscValidPointer(cone, 3);
1777   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1778   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1779   for (c = 0; c < dof; ++c) {
1780     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1781     mesh->cones[off+c] = cone[c];
1782   }
1783   PetscFunctionReturn(0);
1784 }
1785 
1786 /*@C
1787   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1788 
1789   Not collective
1790 
1791   Input Parameters:
1792 + mesh - The DMPlex
1793 - p - The point, which must lie in the chart set with DMPlexSetChart()
1794 
1795   Output Parameter:
1796 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1797                     integer giving the prescription for cone traversal. If it is negative, the cone is
1798                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1799                     the index of the cone point on which to start.
1800 
1801   Level: beginner
1802 
1803   Fortran Notes:
1804   Since it returns an array, this routine is only available in Fortran 90, and you must
1805   include petsc.h90 in your code.
1806 
1807   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1808 
1809 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1810 @*/
1811 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1812 {
1813   DM_Plex       *mesh = (DM_Plex*) dm->data;
1814   PetscInt       off;
1815   PetscErrorCode ierr;
1816 
1817   PetscFunctionBegin;
1818   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1819 #if defined(PETSC_USE_DEBUG)
1820   {
1821     PetscInt dof;
1822     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1823     if (dof) PetscValidPointer(coneOrientation, 3);
1824   }
1825 #endif
1826   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1827 
1828   *coneOrientation = &mesh->coneOrientations[off];
1829   PetscFunctionReturn(0);
1830 }
1831 
1832 /*@
1833   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1834 
1835   Not collective
1836 
1837   Input Parameters:
1838 + mesh - The DMPlex
1839 . p - The point, which must lie in the chart set with DMPlexSetChart()
1840 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1841                     integer giving the prescription for cone traversal. If it is negative, the cone is
1842                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1843                     the index of the cone point on which to start.
1844 
1845   Output Parameter:
1846 
1847   Note:
1848   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1849 
1850   Level: beginner
1851 
1852 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1853 @*/
1854 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1855 {
1856   DM_Plex       *mesh = (DM_Plex*) dm->data;
1857   PetscInt       pStart, pEnd;
1858   PetscInt       dof, off, c;
1859   PetscErrorCode ierr;
1860 
1861   PetscFunctionBegin;
1862   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1863   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1864   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1865   if (dof) PetscValidPointer(coneOrientation, 3);
1866   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1867   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1868   for (c = 0; c < dof; ++c) {
1869     PetscInt cdof, o = coneOrientation[c];
1870 
1871     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1872     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1873     mesh->coneOrientations[off+c] = o;
1874   }
1875   PetscFunctionReturn(0);
1876 }
1877 
1878 /*@
1879   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
1880 
1881   Not collective
1882 
1883   Input Parameters:
1884 + mesh - The DMPlex
1885 . p - The point, which must lie in the chart set with DMPlexSetChart()
1886 . conePos - The local index in the cone where the point should be put
1887 - conePoint - The mesh point to insert
1888 
1889   Level: beginner
1890 
1891 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1892 @*/
1893 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1894 {
1895   DM_Plex       *mesh = (DM_Plex*) dm->data;
1896   PetscInt       pStart, pEnd;
1897   PetscInt       dof, off;
1898   PetscErrorCode ierr;
1899 
1900   PetscFunctionBegin;
1901   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1902   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1903   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1904   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1905   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1906   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1907   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1908   mesh->cones[off+conePos] = conePoint;
1909   PetscFunctionReturn(0);
1910 }
1911 
1912 /*@
1913   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
1914 
1915   Not collective
1916 
1917   Input Parameters:
1918 + mesh - The DMPlex
1919 . p - The point, which must lie in the chart set with DMPlexSetChart()
1920 . conePos - The local index in the cone where the point should be put
1921 - coneOrientation - The point orientation to insert
1922 
1923   Level: beginner
1924 
1925 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1926 @*/
1927 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1928 {
1929   DM_Plex       *mesh = (DM_Plex*) dm->data;
1930   PetscInt       pStart, pEnd;
1931   PetscInt       dof, off;
1932   PetscErrorCode ierr;
1933 
1934   PetscFunctionBegin;
1935   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1936   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1937   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1938   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1939   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1940   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1941   mesh->coneOrientations[off+conePos] = coneOrientation;
1942   PetscFunctionReturn(0);
1943 }
1944 
1945 /*@
1946   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
1947 
1948   Not collective
1949 
1950   Input Parameters:
1951 + mesh - The DMPlex
1952 - p - The point, which must lie in the chart set with DMPlexSetChart()
1953 
1954   Output Parameter:
1955 . size - The support size for point p
1956 
1957   Level: beginner
1958 
1959 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1960 @*/
1961 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1962 {
1963   DM_Plex       *mesh = (DM_Plex*) dm->data;
1964   PetscErrorCode ierr;
1965 
1966   PetscFunctionBegin;
1967   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1968   PetscValidPointer(size, 3);
1969   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1970   PetscFunctionReturn(0);
1971 }
1972 
1973 /*@
1974   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
1975 
1976   Not collective
1977 
1978   Input Parameters:
1979 + mesh - The DMPlex
1980 . p - The point, which must lie in the chart set with DMPlexSetChart()
1981 - size - The support size for point p
1982 
1983   Output Parameter:
1984 
1985   Note:
1986   This should be called after DMPlexSetChart().
1987 
1988   Level: beginner
1989 
1990 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1991 @*/
1992 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1993 {
1994   DM_Plex       *mesh = (DM_Plex*) dm->data;
1995   PetscErrorCode ierr;
1996 
1997   PetscFunctionBegin;
1998   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1999   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2000 
2001   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2002   PetscFunctionReturn(0);
2003 }
2004 
2005 /*@C
2006   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2007 
2008   Not collective
2009 
2010   Input Parameters:
2011 + mesh - The DMPlex
2012 - p - The point, which must lie in the chart set with DMPlexSetChart()
2013 
2014   Output Parameter:
2015 . support - An array of points which are on the out-edges for point p
2016 
2017   Level: beginner
2018 
2019   Fortran Notes:
2020   Since it returns an array, this routine is only available in Fortran 90, and you must
2021   include petsc.h90 in your code.
2022 
2023   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2024 
2025 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2026 @*/
2027 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2028 {
2029   DM_Plex       *mesh = (DM_Plex*) dm->data;
2030   PetscInt       off;
2031   PetscErrorCode ierr;
2032 
2033   PetscFunctionBegin;
2034   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2035   PetscValidPointer(support, 3);
2036   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2037   *support = &mesh->supports[off];
2038   PetscFunctionReturn(0);
2039 }
2040 
2041 /*@
2042   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2043 
2044   Not collective
2045 
2046   Input Parameters:
2047 + mesh - The DMPlex
2048 . p - The point, which must lie in the chart set with DMPlexSetChart()
2049 - support - An array of points which are on the out-edges for point p
2050 
2051   Output Parameter:
2052 
2053   Note:
2054   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2055 
2056   Level: beginner
2057 
2058 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2059 @*/
2060 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2061 {
2062   DM_Plex       *mesh = (DM_Plex*) dm->data;
2063   PetscInt       pStart, pEnd;
2064   PetscInt       dof, off, c;
2065   PetscErrorCode ierr;
2066 
2067   PetscFunctionBegin;
2068   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2069   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2070   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2071   if (dof) PetscValidPointer(support, 3);
2072   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2073   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2074   for (c = 0; c < dof; ++c) {
2075     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2076     mesh->supports[off+c] = support[c];
2077   }
2078   PetscFunctionReturn(0);
2079 }
2080 
2081 /*@
2082   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2083 
2084   Not collective
2085 
2086   Input Parameters:
2087 + mesh - The DMPlex
2088 . p - The point, which must lie in the chart set with DMPlexSetChart()
2089 . supportPos - The local index in the cone where the point should be put
2090 - supportPoint - The mesh point to insert
2091 
2092   Level: beginner
2093 
2094 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2095 @*/
2096 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2097 {
2098   DM_Plex       *mesh = (DM_Plex*) dm->data;
2099   PetscInt       pStart, pEnd;
2100   PetscInt       dof, off;
2101   PetscErrorCode ierr;
2102 
2103   PetscFunctionBegin;
2104   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2105   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2106   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2107   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2108   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2109   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2110   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2111   mesh->supports[off+supportPos] = supportPoint;
2112   PetscFunctionReturn(0);
2113 }
2114 
2115 /*@C
2116   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2117 
2118   Not collective
2119 
2120   Input Parameters:
2121 + mesh - The DMPlex
2122 . p - The point, which must lie in the chart set with DMPlexSetChart()
2123 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2124 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2125 
2126   Output Parameters:
2127 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2128 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2129 
2130   Note:
2131   If using internal storage (points is NULL on input), each call overwrites the last output.
2132 
2133   Fortran Notes:
2134   Since it returns an array, this routine is only available in Fortran 90, and you must
2135   include petsc.h90 in your code.
2136 
2137   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2138 
2139   Level: beginner
2140 
2141 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2142 @*/
2143 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2144 {
2145   DM_Plex        *mesh = (DM_Plex*) dm->data;
2146   PetscInt       *closure, *fifo;
2147   const PetscInt *tmp = NULL, *tmpO = NULL;
2148   PetscInt        tmpSize, t;
2149   PetscInt        depth       = 0, maxSize;
2150   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2151   PetscErrorCode  ierr;
2152 
2153   PetscFunctionBegin;
2154   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2155   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2156   /* This is only 1-level */
2157   if (useCone) {
2158     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2159     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2160     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2161   } else {
2162     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2163     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2164   }
2165   if (depth == 1) {
2166     if (*points) {
2167       closure = *points;
2168     } else {
2169       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2170       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2171     }
2172     closure[0] = p; closure[1] = 0;
2173     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2174       closure[closureSize]   = tmp[t];
2175       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2176     }
2177     if (numPoints) *numPoints = closureSize/2;
2178     if (points)    *points    = closure;
2179     PetscFunctionReturn(0);
2180   }
2181   {
2182     PetscInt c, coneSeries, s,supportSeries;
2183 
2184     c = mesh->maxConeSize;
2185     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2186     s = mesh->maxSupportSize;
2187     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2188     maxSize = 2*PetscMax(coneSeries,supportSeries);
2189   }
2190   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2191   if (*points) {
2192     closure = *points;
2193   } else {
2194     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2195   }
2196   closure[0] = p; closure[1] = 0;
2197   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2198     const PetscInt cp = tmp[t];
2199     const PetscInt co = tmpO ? tmpO[t] : 0;
2200 
2201     closure[closureSize]   = cp;
2202     closure[closureSize+1] = co;
2203     fifo[fifoSize]         = cp;
2204     fifo[fifoSize+1]       = co;
2205   }
2206   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2207   while (fifoSize - fifoStart) {
2208     const PetscInt q   = fifo[fifoStart];
2209     const PetscInt o   = fifo[fifoStart+1];
2210     const PetscInt rev = o >= 0 ? 0 : 1;
2211     const PetscInt off = rev ? -(o+1) : o;
2212 
2213     if (useCone) {
2214       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2215       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2216       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2217     } else {
2218       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2219       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2220       tmpO = NULL;
2221     }
2222     for (t = 0; t < tmpSize; ++t) {
2223       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2224       const PetscInt cp = tmp[i];
2225       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2226       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2227        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2228       PetscInt       co = tmpO ? tmpO[i] : 0;
2229       PetscInt       c;
2230 
2231       if (rev) {
2232         PetscInt childSize, coff;
2233         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2234         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2235         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2236       }
2237       /* Check for duplicate */
2238       for (c = 0; c < closureSize; c += 2) {
2239         if (closure[c] == cp) break;
2240       }
2241       if (c == closureSize) {
2242         closure[closureSize]   = cp;
2243         closure[closureSize+1] = co;
2244         fifo[fifoSize]         = cp;
2245         fifo[fifoSize+1]       = co;
2246         closureSize           += 2;
2247         fifoSize              += 2;
2248       }
2249     }
2250     fifoStart += 2;
2251   }
2252   if (numPoints) *numPoints = closureSize/2;
2253   if (points)    *points    = closure;
2254   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2255   PetscFunctionReturn(0);
2256 }
2257 
2258 /*@C
2259   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2260 
2261   Not collective
2262 
2263   Input Parameters:
2264 + mesh - The DMPlex
2265 . p - The point, which must lie in the chart set with DMPlexSetChart()
2266 . orientation - The orientation of the point
2267 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2268 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2269 
2270   Output Parameters:
2271 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2272 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2273 
2274   Note:
2275   If using internal storage (points is NULL on input), each call overwrites the last output.
2276 
2277   Fortran Notes:
2278   Since it returns an array, this routine is only available in Fortran 90, and you must
2279   include petsc.h90 in your code.
2280 
2281   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2282 
2283   Level: beginner
2284 
2285 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2286 @*/
2287 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2288 {
2289   DM_Plex        *mesh = (DM_Plex*) dm->data;
2290   PetscInt       *closure, *fifo;
2291   const PetscInt *tmp = NULL, *tmpO = NULL;
2292   PetscInt        tmpSize, t;
2293   PetscInt        depth       = 0, maxSize;
2294   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2295   PetscErrorCode  ierr;
2296 
2297   PetscFunctionBegin;
2298   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2299   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2300   /* This is only 1-level */
2301   if (useCone) {
2302     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2303     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2304     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2305   } else {
2306     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2307     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2308   }
2309   if (depth == 1) {
2310     if (*points) {
2311       closure = *points;
2312     } else {
2313       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2314       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2315     }
2316     closure[0] = p; closure[1] = ornt;
2317     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2318       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2319       closure[closureSize]   = tmp[i];
2320       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2321     }
2322     if (numPoints) *numPoints = closureSize/2;
2323     if (points)    *points    = closure;
2324     PetscFunctionReturn(0);
2325   }
2326   {
2327     PetscInt c, coneSeries, s,supportSeries;
2328 
2329     c = mesh->maxConeSize;
2330     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2331     s = mesh->maxSupportSize;
2332     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2333     maxSize = 2*PetscMax(coneSeries,supportSeries);
2334   }
2335   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2336   if (*points) {
2337     closure = *points;
2338   } else {
2339     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2340   }
2341   closure[0] = p; closure[1] = ornt;
2342   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2343     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2344     const PetscInt cp = tmp[i];
2345     PetscInt       co = tmpO ? tmpO[i] : 0;
2346 
2347     if (ornt < 0) {
2348       PetscInt childSize, coff;
2349       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2350       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2351       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2352     }
2353     closure[closureSize]   = cp;
2354     closure[closureSize+1] = co;
2355     fifo[fifoSize]         = cp;
2356     fifo[fifoSize+1]       = co;
2357   }
2358   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2359   while (fifoSize - fifoStart) {
2360     const PetscInt q   = fifo[fifoStart];
2361     const PetscInt o   = fifo[fifoStart+1];
2362     const PetscInt rev = o >= 0 ? 0 : 1;
2363     const PetscInt off = rev ? -(o+1) : o;
2364 
2365     if (useCone) {
2366       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2367       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2368       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2369     } else {
2370       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2371       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2372       tmpO = NULL;
2373     }
2374     for (t = 0; t < tmpSize; ++t) {
2375       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2376       const PetscInt cp = tmp[i];
2377       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2378       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2379        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2380       PetscInt       co = tmpO ? tmpO[i] : 0;
2381       PetscInt       c;
2382 
2383       if (rev) {
2384         PetscInt childSize, coff;
2385         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2386         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2387         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2388       }
2389       /* Check for duplicate */
2390       for (c = 0; c < closureSize; c += 2) {
2391         if (closure[c] == cp) break;
2392       }
2393       if (c == closureSize) {
2394         closure[closureSize]   = cp;
2395         closure[closureSize+1] = co;
2396         fifo[fifoSize]         = cp;
2397         fifo[fifoSize+1]       = co;
2398         closureSize           += 2;
2399         fifoSize              += 2;
2400       }
2401     }
2402     fifoStart += 2;
2403   }
2404   if (numPoints) *numPoints = closureSize/2;
2405   if (points)    *points    = closure;
2406   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2407   PetscFunctionReturn(0);
2408 }
2409 
2410 /*@C
2411   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2412 
2413   Not collective
2414 
2415   Input Parameters:
2416 + mesh - The DMPlex
2417 . p - The point, which must lie in the chart set with DMPlexSetChart()
2418 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2419 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2420 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2421 
2422   Note:
2423   If not using internal storage (points is not NULL on input), this call is unnecessary
2424 
2425   Fortran Notes:
2426   Since it returns an array, this routine is only available in Fortran 90, and you must
2427   include petsc.h90 in your code.
2428 
2429   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2430 
2431   Level: beginner
2432 
2433 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2434 @*/
2435 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2436 {
2437   PetscErrorCode ierr;
2438 
2439   PetscFunctionBegin;
2440   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2441   if (numPoints) PetscValidIntPointer(numPoints,4);
2442   if (points) PetscValidPointer(points,5);
2443   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2444   if (numPoints) *numPoints = 0;
2445   PetscFunctionReturn(0);
2446 }
2447 
2448 /*@
2449   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2450 
2451   Not collective
2452 
2453   Input Parameter:
2454 . mesh - The DMPlex
2455 
2456   Output Parameters:
2457 + maxConeSize - The maximum number of in-edges
2458 - maxSupportSize - The maximum number of out-edges
2459 
2460   Level: beginner
2461 
2462 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2463 @*/
2464 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2465 {
2466   DM_Plex *mesh = (DM_Plex*) dm->data;
2467 
2468   PetscFunctionBegin;
2469   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2470   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2471   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2472   PetscFunctionReturn(0);
2473 }
2474 
2475 PetscErrorCode DMSetUp_Plex(DM dm)
2476 {
2477   DM_Plex       *mesh = (DM_Plex*) dm->data;
2478   PetscInt       size;
2479   PetscErrorCode ierr;
2480 
2481   PetscFunctionBegin;
2482   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2483   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2484   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2485   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2486   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2487   if (mesh->maxSupportSize) {
2488     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2489     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2490     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2491   }
2492   PetscFunctionReturn(0);
2493 }
2494 
2495 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2496 {
2497   PetscErrorCode ierr;
2498 
2499   PetscFunctionBegin;
2500   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2501   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2502   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2503   if (dm->useNatural && dm->sfMigration) {
2504     PetscSF        sfMigrationInv,sfNatural;
2505     PetscSection   section, sectionSeq;
2506 
2507     (*subdm)->sfMigration = dm->sfMigration;
2508     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2509     ierr = DMGetSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2510     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2511     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2512     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2513 
2514     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2515     (*subdm)->sfNatural = sfNatural;
2516     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2517     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2518   }
2519   PetscFunctionReturn(0);
2520 }
2521 
2522 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2523 {
2524   PetscErrorCode ierr;
2525   PetscInt       i = 0;
2526 
2527   PetscFunctionBegin;
2528   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2529   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2530   (*superdm)->useNatural = PETSC_FALSE;
2531   for (i = 0; i < len; i++){
2532     if (dms[i]->useNatural && dms[i]->sfMigration) {
2533       PetscSF        sfMigrationInv,sfNatural;
2534       PetscSection   section, sectionSeq;
2535 
2536       (*superdm)->sfMigration = dms[i]->sfMigration;
2537       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2538       (*superdm)->useNatural = PETSC_TRUE;
2539       ierr = DMGetSection((*superdm), &section);CHKERRQ(ierr);
2540       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2541       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2542       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2543 
2544       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2545       (*superdm)->sfNatural = sfNatural;
2546       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2547       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2548       break;
2549     }
2550   }
2551   PetscFunctionReturn(0);
2552 }
2553 
2554 /*@
2555   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2556 
2557   Not collective
2558 
2559   Input Parameter:
2560 . mesh - The DMPlex
2561 
2562   Output Parameter:
2563 
2564   Note:
2565   This should be called after all calls to DMPlexSetCone()
2566 
2567   Level: beginner
2568 
2569 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2570 @*/
2571 PetscErrorCode DMPlexSymmetrize(DM dm)
2572 {
2573   DM_Plex       *mesh = (DM_Plex*) dm->data;
2574   PetscInt      *offsets;
2575   PetscInt       supportSize;
2576   PetscInt       pStart, pEnd, p;
2577   PetscErrorCode ierr;
2578 
2579   PetscFunctionBegin;
2580   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2581   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2582   /* Calculate support sizes */
2583   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2584   for (p = pStart; p < pEnd; ++p) {
2585     PetscInt dof, off, c;
2586 
2587     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2588     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2589     for (c = off; c < off+dof; ++c) {
2590       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2591     }
2592   }
2593   for (p = pStart; p < pEnd; ++p) {
2594     PetscInt dof;
2595 
2596     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2597 
2598     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2599   }
2600   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2601   /* Calculate supports */
2602   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2603   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2604   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2605   for (p = pStart; p < pEnd; ++p) {
2606     PetscInt dof, off, c;
2607 
2608     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2609     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2610     for (c = off; c < off+dof; ++c) {
2611       const PetscInt q = mesh->cones[c];
2612       PetscInt       offS;
2613 
2614       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2615 
2616       mesh->supports[offS+offsets[q]] = p;
2617       ++offsets[q];
2618     }
2619   }
2620   ierr = PetscFree(offsets);CHKERRQ(ierr);
2621   PetscFunctionReturn(0);
2622 }
2623 
2624 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2625 {
2626   IS             stratumIS;
2627   PetscErrorCode ierr;
2628 
2629   PetscFunctionBegin;
2630   if (pStart >= pEnd) PetscFunctionReturn(0);
2631 #if defined(PETSC_USE_DEBUG)
2632   {
2633     PetscInt  qStart, qEnd, numLevels, level;
2634     PetscBool overlap = PETSC_FALSE;
2635     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2636     for (level = 0; level < numLevels; level++) {
2637       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2638       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2639     }
2640     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2641   }
2642 #endif
2643   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2644   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2645   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2646   PetscFunctionReturn(0);
2647 }
2648 
2649 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2650 
2651 /*@
2652   DMPlexStratify - The DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
2653   can be illustrated by a Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2654   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2655   the DAG.
2656 
2657   Collective on dm
2658 
2659   Input Parameter:
2660 . mesh - The DMPlex
2661 
2662   Output Parameter:
2663 
2664   Notes:
2665   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2666   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2667   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2668   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2669 
2670   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2671 
2672   Level: beginner
2673 
2674 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2675 @*/
2676 PetscErrorCode DMPlexStratify(DM dm)
2677 {
2678   DM_Plex       *mesh = (DM_Plex*) dm->data;
2679   DMLabel        label;
2680   PetscInt       pStart, pEnd, p;
2681   PetscInt       numRoots = 0, numLeaves = 0;
2682   PetscInt       cMax, fMax, eMax, vMax;
2683   PetscErrorCode ierr;
2684 
2685   PetscFunctionBegin;
2686   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2687   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2688 
2689   /* Create depth label */
2690   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2691   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2692   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2693 
2694   {
2695     /* Initialize roots and count leaves */
2696     PetscInt sMin = PETSC_MAX_INT;
2697     PetscInt sMax = PETSC_MIN_INT;
2698     PetscInt coneSize, supportSize;
2699 
2700     for (p = pStart; p < pEnd; ++p) {
2701       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2702       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2703       if (!coneSize && supportSize) {
2704         sMin = PetscMin(p, sMin);
2705         sMax = PetscMax(p, sMax);
2706         ++numRoots;
2707       } else if (!supportSize && coneSize) {
2708         ++numLeaves;
2709       } else if (!supportSize && !coneSize) {
2710         /* Isolated points */
2711         sMin = PetscMin(p, sMin);
2712         sMax = PetscMax(p, sMax);
2713       }
2714     }
2715     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2716   }
2717 
2718   if (numRoots + numLeaves == (pEnd - pStart)) {
2719     PetscInt sMin = PETSC_MAX_INT;
2720     PetscInt sMax = PETSC_MIN_INT;
2721     PetscInt coneSize, supportSize;
2722 
2723     for (p = pStart; p < pEnd; ++p) {
2724       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2725       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2726       if (!supportSize && coneSize) {
2727         sMin = PetscMin(p, sMin);
2728         sMax = PetscMax(p, sMax);
2729       }
2730     }
2731     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2732   } else {
2733     PetscInt level = 0;
2734     PetscInt qStart, qEnd, q;
2735 
2736     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2737     while (qEnd > qStart) {
2738       PetscInt sMin = PETSC_MAX_INT;
2739       PetscInt sMax = PETSC_MIN_INT;
2740 
2741       for (q = qStart; q < qEnd; ++q) {
2742         const PetscInt *support;
2743         PetscInt        supportSize, s;
2744 
2745         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2746         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2747         for (s = 0; s < supportSize; ++s) {
2748           sMin = PetscMin(support[s], sMin);
2749           sMax = PetscMax(support[s], sMax);
2750         }
2751       }
2752       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2753       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2754       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2755     }
2756   }
2757   { /* just in case there is an empty process */
2758     PetscInt numValues, maxValues = 0, v;
2759 
2760     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2761     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2762     for (v = numValues; v < maxValues; v++) {
2763       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2764     }
2765   }
2766   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2767 
2768   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2769   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2770     PetscInt dim;
2771     DMLabel  dimLabel;
2772 
2773     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2774     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2775     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2776     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2777     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2778     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2779     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2780   }
2781   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2782   PetscFunctionReturn(0);
2783 }
2784 
2785 /*@C
2786   DMPlexGetJoin - Get an array for the join of the set of points
2787 
2788   Not Collective
2789 
2790   Input Parameters:
2791 + dm - The DMPlex object
2792 . numPoints - The number of input points for the join
2793 - points - The input points
2794 
2795   Output Parameters:
2796 + numCoveredPoints - The number of points in the join
2797 - coveredPoints - The points in the join
2798 
2799   Level: intermediate
2800 
2801   Note: Currently, this is restricted to a single level join
2802 
2803   Fortran Notes:
2804   Since it returns an array, this routine is only available in Fortran 90, and you must
2805   include petsc.h90 in your code.
2806 
2807   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2808 
2809 .keywords: mesh
2810 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2811 @*/
2812 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2813 {
2814   DM_Plex       *mesh = (DM_Plex*) dm->data;
2815   PetscInt      *join[2];
2816   PetscInt       joinSize, i = 0;
2817   PetscInt       dof, off, p, c, m;
2818   PetscErrorCode ierr;
2819 
2820   PetscFunctionBegin;
2821   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2822   PetscValidIntPointer(points, 3);
2823   PetscValidIntPointer(numCoveredPoints, 4);
2824   PetscValidPointer(coveredPoints, 5);
2825   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2826   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2827   /* Copy in support of first point */
2828   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2829   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2830   for (joinSize = 0; joinSize < dof; ++joinSize) {
2831     join[i][joinSize] = mesh->supports[off+joinSize];
2832   }
2833   /* Check each successive support */
2834   for (p = 1; p < numPoints; ++p) {
2835     PetscInt newJoinSize = 0;
2836 
2837     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2838     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2839     for (c = 0; c < dof; ++c) {
2840       const PetscInt point = mesh->supports[off+c];
2841 
2842       for (m = 0; m < joinSize; ++m) {
2843         if (point == join[i][m]) {
2844           join[1-i][newJoinSize++] = point;
2845           break;
2846         }
2847       }
2848     }
2849     joinSize = newJoinSize;
2850     i        = 1-i;
2851   }
2852   *numCoveredPoints = joinSize;
2853   *coveredPoints    = join[i];
2854   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
2855   PetscFunctionReturn(0);
2856 }
2857 
2858 /*@C
2859   DMPlexRestoreJoin - Restore an array for the join of the set of points
2860 
2861   Not Collective
2862 
2863   Input Parameters:
2864 + dm - The DMPlex object
2865 . numPoints - The number of input points for the join
2866 - points - The input points
2867 
2868   Output Parameters:
2869 + numCoveredPoints - The number of points in the join
2870 - coveredPoints - The points in the join
2871 
2872   Fortran Notes:
2873   Since it returns an array, this routine is only available in Fortran 90, and you must
2874   include petsc.h90 in your code.
2875 
2876   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2877 
2878   Level: intermediate
2879 
2880 .keywords: mesh
2881 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2882 @*/
2883 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2884 {
2885   PetscErrorCode ierr;
2886 
2887   PetscFunctionBegin;
2888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2889   if (points) PetscValidIntPointer(points,3);
2890   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2891   PetscValidPointer(coveredPoints, 5);
2892   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
2893   if (numCoveredPoints) *numCoveredPoints = 0;
2894   PetscFunctionReturn(0);
2895 }
2896 
2897 /*@C
2898   DMPlexGetFullJoin - Get an array for the join of the set of points
2899 
2900   Not Collective
2901 
2902   Input Parameters:
2903 + dm - The DMPlex object
2904 . numPoints - The number of input points for the join
2905 - points - The input points
2906 
2907   Output Parameters:
2908 + numCoveredPoints - The number of points in the join
2909 - coveredPoints - The points in the join
2910 
2911   Fortran Notes:
2912   Since it returns an array, this routine is only available in Fortran 90, and you must
2913   include petsc.h90 in your code.
2914 
2915   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2916 
2917   Level: intermediate
2918 
2919 .keywords: mesh
2920 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2921 @*/
2922 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2923 {
2924   DM_Plex       *mesh = (DM_Plex*) dm->data;
2925   PetscInt      *offsets, **closures;
2926   PetscInt      *join[2];
2927   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2928   PetscInt       p, d, c, m, ms;
2929   PetscErrorCode ierr;
2930 
2931   PetscFunctionBegin;
2932   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2933   PetscValidIntPointer(points, 3);
2934   PetscValidIntPointer(numCoveredPoints, 4);
2935   PetscValidPointer(coveredPoints, 5);
2936 
2937   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2938   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2939   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2940   ms      = mesh->maxSupportSize;
2941   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
2942   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2943   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2944 
2945   for (p = 0; p < numPoints; ++p) {
2946     PetscInt closureSize;
2947 
2948     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2949 
2950     offsets[p*(depth+2)+0] = 0;
2951     for (d = 0; d < depth+1; ++d) {
2952       PetscInt pStart, pEnd, i;
2953 
2954       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2955       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2956         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2957           offsets[p*(depth+2)+d+1] = i;
2958           break;
2959         }
2960       }
2961       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2962     }
2963     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2964   }
2965   for (d = 0; d < depth+1; ++d) {
2966     PetscInt dof;
2967 
2968     /* Copy in support of first point */
2969     dof = offsets[d+1] - offsets[d];
2970     for (joinSize = 0; joinSize < dof; ++joinSize) {
2971       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2972     }
2973     /* Check each successive cone */
2974     for (p = 1; p < numPoints && joinSize; ++p) {
2975       PetscInt newJoinSize = 0;
2976 
2977       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2978       for (c = 0; c < dof; ++c) {
2979         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2980 
2981         for (m = 0; m < joinSize; ++m) {
2982           if (point == join[i][m]) {
2983             join[1-i][newJoinSize++] = point;
2984             break;
2985           }
2986         }
2987       }
2988       joinSize = newJoinSize;
2989       i        = 1-i;
2990     }
2991     if (joinSize) break;
2992   }
2993   *numCoveredPoints = joinSize;
2994   *coveredPoints    = join[i];
2995   for (p = 0; p < numPoints; ++p) {
2996     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2997   }
2998   ierr = PetscFree(closures);CHKERRQ(ierr);
2999   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3000   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3001   PetscFunctionReturn(0);
3002 }
3003 
3004 /*@C
3005   DMPlexGetMeet - Get an array for the meet of the set of points
3006 
3007   Not Collective
3008 
3009   Input Parameters:
3010 + dm - The DMPlex object
3011 . numPoints - The number of input points for the meet
3012 - points - The input points
3013 
3014   Output Parameters:
3015 + numCoveredPoints - The number of points in the meet
3016 - coveredPoints - The points in the meet
3017 
3018   Level: intermediate
3019 
3020   Note: Currently, this is restricted to a single level meet
3021 
3022   Fortran Notes:
3023   Since it returns an array, this routine is only available in Fortran 90, and you must
3024   include petsc.h90 in your code.
3025 
3026   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3027 
3028 .keywords: mesh
3029 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3030 @*/
3031 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3032 {
3033   DM_Plex       *mesh = (DM_Plex*) dm->data;
3034   PetscInt      *meet[2];
3035   PetscInt       meetSize, i = 0;
3036   PetscInt       dof, off, p, c, m;
3037   PetscErrorCode ierr;
3038 
3039   PetscFunctionBegin;
3040   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3041   PetscValidPointer(points, 2);
3042   PetscValidPointer(numCoveringPoints, 3);
3043   PetscValidPointer(coveringPoints, 4);
3044   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3045   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3046   /* Copy in cone of first point */
3047   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3048   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3049   for (meetSize = 0; meetSize < dof; ++meetSize) {
3050     meet[i][meetSize] = mesh->cones[off+meetSize];
3051   }
3052   /* Check each successive cone */
3053   for (p = 1; p < numPoints; ++p) {
3054     PetscInt newMeetSize = 0;
3055 
3056     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3057     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3058     for (c = 0; c < dof; ++c) {
3059       const PetscInt point = mesh->cones[off+c];
3060 
3061       for (m = 0; m < meetSize; ++m) {
3062         if (point == meet[i][m]) {
3063           meet[1-i][newMeetSize++] = point;
3064           break;
3065         }
3066       }
3067     }
3068     meetSize = newMeetSize;
3069     i        = 1-i;
3070   }
3071   *numCoveringPoints = meetSize;
3072   *coveringPoints    = meet[i];
3073   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3074   PetscFunctionReturn(0);
3075 }
3076 
3077 /*@C
3078   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3079 
3080   Not Collective
3081 
3082   Input Parameters:
3083 + dm - The DMPlex object
3084 . numPoints - The number of input points for the meet
3085 - points - The input points
3086 
3087   Output Parameters:
3088 + numCoveredPoints - The number of points in the meet
3089 - coveredPoints - The points in the meet
3090 
3091   Level: intermediate
3092 
3093   Fortran Notes:
3094   Since it returns an array, this routine is only available in Fortran 90, and you must
3095   include petsc.h90 in your code.
3096 
3097   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3098 
3099 .keywords: mesh
3100 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3101 @*/
3102 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3103 {
3104   PetscErrorCode ierr;
3105 
3106   PetscFunctionBegin;
3107   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3108   if (points) PetscValidIntPointer(points,3);
3109   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3110   PetscValidPointer(coveredPoints,5);
3111   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3112   if (numCoveredPoints) *numCoveredPoints = 0;
3113   PetscFunctionReturn(0);
3114 }
3115 
3116 /*@C
3117   DMPlexGetFullMeet - Get an array for the meet of the set of points
3118 
3119   Not Collective
3120 
3121   Input Parameters:
3122 + dm - The DMPlex object
3123 . numPoints - The number of input points for the meet
3124 - points - The input points
3125 
3126   Output Parameters:
3127 + numCoveredPoints - The number of points in the meet
3128 - coveredPoints - The points in the meet
3129 
3130   Level: intermediate
3131 
3132   Fortran Notes:
3133   Since it returns an array, this routine is only available in Fortran 90, and you must
3134   include petsc.h90 in your code.
3135 
3136   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3137 
3138 .keywords: mesh
3139 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3140 @*/
3141 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3142 {
3143   DM_Plex       *mesh = (DM_Plex*) dm->data;
3144   PetscInt      *offsets, **closures;
3145   PetscInt      *meet[2];
3146   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3147   PetscInt       p, h, c, m, mc;
3148   PetscErrorCode ierr;
3149 
3150   PetscFunctionBegin;
3151   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3152   PetscValidPointer(points, 2);
3153   PetscValidPointer(numCoveredPoints, 3);
3154   PetscValidPointer(coveredPoints, 4);
3155 
3156   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3157   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3158   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3159   mc      = mesh->maxConeSize;
3160   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3161   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3162   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3163 
3164   for (p = 0; p < numPoints; ++p) {
3165     PetscInt closureSize;
3166 
3167     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3168 
3169     offsets[p*(height+2)+0] = 0;
3170     for (h = 0; h < height+1; ++h) {
3171       PetscInt pStart, pEnd, i;
3172 
3173       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3174       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3175         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3176           offsets[p*(height+2)+h+1] = i;
3177           break;
3178         }
3179       }
3180       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3181     }
3182     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3183   }
3184   for (h = 0; h < height+1; ++h) {
3185     PetscInt dof;
3186 
3187     /* Copy in cone of first point */
3188     dof = offsets[h+1] - offsets[h];
3189     for (meetSize = 0; meetSize < dof; ++meetSize) {
3190       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3191     }
3192     /* Check each successive cone */
3193     for (p = 1; p < numPoints && meetSize; ++p) {
3194       PetscInt newMeetSize = 0;
3195 
3196       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3197       for (c = 0; c < dof; ++c) {
3198         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3199 
3200         for (m = 0; m < meetSize; ++m) {
3201           if (point == meet[i][m]) {
3202             meet[1-i][newMeetSize++] = point;
3203             break;
3204           }
3205         }
3206       }
3207       meetSize = newMeetSize;
3208       i        = 1-i;
3209     }
3210     if (meetSize) break;
3211   }
3212   *numCoveredPoints = meetSize;
3213   *coveredPoints    = meet[i];
3214   for (p = 0; p < numPoints; ++p) {
3215     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3216   }
3217   ierr = PetscFree(closures);CHKERRQ(ierr);
3218   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3219   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3220   PetscFunctionReturn(0);
3221 }
3222 
3223 /*@C
3224   DMPlexEqual - Determine if two DMs have the same topology
3225 
3226   Not Collective
3227 
3228   Input Parameters:
3229 + dmA - A DMPlex object
3230 - dmB - A DMPlex object
3231 
3232   Output Parameters:
3233 . equal - PETSC_TRUE if the topologies are identical
3234 
3235   Level: intermediate
3236 
3237   Notes:
3238   We are not solving graph isomorphism, so we do not permutation.
3239 
3240 .keywords: mesh
3241 .seealso: DMPlexGetCone()
3242 @*/
3243 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3244 {
3245   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3246   PetscErrorCode ierr;
3247 
3248   PetscFunctionBegin;
3249   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3250   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3251   PetscValidPointer(equal, 3);
3252 
3253   *equal = PETSC_FALSE;
3254   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3255   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3256   if (depth != depthB) PetscFunctionReturn(0);
3257   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3258   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3259   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3260   for (p = pStart; p < pEnd; ++p) {
3261     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3262     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3263 
3264     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3265     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3266     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3267     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3268     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3269     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3270     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3271     for (c = 0; c < coneSize; ++c) {
3272       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3273       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3274     }
3275     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3276     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3277     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3278     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3279     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3280     for (s = 0; s < supportSize; ++s) {
3281       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3282     }
3283   }
3284   *equal = PETSC_TRUE;
3285   PetscFunctionReturn(0);
3286 }
3287 
3288 /*@C
3289   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3290 
3291   Not Collective
3292 
3293   Input Parameters:
3294 + dm         - The DMPlex
3295 . cellDim    - The cell dimension
3296 - numCorners - The number of vertices on a cell
3297 
3298   Output Parameters:
3299 . numFaceVertices - The number of vertices on a face
3300 
3301   Level: developer
3302 
3303   Notes:
3304   Of course this can only work for a restricted set of symmetric shapes
3305 
3306 .seealso: DMPlexGetCone()
3307 @*/
3308 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3309 {
3310   MPI_Comm       comm;
3311   PetscErrorCode ierr;
3312 
3313   PetscFunctionBegin;
3314   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3315   PetscValidPointer(numFaceVertices,3);
3316   switch (cellDim) {
3317   case 0:
3318     *numFaceVertices = 0;
3319     break;
3320   case 1:
3321     *numFaceVertices = 1;
3322     break;
3323   case 2:
3324     switch (numCorners) {
3325     case 3: /* triangle */
3326       *numFaceVertices = 2; /* Edge has 2 vertices */
3327       break;
3328     case 4: /* quadrilateral */
3329       *numFaceVertices = 2; /* Edge has 2 vertices */
3330       break;
3331     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3332       *numFaceVertices = 3; /* Edge has 3 vertices */
3333       break;
3334     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3335       *numFaceVertices = 3; /* Edge has 3 vertices */
3336       break;
3337     default:
3338       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3339     }
3340     break;
3341   case 3:
3342     switch (numCorners) {
3343     case 4: /* tetradehdron */
3344       *numFaceVertices = 3; /* Face has 3 vertices */
3345       break;
3346     case 6: /* tet cohesive cells */
3347       *numFaceVertices = 4; /* Face has 4 vertices */
3348       break;
3349     case 8: /* hexahedron */
3350       *numFaceVertices = 4; /* Face has 4 vertices */
3351       break;
3352     case 9: /* tet cohesive Lagrange cells */
3353       *numFaceVertices = 6; /* Face has 6 vertices */
3354       break;
3355     case 10: /* quadratic tetrahedron */
3356       *numFaceVertices = 6; /* Face has 6 vertices */
3357       break;
3358     case 12: /* hex cohesive Lagrange cells */
3359       *numFaceVertices = 6; /* Face has 6 vertices */
3360       break;
3361     case 18: /* quadratic tet cohesive Lagrange cells */
3362       *numFaceVertices = 6; /* Face has 6 vertices */
3363       break;
3364     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3365       *numFaceVertices = 9; /* Face has 9 vertices */
3366       break;
3367     default:
3368       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3369     }
3370     break;
3371   default:
3372     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3373   }
3374   PetscFunctionReturn(0);
3375 }
3376 
3377 /*@
3378   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3379 
3380   Not Collective
3381 
3382   Input Parameter:
3383 . dm    - The DMPlex object
3384 
3385   Output Parameter:
3386 . depthLabel - The DMLabel recording point depth
3387 
3388   Level: developer
3389 
3390 .keywords: mesh, points
3391 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3392 @*/
3393 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3394 {
3395   PetscErrorCode ierr;
3396 
3397   PetscFunctionBegin;
3398   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3399   PetscValidPointer(depthLabel, 2);
3400   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3401   *depthLabel = dm->depthLabel;
3402   PetscFunctionReturn(0);
3403 }
3404 
3405 /*@
3406   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3407 
3408   Not Collective
3409 
3410   Input Parameter:
3411 . dm    - The DMPlex object
3412 
3413   Output Parameter:
3414 . depth - The number of strata (breadth first levels) in the DAG
3415 
3416   Level: developer
3417 
3418 .keywords: mesh, points
3419 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3420 @*/
3421 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3422 {
3423   DMLabel        label;
3424   PetscInt       d = 0;
3425   PetscErrorCode ierr;
3426 
3427   PetscFunctionBegin;
3428   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3429   PetscValidPointer(depth, 2);
3430   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3431   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3432   *depth = d-1;
3433   PetscFunctionReturn(0);
3434 }
3435 
3436 /*@
3437   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3438 
3439   Not Collective
3440 
3441   Input Parameters:
3442 + dm           - The DMPlex object
3443 - stratumValue - The requested depth
3444 
3445   Output Parameters:
3446 + start - The first point at this depth
3447 - end   - One beyond the last point at this depth
3448 
3449   Notes:
3450   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3451   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3452   higher dimension, e.g., "edges".
3453 
3454   Level: developer
3455 
3456 .keywords: mesh, points
3457 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3458 @*/
3459 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3460 {
3461   DMLabel        label;
3462   PetscInt       pStart, pEnd;
3463   PetscErrorCode ierr;
3464 
3465   PetscFunctionBegin;
3466   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3467   if (start) {PetscValidPointer(start, 3); *start = 0;}
3468   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3469   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3470   if (pStart == pEnd) PetscFunctionReturn(0);
3471   if (stratumValue < 0) {
3472     if (start) *start = pStart;
3473     if (end)   *end   = pEnd;
3474     PetscFunctionReturn(0);
3475   }
3476   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3477   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3478   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3479   PetscFunctionReturn(0);
3480 }
3481 
3482 /*@
3483   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3484 
3485   Not Collective
3486 
3487   Input Parameters:
3488 + dm           - The DMPlex object
3489 - stratumValue - The requested height
3490 
3491   Output Parameters:
3492 + start - The first point at this height
3493 - end   - One beyond the last point at this height
3494 
3495   Notes:
3496   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3497   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3498   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3499 
3500   Level: developer
3501 
3502 .keywords: mesh, points
3503 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3504 @*/
3505 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3506 {
3507   DMLabel        label;
3508   PetscInt       depth, pStart, pEnd;
3509   PetscErrorCode ierr;
3510 
3511   PetscFunctionBegin;
3512   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3513   if (start) {PetscValidPointer(start, 3); *start = 0;}
3514   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3515   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3516   if (pStart == pEnd) PetscFunctionReturn(0);
3517   if (stratumValue < 0) {
3518     if (start) *start = pStart;
3519     if (end)   *end   = pEnd;
3520     PetscFunctionReturn(0);
3521   }
3522   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3523   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3524   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3525   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3526   PetscFunctionReturn(0);
3527 }
3528 
3529 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3530 {
3531   PetscSection   section, s;
3532   Mat            m;
3533   PetscInt       maxHeight;
3534   PetscErrorCode ierr;
3535 
3536   PetscFunctionBegin;
3537   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3538   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3539   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3540   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3541   ierr = DMSetSection(*cdm, section);CHKERRQ(ierr);
3542   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3543   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3544   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3545   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3546   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3547   ierr = MatDestroy(&m);CHKERRQ(ierr);
3548 
3549   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3550   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3551   PetscFunctionReturn(0);
3552 }
3553 
3554 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3555 {
3556   Vec            coordsLocal;
3557   DM             coordsDM;
3558   PetscErrorCode ierr;
3559 
3560   PetscFunctionBegin;
3561   *field = NULL;
3562   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3563   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3564   if (coordsLocal && coordsDM) {
3565     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3566   }
3567   PetscFunctionReturn(0);
3568 }
3569 
3570 /*@C
3571   DMPlexGetConeSection - Return a section which describes the layout of cone data
3572 
3573   Not Collective
3574 
3575   Input Parameters:
3576 . dm        - The DMPlex object
3577 
3578   Output Parameter:
3579 . section - The PetscSection object
3580 
3581   Level: developer
3582 
3583 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3584 @*/
3585 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3586 {
3587   DM_Plex *mesh = (DM_Plex*) dm->data;
3588 
3589   PetscFunctionBegin;
3590   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3591   if (section) *section = mesh->coneSection;
3592   PetscFunctionReturn(0);
3593 }
3594 
3595 /*@C
3596   DMPlexGetSupportSection - Return a section which describes the layout of support data
3597 
3598   Not Collective
3599 
3600   Input Parameters:
3601 . dm        - The DMPlex object
3602 
3603   Output Parameter:
3604 . section - The PetscSection object
3605 
3606   Level: developer
3607 
3608 .seealso: DMPlexGetConeSection()
3609 @*/
3610 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3611 {
3612   DM_Plex *mesh = (DM_Plex*) dm->data;
3613 
3614   PetscFunctionBegin;
3615   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3616   if (section) *section = mesh->supportSection;
3617   PetscFunctionReturn(0);
3618 }
3619 
3620 /*@C
3621   DMPlexGetCones - Return cone data
3622 
3623   Not Collective
3624 
3625   Input Parameters:
3626 . dm        - The DMPlex object
3627 
3628   Output Parameter:
3629 . cones - The cone for each point
3630 
3631   Level: developer
3632 
3633 .seealso: DMPlexGetConeSection()
3634 @*/
3635 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3636 {
3637   DM_Plex *mesh = (DM_Plex*) dm->data;
3638 
3639   PetscFunctionBegin;
3640   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3641   if (cones) *cones = mesh->cones;
3642   PetscFunctionReturn(0);
3643 }
3644 
3645 /*@C
3646   DMPlexGetConeOrientations - Return cone orientation data
3647 
3648   Not Collective
3649 
3650   Input Parameters:
3651 . dm        - The DMPlex object
3652 
3653   Output Parameter:
3654 . coneOrientations - The cone orientation for each point
3655 
3656   Level: developer
3657 
3658 .seealso: DMPlexGetConeSection()
3659 @*/
3660 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3661 {
3662   DM_Plex *mesh = (DM_Plex*) dm->data;
3663 
3664   PetscFunctionBegin;
3665   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3666   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3667   PetscFunctionReturn(0);
3668 }
3669 
3670 /******************************** FEM Support **********************************/
3671 
3672 /*@
3673 
3674   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3675   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3676   section provided (or default section of the DM).
3677 
3678   Input Parameters:
3679 + dm      - The DM
3680 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3681 - section - The PetscSection to reorder, or NULL for the default section
3682 
3683   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3684   degree of the basis.
3685 
3686   Example:
3687   A typical interpolated single-quad mesh might order points as
3688 .vb
3689   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3690 
3691   v4 -- e6 -- v3
3692   |           |
3693   e7    c0    e8
3694   |           |
3695   v1 -- e5 -- v2
3696 .ve
3697 
3698   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3699   dofs in the order of points, e.g.,
3700 .vb
3701     c0 -> [0,1,2,3]
3702     v1 -> [4]
3703     ...
3704     e5 -> [8, 9]
3705 .ve
3706 
3707   which corresponds to the dofs
3708 .vb
3709     6   10  11  7
3710     13  2   3   15
3711     12  0   1   14
3712     4   8   9   5
3713 .ve
3714 
3715   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3716 .vb
3717   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3718 .ve
3719 
3720   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3721 .vb
3722    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3723 .ve
3724 
3725   Level: developer
3726 
3727 .seealso: DMGetSection(), PetscSectionSetClosurePermutation()
3728 @*/
3729 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3730 {
3731   DMLabel        label;
3732   PetscInt      *perm;
3733   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3734   PetscErrorCode ierr;
3735 
3736   PetscFunctionBegin;
3737   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3738   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3739   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3740   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3741   if (depth == 1) {eStart = point;}
3742   else if  (depth == dim) {
3743     const PetscInt *cone;
3744 
3745     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3746     if (dim == 2) eStart = cone[0];
3747     else if (dim == 3) {
3748       const PetscInt *cone2;
3749       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3750       eStart = cone2[0];
3751     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3752   } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3753   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
3754   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3755   if (dim < 1) PetscFunctionReturn(0);
3756   for (f = 0; f < Nf; ++f) {
3757     /* An order k SEM disc has k-1 dofs on an edge */
3758     ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3759     ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3760     k = k/Nc + 1;
3761     size += PetscPowInt(k+1, dim)*Nc;
3762   }
3763   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3764   for (f = 0; f < Nf; ++f) {
3765     switch (dim) {
3766     case 1:
3767       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3768       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3769       k = k/Nc + 1;
3770       /*
3771         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3772         We want              [ vtx0; edge of length k-1; vtx1 ]
3773       */
3774       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3775       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3776       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3777       foffset = offset;
3778       break;
3779     case 2:
3780       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3781       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3782       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3783       k = k/Nc + 1;
3784       /* The SEM order is
3785 
3786          v_lb, {e_b}, v_rb,
3787          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3788          v_lt, reverse {e_t}, v_rt
3789       */
3790       {
3791         const PetscInt of   = 0;
3792         const PetscInt oeb  = of   + PetscSqr(k-1);
3793         const PetscInt oer  = oeb  + (k-1);
3794         const PetscInt oet  = oer  + (k-1);
3795         const PetscInt oel  = oet  + (k-1);
3796         const PetscInt ovlb = oel  + (k-1);
3797         const PetscInt ovrb = ovlb + 1;
3798         const PetscInt ovrt = ovrb + 1;
3799         const PetscInt ovlt = ovrt + 1;
3800         PetscInt       o;
3801 
3802         /* bottom */
3803         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3804         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3805         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3806         /* middle */
3807         for (i = 0; i < k-1; ++i) {
3808           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3809           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3810           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3811         }
3812         /* top */
3813         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3814         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3815         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3816         foffset = offset;
3817       }
3818       break;
3819     case 3:
3820       /* The original hex closure is
3821 
3822          {c,
3823           f_b, f_t, f_f, f_b, f_r, f_l,
3824           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3825           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3826       */
3827       ierr = PetscSectionGetFieldDof(section, eStart, f, &k);CHKERRQ(ierr);
3828       ierr = PetscSectionGetFieldComponents(section, f, &Nc);CHKERRQ(ierr);
3829       k = k/Nc + 1;
3830       /* The SEM order is
3831          Bottom Slice
3832          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3833          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3834          v_blb, {e_bb}, v_brb,
3835 
3836          Middle Slice (j)
3837          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3838          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3839          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3840 
3841          Top Slice
3842          v_tlf, {e_tf}, v_trf,
3843          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
3844          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
3845       */
3846       {
3847         const PetscInt oc    = 0;
3848         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
3849         const PetscInt oft   = ofb   + PetscSqr(k-1);
3850         const PetscInt off   = oft   + PetscSqr(k-1);
3851         const PetscInt ofk   = off   + PetscSqr(k-1);
3852         const PetscInt ofr   = ofk   + PetscSqr(k-1);
3853         const PetscInt ofl   = ofr   + PetscSqr(k-1);
3854         const PetscInt oebl  = ofl   + PetscSqr(k-1);
3855         const PetscInt oebb  = oebl  + (k-1);
3856         const PetscInt oebr  = oebb  + (k-1);
3857         const PetscInt oebf  = oebr  + (k-1);
3858         const PetscInt oetf  = oebf  + (k-1);
3859         const PetscInt oetr  = oetf  + (k-1);
3860         const PetscInt oetb  = oetr  + (k-1);
3861         const PetscInt oetl  = oetb  + (k-1);
3862         const PetscInt oerf  = oetl  + (k-1);
3863         const PetscInt oelf  = oerf  + (k-1);
3864         const PetscInt oelb  = oelf  + (k-1);
3865         const PetscInt oerb  = oelb  + (k-1);
3866         const PetscInt ovblf = oerb  + (k-1);
3867         const PetscInt ovblb = ovblf + 1;
3868         const PetscInt ovbrb = ovblb + 1;
3869         const PetscInt ovbrf = ovbrb + 1;
3870         const PetscInt ovtlf = ovbrf + 1;
3871         const PetscInt ovtrf = ovtlf + 1;
3872         const PetscInt ovtrb = ovtrf + 1;
3873         const PetscInt ovtlb = ovtrb + 1;
3874         PetscInt       o, n;
3875 
3876         /* Bottom Slice */
3877         /*   bottom */
3878         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
3879         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3880         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
3881         /*   middle */
3882         for (i = 0; i < k-1; ++i) {
3883           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
3884           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
3885           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
3886         }
3887         /*   top */
3888         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
3889         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3890         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
3891 
3892         /* Middle Slice */
3893         for (j = 0; j < k-1; ++j) {
3894           /*   bottom */
3895           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
3896           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3897           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
3898           /*   middle */
3899           for (i = 0; i < k-1; ++i) {
3900             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
3901             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
3902             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
3903           }
3904           /*   top */
3905           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
3906           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3907           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
3908         }
3909 
3910         /* Top Slice */
3911         /*   bottom */
3912         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
3913         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3914         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
3915         /*   middle */
3916         for (i = 0; i < k-1; ++i) {
3917           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
3918           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
3919           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
3920         }
3921         /*   top */
3922         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
3923         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3924         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
3925 
3926         foffset = offset;
3927       }
3928       break;
3929     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
3930     }
3931   }
3932   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
3933   /* Check permutation */
3934   {
3935     PetscInt *check;
3936 
3937     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
3938     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
3939     for (i = 0; i < size; ++i) check[perm[i]] = i;
3940     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
3941     ierr = PetscFree(check);CHKERRQ(ierr);
3942   }
3943   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
3944   PetscFunctionReturn(0);
3945 }
3946 
3947 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
3948 {
3949   PetscDS        prob;
3950   PetscInt       depth, Nf, h;
3951   DMLabel        label;
3952   PetscErrorCode ierr;
3953 
3954   PetscFunctionBeginHot;
3955   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
3956   Nf      = prob->Nf;
3957   label   = dm->depthLabel;
3958   *dspace = NULL;
3959   if (field < Nf) {
3960     PetscObject disc = prob->disc[field];
3961 
3962     if (disc->classid == PETSCFE_CLASSID) {
3963       PetscDualSpace dsp;
3964 
3965       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
3966       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
3967       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
3968       h    = depth - 1 - h;
3969       if (h) {
3970         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
3971       } else {
3972         *dspace = dsp;
3973       }
3974     }
3975   }
3976   PetscFunctionReturn(0);
3977 }
3978 
3979 
3980 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
3981 {
3982   PetscScalar    *array, *vArray;
3983   const PetscInt *cone, *coneO;
3984   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
3985   PetscErrorCode  ierr;
3986 
3987   PetscFunctionBeginHot;
3988   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
3989   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
3990   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3991   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
3992   if (!values || !*values) {
3993     if ((point >= pStart) && (point < pEnd)) {
3994       PetscInt dof;
3995 
3996       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
3997       size += dof;
3998     }
3999     for (p = 0; p < numPoints; ++p) {
4000       const PetscInt cp = cone[p];
4001       PetscInt       dof;
4002 
4003       if ((cp < pStart) || (cp >= pEnd)) continue;
4004       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4005       size += dof;
4006     }
4007     if (!values) {
4008       if (csize) *csize = size;
4009       PetscFunctionReturn(0);
4010     }
4011     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4012   } else {
4013     array = *values;
4014   }
4015   size = 0;
4016   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4017   if ((point >= pStart) && (point < pEnd)) {
4018     PetscInt     dof, off, d;
4019     PetscScalar *varr;
4020 
4021     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4022     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4023     varr = &vArray[off];
4024     for (d = 0; d < dof; ++d, ++offset) {
4025       array[offset] = varr[d];
4026     }
4027     size += dof;
4028   }
4029   for (p = 0; p < numPoints; ++p) {
4030     const PetscInt cp = cone[p];
4031     PetscInt       o  = coneO[p];
4032     PetscInt       dof, off, d;
4033     PetscScalar   *varr;
4034 
4035     if ((cp < pStart) || (cp >= pEnd)) continue;
4036     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4037     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4038     varr = &vArray[off];
4039     if (o >= 0) {
4040       for (d = 0; d < dof; ++d, ++offset) {
4041         array[offset] = varr[d];
4042       }
4043     } else {
4044       for (d = dof-1; d >= 0; --d, ++offset) {
4045         array[offset] = varr[d];
4046       }
4047     }
4048     size += dof;
4049   }
4050   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4051   if (!*values) {
4052     if (csize) *csize = size;
4053     *values = array;
4054   } else {
4055     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4056     *csize = size;
4057   }
4058   PetscFunctionReturn(0);
4059 }
4060 
4061 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4062 {
4063   const PetscInt *cla;
4064   PetscInt       np, *pts = NULL;
4065   PetscErrorCode ierr;
4066 
4067   PetscFunctionBeginHot;
4068   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4069   if (!*clPoints) {
4070     PetscInt pStart, pEnd, p, q;
4071 
4072     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4073     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4074     /* Compress out points not in the section */
4075     for (p = 0, q = 0; p < np; p++) {
4076       PetscInt r = pts[2*p];
4077       if ((r >= pStart) && (r < pEnd)) {
4078         pts[q*2]   = r;
4079         pts[q*2+1] = pts[2*p+1];
4080         ++q;
4081       }
4082     }
4083     np = q;
4084     cla = NULL;
4085   } else {
4086     PetscInt dof, off;
4087 
4088     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4089     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4090     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4091     np   = dof/2;
4092     pts  = (PetscInt *) &cla[off];
4093   }
4094   *numPoints = np;
4095   *points    = pts;
4096   *clp       = cla;
4097 
4098   PetscFunctionReturn(0);
4099 }
4100 
4101 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4102 {
4103   PetscErrorCode ierr;
4104 
4105   PetscFunctionBeginHot;
4106   if (!*clPoints) {
4107     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4108   } else {
4109     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4110   }
4111   *numPoints = 0;
4112   *points    = NULL;
4113   *clSec     = NULL;
4114   *clPoints  = NULL;
4115   *clp       = NULL;
4116   PetscFunctionReturn(0);
4117 }
4118 
4119 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4120 {
4121   PetscInt          offset = 0, p;
4122   const PetscInt    **perms = NULL;
4123   const PetscScalar **flips = NULL;
4124   PetscErrorCode    ierr;
4125 
4126   PetscFunctionBeginHot;
4127   *size = 0;
4128   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4129   for (p = 0; p < numPoints; p++) {
4130     const PetscInt    point = points[2*p];
4131     const PetscInt    *perm = perms ? perms[p] : NULL;
4132     const PetscScalar *flip = flips ? flips[p] : NULL;
4133     PetscInt          dof, off, d;
4134     const PetscScalar *varr;
4135 
4136     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4137     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4138     varr = &vArray[off];
4139     if (clperm) {
4140       if (perm) {
4141         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4142       } else {
4143         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4144       }
4145       if (flip) {
4146         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4147       }
4148     } else {
4149       if (perm) {
4150         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4151       } else {
4152         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4153       }
4154       if (flip) {
4155         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4156       }
4157     }
4158     offset += dof;
4159   }
4160   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4161   *size = offset;
4162   PetscFunctionReturn(0);
4163 }
4164 
4165 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4166 {
4167   PetscInt          offset = 0, f;
4168   PetscErrorCode    ierr;
4169 
4170   PetscFunctionBeginHot;
4171   *size = 0;
4172   for (f = 0; f < numFields; ++f) {
4173     PetscInt          p;
4174     const PetscInt    **perms = NULL;
4175     const PetscScalar **flips = NULL;
4176 
4177     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4178     for (p = 0; p < numPoints; p++) {
4179       const PetscInt    point = points[2*p];
4180       PetscInt          fdof, foff, b;
4181       const PetscScalar *varr;
4182       const PetscInt    *perm = perms ? perms[p] : NULL;
4183       const PetscScalar *flip = flips ? flips[p] : NULL;
4184 
4185       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4186       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4187       varr = &vArray[foff];
4188       if (clperm) {
4189         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4190         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4191         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4192       } else {
4193         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4194         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4195         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4196       }
4197       offset += fdof;
4198     }
4199     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4200   }
4201   *size = offset;
4202   PetscFunctionReturn(0);
4203 }
4204 
4205 /*@C
4206   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4207 
4208   Not collective
4209 
4210   Input Parameters:
4211 + dm - The DM
4212 . section - The section describing the layout in v, or NULL to use the default section
4213 . v - The local vector
4214 . point - The point in the DM
4215 . csize - The size of the input values array, or NULL
4216 - values - An array to use for the values, or NULL to have it allocated automatically
4217 
4218   Output Parameters:
4219 + csize - The number of values in the closure
4220 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4221 
4222 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4223 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4224 $ assembly function, and a user may already have allocated storage for this operation.
4225 $
4226 $ A typical use could be
4227 $
4228 $  values = NULL;
4229 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4230 $  for (cl = 0; cl < clSize; ++cl) {
4231 $    <Compute on closure>
4232 $  }
4233 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4234 $
4235 $ or
4236 $
4237 $  PetscMalloc1(clMaxSize, &values);
4238 $  for (p = pStart; p < pEnd; ++p) {
4239 $    clSize = clMaxSize;
4240 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4241 $    for (cl = 0; cl < clSize; ++cl) {
4242 $      <Compute on closure>
4243 $    }
4244 $  }
4245 $  PetscFree(values);
4246 
4247   Fortran Notes:
4248   Since it returns an array, this routine is only available in Fortran 90, and you must
4249   include petsc.h90 in your code.
4250 
4251   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4252 
4253   Level: intermediate
4254 
4255 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4256 @*/
4257 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4258 {
4259   PetscSection       clSection;
4260   IS                 clPoints;
4261   PetscScalar       *array;
4262   const PetscScalar *vArray;
4263   PetscInt          *points = NULL;
4264   const PetscInt    *clp, *perm;
4265   PetscInt           depth, numFields, numPoints, size;
4266   PetscErrorCode     ierr;
4267 
4268   PetscFunctionBeginHot;
4269   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4270   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4271   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4272   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4273   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4274   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4275   if (depth == 1 && numFields < 2) {
4276     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4277     PetscFunctionReturn(0);
4278   }
4279   /* Get points */
4280   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4281   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4282   /* Get array */
4283   if (!values || !*values) {
4284     PetscInt asize = 0, dof, p;
4285 
4286     for (p = 0; p < numPoints*2; p += 2) {
4287       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4288       asize += dof;
4289     }
4290     if (!values) {
4291       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4292       if (csize) *csize = asize;
4293       PetscFunctionReturn(0);
4294     }
4295     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4296   } else {
4297     array = *values;
4298   }
4299   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4300   /* Get values */
4301   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4302   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4303   /* Cleanup points */
4304   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4305   /* Cleanup array */
4306   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4307   if (!*values) {
4308     if (csize) *csize = size;
4309     *values = array;
4310   } else {
4311     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4312     *csize = size;
4313   }
4314   PetscFunctionReturn(0);
4315 }
4316 
4317 /*@C
4318   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4319 
4320   Not collective
4321 
4322   Input Parameters:
4323 + dm - The DM
4324 . section - The section describing the layout in v, or NULL to use the default section
4325 . v - The local vector
4326 . point - The point in the DM
4327 . csize - The number of values in the closure, or NULL
4328 - values - The array of values, which is a borrowed array and should not be freed
4329 
4330   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4331 
4332   Fortran Notes:
4333   Since it returns an array, this routine is only available in Fortran 90, and you must
4334   include petsc.h90 in your code.
4335 
4336   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4337 
4338   Level: intermediate
4339 
4340 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4341 @*/
4342 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4343 {
4344   PetscInt       size = 0;
4345   PetscErrorCode ierr;
4346 
4347   PetscFunctionBegin;
4348   /* Should work without recalculating size */
4349   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4350   *values = NULL;
4351   PetscFunctionReturn(0);
4352 }
4353 
4354 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4355 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4356 
4357 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4358 {
4359   PetscInt        cdof;   /* The number of constraints on this point */
4360   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4361   PetscScalar    *a;
4362   PetscInt        off, cind = 0, k;
4363   PetscErrorCode  ierr;
4364 
4365   PetscFunctionBegin;
4366   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4367   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4368   a    = &array[off];
4369   if (!cdof || setBC) {
4370     if (clperm) {
4371       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4372       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4373     } else {
4374       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4375       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4376     }
4377   } else {
4378     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4379     if (clperm) {
4380       if (perm) {for (k = 0; k < dof; ++k) {
4381           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4382           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4383         }
4384       } else {
4385         for (k = 0; k < dof; ++k) {
4386           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4387           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4388         }
4389       }
4390     } else {
4391       if (perm) {
4392         for (k = 0; k < dof; ++k) {
4393           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4394           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4395         }
4396       } else {
4397         for (k = 0; k < dof; ++k) {
4398           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4399           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4400         }
4401       }
4402     }
4403   }
4404   PetscFunctionReturn(0);
4405 }
4406 
4407 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4408 {
4409   PetscInt        cdof;   /* The number of constraints on this point */
4410   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4411   PetscScalar    *a;
4412   PetscInt        off, cind = 0, k;
4413   PetscErrorCode  ierr;
4414 
4415   PetscFunctionBegin;
4416   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4417   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4418   a    = &array[off];
4419   if (cdof) {
4420     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4421     if (clperm) {
4422       if (perm) {
4423         for (k = 0; k < dof; ++k) {
4424           if ((cind < cdof) && (k == cdofs[cind])) {
4425             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4426             cind++;
4427           }
4428         }
4429       } else {
4430         for (k = 0; k < dof; ++k) {
4431           if ((cind < cdof) && (k == cdofs[cind])) {
4432             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4433             cind++;
4434           }
4435         }
4436       }
4437     } else {
4438       if (perm) {
4439         for (k = 0; k < dof; ++k) {
4440           if ((cind < cdof) && (k == cdofs[cind])) {
4441             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4442             cind++;
4443           }
4444         }
4445       } else {
4446         for (k = 0; k < dof; ++k) {
4447           if ((cind < cdof) && (k == cdofs[cind])) {
4448             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4449             cind++;
4450           }
4451         }
4452       }
4453     }
4454   }
4455   PetscFunctionReturn(0);
4456 }
4457 
4458 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4459 {
4460   PetscScalar    *a;
4461   PetscInt        fdof, foff, fcdof, foffset = *offset;
4462   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4463   PetscInt        cind = 0, b;
4464   PetscErrorCode  ierr;
4465 
4466   PetscFunctionBegin;
4467   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4468   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4469   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4470   a    = &array[foff];
4471   if (!fcdof || setBC) {
4472     if (clperm) {
4473       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4474       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4475     } else {
4476       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4477       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4478     }
4479   } else {
4480     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4481     if (clperm) {
4482       if (perm) {
4483         for (b = 0; b < fdof; b++) {
4484           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4485           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4486         }
4487       } else {
4488         for (b = 0; b < fdof; b++) {
4489           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4490           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4491         }
4492       }
4493     } else {
4494       if (perm) {
4495         for (b = 0; b < fdof; b++) {
4496           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4497           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4498         }
4499       } else {
4500         for (b = 0; b < fdof; b++) {
4501           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4502           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4503         }
4504       }
4505     }
4506   }
4507   *offset += fdof;
4508   PetscFunctionReturn(0);
4509 }
4510 
4511 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4512 {
4513   PetscScalar    *a;
4514   PetscInt        fdof, foff, fcdof, foffset = *offset;
4515   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4516   PetscInt        cind = 0, ncind = 0, b;
4517   PetscBool       ncSet, fcSet;
4518   PetscErrorCode  ierr;
4519 
4520   PetscFunctionBegin;
4521   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4522   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4523   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4524   a    = &array[foff];
4525   if (fcdof) {
4526     /* We just override fcdof and fcdofs with Ncc and comps */
4527     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4528     if (clperm) {
4529       if (perm) {
4530         if (comps) {
4531           for (b = 0; b < fdof; b++) {
4532             ncSet = fcSet = PETSC_FALSE;
4533             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4534             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4535             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4536           }
4537         } else {
4538           for (b = 0; b < fdof; b++) {
4539             if ((cind < fcdof) && (b == fcdofs[cind])) {
4540               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4541               ++cind;
4542             }
4543           }
4544         }
4545       } else {
4546         if (comps) {
4547           for (b = 0; b < fdof; b++) {
4548             ncSet = fcSet = PETSC_FALSE;
4549             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4550             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4551             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4552           }
4553         } else {
4554           for (b = 0; b < fdof; b++) {
4555             if ((cind < fcdof) && (b == fcdofs[cind])) {
4556               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4557               ++cind;
4558             }
4559           }
4560         }
4561       }
4562     } else {
4563       if (perm) {
4564         if (comps) {
4565           for (b = 0; b < fdof; b++) {
4566             ncSet = fcSet = PETSC_FALSE;
4567             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4568             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4569             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4570           }
4571         } else {
4572           for (b = 0; b < fdof; b++) {
4573             if ((cind < fcdof) && (b == fcdofs[cind])) {
4574               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4575               ++cind;
4576             }
4577           }
4578         }
4579       } else {
4580         if (comps) {
4581           for (b = 0; b < fdof; b++) {
4582             ncSet = fcSet = PETSC_FALSE;
4583             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4584             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4585             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4586           }
4587         } else {
4588           for (b = 0; b < fdof; b++) {
4589             if ((cind < fcdof) && (b == fcdofs[cind])) {
4590               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4591               ++cind;
4592             }
4593           }
4594         }
4595       }
4596     }
4597   }
4598   *offset += fdof;
4599   PetscFunctionReturn(0);
4600 }
4601 
4602 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4603 {
4604   PetscScalar    *array;
4605   const PetscInt *cone, *coneO;
4606   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4607   PetscErrorCode  ierr;
4608 
4609   PetscFunctionBeginHot;
4610   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4611   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4612   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4613   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4614   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4615   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4616     const PetscInt cp = !p ? point : cone[p-1];
4617     const PetscInt o  = !p ? 0     : coneO[p-1];
4618 
4619     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4620     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4621     /* ADD_VALUES */
4622     {
4623       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4624       PetscScalar    *a;
4625       PetscInt        cdof, coff, cind = 0, k;
4626 
4627       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4628       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4629       a    = &array[coff];
4630       if (!cdof) {
4631         if (o >= 0) {
4632           for (k = 0; k < dof; ++k) {
4633             a[k] += values[off+k];
4634           }
4635         } else {
4636           for (k = 0; k < dof; ++k) {
4637             a[k] += values[off+dof-k-1];
4638           }
4639         }
4640       } else {
4641         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4642         if (o >= 0) {
4643           for (k = 0; k < dof; ++k) {
4644             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4645             a[k] += values[off+k];
4646           }
4647         } else {
4648           for (k = 0; k < dof; ++k) {
4649             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4650             a[k] += values[off+dof-k-1];
4651           }
4652         }
4653       }
4654     }
4655   }
4656   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4657   PetscFunctionReturn(0);
4658 }
4659 
4660 /*@C
4661   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4662 
4663   Not collective
4664 
4665   Input Parameters:
4666 + dm - The DM
4667 . section - The section describing the layout in v, or NULL to use the default section
4668 . v - The local vector
4669 . point - The point in the DM
4670 . values - The array of values
4671 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4672          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4673 
4674   Fortran Notes:
4675   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4676 
4677   Level: intermediate
4678 
4679 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4680 @*/
4681 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4682 {
4683   PetscSection    clSection;
4684   IS              clPoints;
4685   PetscScalar    *array;
4686   PetscInt       *points = NULL;
4687   const PetscInt *clp, *clperm;
4688   PetscInt        depth, numFields, numPoints, p;
4689   PetscErrorCode  ierr;
4690 
4691   PetscFunctionBeginHot;
4692   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4693   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4694   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4695   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4696   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4697   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4698   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4699     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4700     PetscFunctionReturn(0);
4701   }
4702   /* Get points */
4703   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4704   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4705   /* Get array */
4706   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4707   /* Get values */
4708   if (numFields > 0) {
4709     PetscInt offset = 0, f;
4710     for (f = 0; f < numFields; ++f) {
4711       const PetscInt    **perms = NULL;
4712       const PetscScalar **flips = NULL;
4713 
4714       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4715       switch (mode) {
4716       case INSERT_VALUES:
4717         for (p = 0; p < numPoints; p++) {
4718           const PetscInt    point = points[2*p];
4719           const PetscInt    *perm = perms ? perms[p] : NULL;
4720           const PetscScalar *flip = flips ? flips[p] : NULL;
4721           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4722         } break;
4723       case INSERT_ALL_VALUES:
4724         for (p = 0; p < numPoints; p++) {
4725           const PetscInt    point = points[2*p];
4726           const PetscInt    *perm = perms ? perms[p] : NULL;
4727           const PetscScalar *flip = flips ? flips[p] : NULL;
4728           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4729         } break;
4730       case INSERT_BC_VALUES:
4731         for (p = 0; p < numPoints; p++) {
4732           const PetscInt    point = points[2*p];
4733           const PetscInt    *perm = perms ? perms[p] : NULL;
4734           const PetscScalar *flip = flips ? flips[p] : NULL;
4735           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4736         } break;
4737       case ADD_VALUES:
4738         for (p = 0; p < numPoints; p++) {
4739           const PetscInt    point = points[2*p];
4740           const PetscInt    *perm = perms ? perms[p] : NULL;
4741           const PetscScalar *flip = flips ? flips[p] : NULL;
4742           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4743         } break;
4744       case ADD_ALL_VALUES:
4745         for (p = 0; p < numPoints; p++) {
4746           const PetscInt    point = points[2*p];
4747           const PetscInt    *perm = perms ? perms[p] : NULL;
4748           const PetscScalar *flip = flips ? flips[p] : NULL;
4749           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4750         } break;
4751       case ADD_BC_VALUES:
4752         for (p = 0; p < numPoints; p++) {
4753           const PetscInt    point = points[2*p];
4754           const PetscInt    *perm = perms ? perms[p] : NULL;
4755           const PetscScalar *flip = flips ? flips[p] : NULL;
4756           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4757         } break;
4758       default:
4759         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4760       }
4761       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4762     }
4763   } else {
4764     PetscInt dof, off;
4765     const PetscInt    **perms = NULL;
4766     const PetscScalar **flips = NULL;
4767 
4768     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4769     switch (mode) {
4770     case INSERT_VALUES:
4771       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4772         const PetscInt    point = points[2*p];
4773         const PetscInt    *perm = perms ? perms[p] : NULL;
4774         const PetscScalar *flip = flips ? flips[p] : NULL;
4775         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4776         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4777       } break;
4778     case INSERT_ALL_VALUES:
4779       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4780         const PetscInt    point = points[2*p];
4781         const PetscInt    *perm = perms ? perms[p] : NULL;
4782         const PetscScalar *flip = flips ? flips[p] : NULL;
4783         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4784         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4785       } break;
4786     case INSERT_BC_VALUES:
4787       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4788         const PetscInt    point = points[2*p];
4789         const PetscInt    *perm = perms ? perms[p] : NULL;
4790         const PetscScalar *flip = flips ? flips[p] : NULL;
4791         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4792         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4793       } break;
4794     case ADD_VALUES:
4795       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4796         const PetscInt    point = points[2*p];
4797         const PetscInt    *perm = perms ? perms[p] : NULL;
4798         const PetscScalar *flip = flips ? flips[p] : NULL;
4799         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4800         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4801       } break;
4802     case ADD_ALL_VALUES:
4803       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4804         const PetscInt    point = points[2*p];
4805         const PetscInt    *perm = perms ? perms[p] : NULL;
4806         const PetscScalar *flip = flips ? flips[p] : NULL;
4807         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4808         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4809       } break;
4810     case ADD_BC_VALUES:
4811       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4812         const PetscInt    point = points[2*p];
4813         const PetscInt    *perm = perms ? perms[p] : NULL;
4814         const PetscScalar *flip = flips ? flips[p] : NULL;
4815         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4816         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4817       } break;
4818     default:
4819       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4820     }
4821     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4822   }
4823   /* Cleanup points */
4824   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4825   /* Cleanup array */
4826   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4827   PetscFunctionReturn(0);
4828 }
4829 
4830 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4831 {
4832   PetscSection      clSection;
4833   IS                clPoints;
4834   PetscScalar       *array;
4835   PetscInt          *points = NULL;
4836   const PetscInt    *clp, *clperm;
4837   PetscInt          numFields, numPoints, p;
4838   PetscInt          offset = 0, f;
4839   PetscErrorCode    ierr;
4840 
4841   PetscFunctionBeginHot;
4842   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4843   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4844   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4845   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4846   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4847   /* Get points */
4848   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4849   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4850   /* Get array */
4851   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4852   /* Get values */
4853   for (f = 0; f < numFields; ++f) {
4854     const PetscInt    **perms = NULL;
4855     const PetscScalar **flips = NULL;
4856 
4857     if (!fieldActive[f]) {
4858       for (p = 0; p < numPoints*2; p += 2) {
4859         PetscInt fdof;
4860         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
4861         offset += fdof;
4862       }
4863       continue;
4864     }
4865     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4866     switch (mode) {
4867     case INSERT_VALUES:
4868       for (p = 0; p < numPoints; p++) {
4869         const PetscInt    point = points[2*p];
4870         const PetscInt    *perm = perms ? perms[p] : NULL;
4871         const PetscScalar *flip = flips ? flips[p] : NULL;
4872         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4873       } break;
4874     case INSERT_ALL_VALUES:
4875       for (p = 0; p < numPoints; p++) {
4876         const PetscInt    point = points[2*p];
4877         const PetscInt    *perm = perms ? perms[p] : NULL;
4878         const PetscScalar *flip = flips ? flips[p] : NULL;
4879         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4880         } break;
4881     case INSERT_BC_VALUES:
4882       for (p = 0; p < numPoints; p++) {
4883         const PetscInt    point = points[2*p];
4884         const PetscInt    *perm = perms ? perms[p] : NULL;
4885         const PetscScalar *flip = flips ? flips[p] : NULL;
4886         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
4887       } break;
4888     case ADD_VALUES:
4889       for (p = 0; p < numPoints; p++) {
4890         const PetscInt    point = points[2*p];
4891         const PetscInt    *perm = perms ? perms[p] : NULL;
4892         const PetscScalar *flip = flips ? flips[p] : NULL;
4893         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4894       } break;
4895     case ADD_ALL_VALUES:
4896       for (p = 0; p < numPoints; p++) {
4897         const PetscInt    point = points[2*p];
4898         const PetscInt    *perm = perms ? perms[p] : NULL;
4899         const PetscScalar *flip = flips ? flips[p] : NULL;
4900         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4901       } break;
4902     default:
4903       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4904     }
4905     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4906   }
4907   /* Cleanup points */
4908   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4909   /* Cleanup array */
4910   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4911   PetscFunctionReturn(0);
4912 }
4913 
4914 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
4915 {
4916   PetscMPIInt    rank;
4917   PetscInt       i, j;
4918   PetscErrorCode ierr;
4919 
4920   PetscFunctionBegin;
4921   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
4922   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
4923   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
4924   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
4925   numCIndices = numCIndices ? numCIndices : numRIndices;
4926   for (i = 0; i < numRIndices; i++) {
4927     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
4928     for (j = 0; j < numCIndices; j++) {
4929 #if defined(PETSC_USE_COMPLEX)
4930       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
4931 #else
4932       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
4933 #endif
4934     }
4935     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
4936   }
4937   PetscFunctionReturn(0);
4938 }
4939 
4940 /*
4941   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
4942 
4943   Input Parameters:
4944 + section - The section for this data layout
4945 . point   - The point contributing dofs with these indices
4946 . off     - The global offset of this point
4947 . loff    - The local offset of each field
4948 . setBC   - The flag determining whether to include indices of bounsary values
4949 . perm    - A permutation of the dofs on this point, or NULL
4950 - indperm - A permutation of the entire indices array, or NULL
4951 
4952   Output Parameter:
4953 . indices - Indices for dofs on this point
4954 
4955   Level: developer
4956 
4957   Note: The indices could be local or global, depending on the value of 'off'.
4958 */
4959 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
4960 {
4961   PetscInt        dof;   /* The number of unknowns on this point */
4962   PetscInt        cdof;  /* The number of constraints on this point */
4963   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4964   PetscInt        cind = 0, k;
4965   PetscErrorCode  ierr;
4966 
4967   PetscFunctionBegin;
4968   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4969   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4970   if (!cdof || setBC) {
4971     for (k = 0; k < dof; ++k) {
4972       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
4973       const PetscInt ind    = indperm ? indperm[preind] : preind;
4974 
4975       indices[ind] = off + k;
4976     }
4977   } else {
4978     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4979     for (k = 0; k < dof; ++k) {
4980       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
4981       const PetscInt ind    = indperm ? indperm[preind] : preind;
4982 
4983       if ((cind < cdof) && (k == cdofs[cind])) {
4984         /* Insert check for returning constrained indices */
4985         indices[ind] = -(off+k+1);
4986         ++cind;
4987       } else {
4988         indices[ind] = off+k-cind;
4989       }
4990     }
4991   }
4992   *loff += dof;
4993   PetscFunctionReturn(0);
4994 }
4995 
4996 /*
4997   This version only believes the point offset from the globalSection
4998 
4999  . off - The global offset of this point
5000 */
5001 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5002 {
5003   PetscInt       numFields, foff, f;
5004   PetscErrorCode ierr;
5005 
5006   PetscFunctionBegin;
5007   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5008   for (f = 0, foff = 0; f < numFields; ++f) {
5009     PetscInt        fdof, cfdof;
5010     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5011     PetscInt        cind = 0, b;
5012     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5013 
5014     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5015     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5016     if (!cfdof || setBC) {
5017       for (b = 0; b < fdof; ++b) {
5018         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5019         const PetscInt ind    = indperm ? indperm[preind] : preind;
5020 
5021         indices[ind] = off+foff+b;
5022       }
5023     } else {
5024       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5025       for (b = 0; b < fdof; ++b) {
5026         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5027         const PetscInt ind    = indperm ? indperm[preind] : preind;
5028 
5029         if ((cind < cfdof) && (b == fcdofs[cind])) {
5030           indices[ind] = -(off+foff+b+1);
5031           ++cind;
5032         } else {
5033           indices[ind] = off+foff+b-cind;
5034         }
5035       }
5036     }
5037     foff     += (setBC ? fdof : (fdof - cfdof));
5038     foffs[f] += fdof;
5039   }
5040   PetscFunctionReturn(0);
5041 }
5042 
5043 /*
5044   This version believes the globalSection offsets for each field, rather than just the point offset
5045 
5046  . foffs - The offset into 'indices' for each field, since it is segregated by field
5047 */
5048 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5049 {
5050   PetscInt       numFields, foff, f;
5051   PetscErrorCode ierr;
5052 
5053   PetscFunctionBegin;
5054   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5055   for (f = 0; f < numFields; ++f) {
5056     PetscInt        fdof, cfdof;
5057     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5058     PetscInt        cind = 0, b;
5059     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5060 
5061     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5062     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5063     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5064     if (!cfdof || setBC) {
5065       for (b = 0; b < fdof; ++b) {
5066         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5067         const PetscInt ind    = indperm ? indperm[preind] : preind;
5068 
5069         indices[ind] = foff+b;
5070       }
5071     } else {
5072       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5073       for (b = 0; b < fdof; ++b) {
5074         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5075         const PetscInt ind    = indperm ? indperm[preind] : preind;
5076 
5077         if ((cind < cfdof) && (b == fcdofs[cind])) {
5078           indices[ind] = -(foff+b+1);
5079           ++cind;
5080         } else {
5081           indices[ind] = foff+b-cind;
5082         }
5083       }
5084     }
5085     foffs[f] += fdof;
5086   }
5087   PetscFunctionReturn(0);
5088 }
5089 
5090 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5091 {
5092   Mat             cMat;
5093   PetscSection    aSec, cSec;
5094   IS              aIS;
5095   PetscInt        aStart = -1, aEnd = -1;
5096   const PetscInt  *anchors;
5097   PetscInt        numFields, f, p, q, newP = 0;
5098   PetscInt        newNumPoints = 0, newNumIndices = 0;
5099   PetscInt        *newPoints, *indices, *newIndices;
5100   PetscInt        maxAnchor, maxDof;
5101   PetscInt        newOffsets[32];
5102   PetscInt        *pointMatOffsets[32];
5103   PetscInt        *newPointOffsets[32];
5104   PetscScalar     *pointMat[32];
5105   PetscScalar     *newValues=NULL,*tmpValues;
5106   PetscBool       anyConstrained = PETSC_FALSE;
5107   PetscErrorCode  ierr;
5108 
5109   PetscFunctionBegin;
5110   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5111   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5112   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5113 
5114   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5115   /* if there are point-to-point constraints */
5116   if (aSec) {
5117     ierr = PetscMemzero(newOffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5118     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5119     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5120     /* figure out how many points are going to be in the new element matrix
5121      * (we allow double counting, because it's all just going to be summed
5122      * into the global matrix anyway) */
5123     for (p = 0; p < 2*numPoints; p+=2) {
5124       PetscInt b    = points[p];
5125       PetscInt bDof = 0, bSecDof;
5126 
5127       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5128       if (!bSecDof) {
5129         continue;
5130       }
5131       if (b >= aStart && b < aEnd) {
5132         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5133       }
5134       if (bDof) {
5135         /* this point is constrained */
5136         /* it is going to be replaced by its anchors */
5137         PetscInt bOff, q;
5138 
5139         anyConstrained = PETSC_TRUE;
5140         newNumPoints  += bDof;
5141         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5142         for (q = 0; q < bDof; q++) {
5143           PetscInt a = anchors[bOff + q];
5144           PetscInt aDof;
5145 
5146           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5147           newNumIndices += aDof;
5148           for (f = 0; f < numFields; ++f) {
5149             PetscInt fDof;
5150 
5151             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5152             newOffsets[f+1] += fDof;
5153           }
5154         }
5155       }
5156       else {
5157         /* this point is not constrained */
5158         newNumPoints++;
5159         newNumIndices += bSecDof;
5160         for (f = 0; f < numFields; ++f) {
5161           PetscInt fDof;
5162 
5163           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5164           newOffsets[f+1] += fDof;
5165         }
5166       }
5167     }
5168   }
5169   if (!anyConstrained) {
5170     if (outNumPoints)  *outNumPoints  = 0;
5171     if (outNumIndices) *outNumIndices = 0;
5172     if (outPoints)     *outPoints     = NULL;
5173     if (outValues)     *outValues     = NULL;
5174     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5175     PetscFunctionReturn(0);
5176   }
5177 
5178   if (outNumPoints)  *outNumPoints  = newNumPoints;
5179   if (outNumIndices) *outNumIndices = newNumIndices;
5180 
5181   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5182 
5183   if (!outPoints && !outValues) {
5184     if (offsets) {
5185       for (f = 0; f <= numFields; f++) {
5186         offsets[f] = newOffsets[f];
5187       }
5188     }
5189     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5190     PetscFunctionReturn(0);
5191   }
5192 
5193   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5194 
5195   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5196 
5197   /* workspaces */
5198   if (numFields) {
5199     for (f = 0; f < numFields; f++) {
5200       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5201       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5202     }
5203   }
5204   else {
5205     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5206     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5207   }
5208 
5209   /* get workspaces for the point-to-point matrices */
5210   if (numFields) {
5211     PetscInt totalOffset, totalMatOffset;
5212 
5213     for (p = 0; p < numPoints; p++) {
5214       PetscInt b    = points[2*p];
5215       PetscInt bDof = 0, bSecDof;
5216 
5217       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5218       if (!bSecDof) {
5219         for (f = 0; f < numFields; f++) {
5220           newPointOffsets[f][p + 1] = 0;
5221           pointMatOffsets[f][p + 1] = 0;
5222         }
5223         continue;
5224       }
5225       if (b >= aStart && b < aEnd) {
5226         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5227       }
5228       if (bDof) {
5229         for (f = 0; f < numFields; f++) {
5230           PetscInt fDof, q, bOff, allFDof = 0;
5231 
5232           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5233           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5234           for (q = 0; q < bDof; q++) {
5235             PetscInt a = anchors[bOff + q];
5236             PetscInt aFDof;
5237 
5238             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5239             allFDof += aFDof;
5240           }
5241           newPointOffsets[f][p+1] = allFDof;
5242           pointMatOffsets[f][p+1] = fDof * allFDof;
5243         }
5244       }
5245       else {
5246         for (f = 0; f < numFields; f++) {
5247           PetscInt fDof;
5248 
5249           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5250           newPointOffsets[f][p+1] = fDof;
5251           pointMatOffsets[f][p+1] = 0;
5252         }
5253       }
5254     }
5255     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5256       newPointOffsets[f][0] = totalOffset;
5257       pointMatOffsets[f][0] = totalMatOffset;
5258       for (p = 0; p < numPoints; p++) {
5259         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5260         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5261       }
5262       totalOffset    = newPointOffsets[f][numPoints];
5263       totalMatOffset = pointMatOffsets[f][numPoints];
5264       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5265     }
5266   }
5267   else {
5268     for (p = 0; p < numPoints; p++) {
5269       PetscInt b    = points[2*p];
5270       PetscInt bDof = 0, bSecDof;
5271 
5272       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5273       if (!bSecDof) {
5274         newPointOffsets[0][p + 1] = 0;
5275         pointMatOffsets[0][p + 1] = 0;
5276         continue;
5277       }
5278       if (b >= aStart && b < aEnd) {
5279         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5280       }
5281       if (bDof) {
5282         PetscInt bOff, q, allDof = 0;
5283 
5284         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5285         for (q = 0; q < bDof; q++) {
5286           PetscInt a = anchors[bOff + q], aDof;
5287 
5288           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5289           allDof += aDof;
5290         }
5291         newPointOffsets[0][p+1] = allDof;
5292         pointMatOffsets[0][p+1] = bSecDof * allDof;
5293       }
5294       else {
5295         newPointOffsets[0][p+1] = bSecDof;
5296         pointMatOffsets[0][p+1] = 0;
5297       }
5298     }
5299     newPointOffsets[0][0] = 0;
5300     pointMatOffsets[0][0] = 0;
5301     for (p = 0; p < numPoints; p++) {
5302       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5303       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5304     }
5305     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5306   }
5307 
5308   /* output arrays */
5309   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5310 
5311   /* get the point-to-point matrices; construct newPoints */
5312   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5313   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5314   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5315   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5316   if (numFields) {
5317     for (p = 0, newP = 0; p < numPoints; p++) {
5318       PetscInt b    = points[2*p];
5319       PetscInt o    = points[2*p+1];
5320       PetscInt bDof = 0, bSecDof;
5321 
5322       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5323       if (!bSecDof) {
5324         continue;
5325       }
5326       if (b >= aStart && b < aEnd) {
5327         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5328       }
5329       if (bDof) {
5330         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5331 
5332         fStart[0] = 0;
5333         fEnd[0]   = 0;
5334         for (f = 0; f < numFields; f++) {
5335           PetscInt fDof;
5336 
5337           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5338           fStart[f+1] = fStart[f] + fDof;
5339           fEnd[f+1]   = fStart[f+1];
5340         }
5341         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5342         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5343 
5344         fAnchorStart[0] = 0;
5345         fAnchorEnd[0]   = 0;
5346         for (f = 0; f < numFields; f++) {
5347           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5348 
5349           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5350           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5351         }
5352         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5353         for (q = 0; q < bDof; q++) {
5354           PetscInt a = anchors[bOff + q], aOff;
5355 
5356           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5357           newPoints[2*(newP + q)]     = a;
5358           newPoints[2*(newP + q) + 1] = 0;
5359           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5360           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5361         }
5362         newP += bDof;
5363 
5364         if (outValues) {
5365           /* get the point-to-point submatrix */
5366           for (f = 0; f < numFields; f++) {
5367             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5368           }
5369         }
5370       }
5371       else {
5372         newPoints[2 * newP]     = b;
5373         newPoints[2 * newP + 1] = o;
5374         newP++;
5375       }
5376     }
5377   } else {
5378     for (p = 0; p < numPoints; p++) {
5379       PetscInt b    = points[2*p];
5380       PetscInt o    = points[2*p+1];
5381       PetscInt bDof = 0, bSecDof;
5382 
5383       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5384       if (!bSecDof) {
5385         continue;
5386       }
5387       if (b >= aStart && b < aEnd) {
5388         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5389       }
5390       if (bDof) {
5391         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5392 
5393         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5394         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5395 
5396         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5397         for (q = 0; q < bDof; q++) {
5398           PetscInt a = anchors[bOff + q], aOff;
5399 
5400           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5401 
5402           newPoints[2*(newP + q)]     = a;
5403           newPoints[2*(newP + q) + 1] = 0;
5404           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5405           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5406         }
5407         newP += bDof;
5408 
5409         /* get the point-to-point submatrix */
5410         if (outValues) {
5411           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5412         }
5413       }
5414       else {
5415         newPoints[2 * newP]     = b;
5416         newPoints[2 * newP + 1] = o;
5417         newP++;
5418       }
5419     }
5420   }
5421 
5422   if (outValues) {
5423     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5424     ierr = PetscMemzero(tmpValues,newNumIndices*numIndices*sizeof(*tmpValues));CHKERRQ(ierr);
5425     /* multiply constraints on the right */
5426     if (numFields) {
5427       for (f = 0; f < numFields; f++) {
5428         PetscInt oldOff = offsets[f];
5429 
5430         for (p = 0; p < numPoints; p++) {
5431           PetscInt cStart = newPointOffsets[f][p];
5432           PetscInt b      = points[2 * p];
5433           PetscInt c, r, k;
5434           PetscInt dof;
5435 
5436           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5437           if (!dof) {
5438             continue;
5439           }
5440           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5441             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5442             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5443 
5444             for (r = 0; r < numIndices; r++) {
5445               for (c = 0; c < nCols; c++) {
5446                 for (k = 0; k < dof; k++) {
5447                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5448                 }
5449               }
5450             }
5451           }
5452           else {
5453             /* copy this column as is */
5454             for (r = 0; r < numIndices; r++) {
5455               for (c = 0; c < dof; c++) {
5456                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5457               }
5458             }
5459           }
5460           oldOff += dof;
5461         }
5462       }
5463     }
5464     else {
5465       PetscInt oldOff = 0;
5466       for (p = 0; p < numPoints; p++) {
5467         PetscInt cStart = newPointOffsets[0][p];
5468         PetscInt b      = points[2 * p];
5469         PetscInt c, r, k;
5470         PetscInt dof;
5471 
5472         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5473         if (!dof) {
5474           continue;
5475         }
5476         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5477           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5478           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5479 
5480           for (r = 0; r < numIndices; r++) {
5481             for (c = 0; c < nCols; c++) {
5482               for (k = 0; k < dof; k++) {
5483                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5484               }
5485             }
5486           }
5487         }
5488         else {
5489           /* copy this column as is */
5490           for (r = 0; r < numIndices; r++) {
5491             for (c = 0; c < dof; c++) {
5492               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5493             }
5494           }
5495         }
5496         oldOff += dof;
5497       }
5498     }
5499 
5500     if (multiplyLeft) {
5501       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5502       ierr = PetscMemzero(newValues,newNumIndices*newNumIndices*sizeof(*newValues));CHKERRQ(ierr);
5503       /* multiply constraints transpose on the left */
5504       if (numFields) {
5505         for (f = 0; f < numFields; f++) {
5506           PetscInt oldOff = offsets[f];
5507 
5508           for (p = 0; p < numPoints; p++) {
5509             PetscInt rStart = newPointOffsets[f][p];
5510             PetscInt b      = points[2 * p];
5511             PetscInt c, r, k;
5512             PetscInt dof;
5513 
5514             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5515             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5516               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5517               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5518 
5519               for (r = 0; r < nRows; r++) {
5520                 for (c = 0; c < newNumIndices; c++) {
5521                   for (k = 0; k < dof; k++) {
5522                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5523                   }
5524                 }
5525               }
5526             }
5527             else {
5528               /* copy this row as is */
5529               for (r = 0; r < dof; r++) {
5530                 for (c = 0; c < newNumIndices; c++) {
5531                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5532                 }
5533               }
5534             }
5535             oldOff += dof;
5536           }
5537         }
5538       }
5539       else {
5540         PetscInt oldOff = 0;
5541 
5542         for (p = 0; p < numPoints; p++) {
5543           PetscInt rStart = newPointOffsets[0][p];
5544           PetscInt b      = points[2 * p];
5545           PetscInt c, r, k;
5546           PetscInt dof;
5547 
5548           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5549           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5550             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5551             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5552 
5553             for (r = 0; r < nRows; r++) {
5554               for (c = 0; c < newNumIndices; c++) {
5555                 for (k = 0; k < dof; k++) {
5556                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5557                 }
5558               }
5559             }
5560           }
5561           else {
5562             /* copy this row as is */
5563             for (r = 0; r < dof; r++) {
5564               for (c = 0; c < newNumIndices; c++) {
5565                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5566               }
5567             }
5568           }
5569           oldOff += dof;
5570         }
5571       }
5572 
5573       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5574     }
5575     else {
5576       newValues = tmpValues;
5577     }
5578   }
5579 
5580   /* clean up */
5581   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5582   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5583 
5584   if (numFields) {
5585     for (f = 0; f < numFields; f++) {
5586       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5587       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5588       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5589     }
5590   }
5591   else {
5592     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5593     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5594     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5595   }
5596   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5597 
5598   /* output */
5599   if (outPoints) {
5600     *outPoints = newPoints;
5601   }
5602   else {
5603     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5604   }
5605   if (outValues) {
5606     *outValues = newValues;
5607   }
5608   for (f = 0; f <= numFields; f++) {
5609     offsets[f] = newOffsets[f];
5610   }
5611   PetscFunctionReturn(0);
5612 }
5613 
5614 /*@C
5615   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5616 
5617   Not collective
5618 
5619   Input Parameters:
5620 + dm - The DM
5621 . section - The section describing the layout in v, or NULL to use the default section
5622 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5623 - point - The mesh point
5624 
5625   Output parameters:
5626 + numIndices - The number of indices
5627 . indices - The indices
5628 - outOffsets - Field offset if not NULL
5629 
5630   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5631 
5632   Level: advanced
5633 
5634 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5635 @*/
5636 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5637 {
5638   PetscSection    clSection;
5639   IS              clPoints;
5640   const PetscInt *clp, *clperm;
5641   const PetscInt  **perms[32] = {NULL};
5642   PetscInt       *points = NULL, *pointsNew;
5643   PetscInt        numPoints, numPointsNew;
5644   PetscInt        offsets[32];
5645   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5646   PetscErrorCode  ierr;
5647 
5648   PetscFunctionBegin;
5649   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5650   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5651   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5652   if (numIndices) PetscValidPointer(numIndices, 4);
5653   PetscValidPointer(indices, 5);
5654   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5655   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5656   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5657   /* Get points in closure */
5658   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5659   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5660   /* Get number of indices and indices per field */
5661   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5662     PetscInt dof, fdof;
5663 
5664     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5665     for (f = 0; f < Nf; ++f) {
5666       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5667       offsets[f+1] += fdof;
5668     }
5669     Nind += dof;
5670   }
5671   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5672   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5673   if (!Nf) offsets[1] = Nind;
5674   /* Get dual space symmetries */
5675   for (f = 0; f < PetscMax(1,Nf); f++) {
5676     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5677     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5678   }
5679   /* Correct for hanging node constraints */
5680   {
5681     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5682     if (numPointsNew) {
5683       for (f = 0; f < PetscMax(1,Nf); f++) {
5684         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5685         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5686       }
5687       for (f = 0; f < PetscMax(1,Nf); f++) {
5688         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5689         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5690       }
5691       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5692       numPoints = numPointsNew;
5693       Nind      = NindNew;
5694       points    = pointsNew;
5695     }
5696   }
5697   /* Calculate indices */
5698   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5699   if (Nf) {
5700     if (outOffsets) {
5701       PetscInt f;
5702 
5703       for (f = 0; f <= Nf; f++) {
5704         outOffsets[f] = offsets[f];
5705       }
5706     }
5707     for (p = 0; p < numPoints; p++) {
5708       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5709       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);
5710     }
5711   } else {
5712     for (p = 0, off = 0; p < numPoints; p++) {
5713       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5714 
5715       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5716       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);
5717     }
5718   }
5719   /* Cleanup points */
5720   for (f = 0; f < PetscMax(1,Nf); f++) {
5721     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5722     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5723   }
5724   if (numPointsNew) {
5725     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5726   } else {
5727     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5728   }
5729   if (numIndices) *numIndices = Nind;
5730   PetscFunctionReturn(0);
5731 }
5732 
5733 /*@C
5734   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5735 
5736   Not collective
5737 
5738   Input Parameters:
5739 + dm - The DM
5740 . section - The section describing the layout in v, or NULL to use the default section
5741 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5742 . point - The mesh point
5743 . numIndices - The number of indices
5744 . indices - The indices
5745 - outOffsets - Field offset if not NULL
5746 
5747   Level: advanced
5748 
5749 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5750 @*/
5751 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5752 {
5753   PetscErrorCode ierr;
5754 
5755   PetscFunctionBegin;
5756   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5757   PetscValidPointer(indices, 5);
5758   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5759   PetscFunctionReturn(0);
5760 }
5761 
5762 /*@C
5763   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5764 
5765   Not collective
5766 
5767   Input Parameters:
5768 + dm - The DM
5769 . section - The section describing the layout in v, or NULL to use the default section
5770 . globalSection - The section describing the layout in v, or NULL to use the default global section
5771 . A - The matrix
5772 . point - The point in the DM
5773 . values - The array of values
5774 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5775 
5776   Fortran Notes:
5777   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5778 
5779   Level: intermediate
5780 
5781 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5782 @*/
5783 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5784 {
5785   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5786   PetscSection        clSection;
5787   IS                  clPoints;
5788   PetscInt           *points = NULL, *newPoints;
5789   const PetscInt     *clp, *clperm;
5790   PetscInt           *indices;
5791   PetscInt            offsets[32];
5792   const PetscInt    **perms[32] = {NULL};
5793   const PetscScalar **flips[32] = {NULL};
5794   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5795   PetscScalar        *valCopy = NULL;
5796   PetscScalar        *newValues;
5797   PetscErrorCode      ierr;
5798 
5799   PetscFunctionBegin;
5800   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5801   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
5802   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5803   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5804   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5805   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5806   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5807   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5808   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5809   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5810   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5811   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5812     PetscInt fdof;
5813 
5814     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5815     for (f = 0; f < numFields; ++f) {
5816       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5817       offsets[f+1] += fdof;
5818     }
5819     numIndices += dof;
5820   }
5821   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5822 
5823   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5824   /* Get symmetries */
5825   for (f = 0; f < PetscMax(1,numFields); f++) {
5826     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5827     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5828     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5829       PetscInt foffset = offsets[f];
5830 
5831       for (p = 0; p < numPoints; p++) {
5832         PetscInt point          = points[2*p], fdof;
5833         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5834 
5835         if (!numFields) {
5836           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5837         } else {
5838           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5839         }
5840         if (flip) {
5841           PetscInt i, j, k;
5842 
5843           if (!valCopy) {
5844             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5845             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
5846             values = valCopy;
5847           }
5848           for (i = 0; i < fdof; i++) {
5849             PetscScalar fval = flip[i];
5850 
5851             for (k = 0; k < numIndices; k++) {
5852               valCopy[numIndices * (foffset + i) + k] *= fval;
5853               valCopy[numIndices * k + (foffset + i)] *= fval;
5854             }
5855           }
5856         }
5857         foffset += fdof;
5858       }
5859     }
5860   }
5861   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
5862   if (newNumPoints) {
5863     if (valCopy) {
5864       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5865     }
5866     for (f = 0; f < PetscMax(1,numFields); f++) {
5867       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5868       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5869     }
5870     for (f = 0; f < PetscMax(1,numFields); f++) {
5871       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5872       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5873     }
5874     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5875     numPoints  = newNumPoints;
5876     numIndices = newNumIndices;
5877     points     = newPoints;
5878     values     = newValues;
5879   }
5880   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5881   if (numFields) {
5882     PetscBool useFieldOffsets;
5883 
5884     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
5885     if (useFieldOffsets) {
5886       for (p = 0; p < numPoints; p++) {
5887         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
5888       }
5889     } else {
5890       for (p = 0; p < numPoints; p++) {
5891         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5892         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
5893       }
5894     }
5895   } else {
5896     for (p = 0, off = 0; p < numPoints; p++) {
5897       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5898       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5899       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
5900     }
5901   }
5902   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5903   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5904   if (mesh->printFEM > 1) {
5905     PetscInt i;
5906     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
5907     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
5908     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
5909   }
5910   if (ierr) {
5911     PetscMPIInt    rank;
5912     PetscErrorCode ierr2;
5913 
5914     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5915     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5916     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5917     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
5918     CHKERRQ(ierr);
5919   }
5920   for (f = 0; f < PetscMax(1,numFields); f++) {
5921     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5922     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5923   }
5924   if (newNumPoints) {
5925     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5926     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5927   }
5928   else {
5929     if (valCopy) {
5930       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5931     }
5932     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5933   }
5934   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5935   PetscFunctionReturn(0);
5936 }
5937 
5938 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5939 {
5940   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5941   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5942   PetscInt       *cpoints = NULL;
5943   PetscInt       *findices, *cindices;
5944   const PetscInt *fclperm, *cclperm;
5945   PetscInt        foffsets[32], coffsets[32];
5946   CellRefiner     cellRefiner;
5947   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5948   PetscErrorCode  ierr;
5949 
5950   PetscFunctionBegin;
5951   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5952   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5953   if (!fsection) {ierr = DMGetSection(dmf, &fsection);CHKERRQ(ierr);}
5954   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5955   if (!csection) {ierr = DMGetSection(dmc, &csection);CHKERRQ(ierr);}
5956   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5957   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5958   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5959   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5960   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5961   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5962   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5963   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5964   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5965   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
5966   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
5967   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
5968   /* Column indices */
5969   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5970   maxFPoints = numCPoints;
5971   /* Compress out points not in the section */
5972   /*   TODO: Squeeze out points with 0 dof as well */
5973   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5974   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5975     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5976       cpoints[q*2]   = cpoints[p];
5977       cpoints[q*2+1] = cpoints[p+1];
5978       ++q;
5979     }
5980   }
5981   numCPoints = q;
5982   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5983     PetscInt fdof;
5984 
5985     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5986     if (!dof) continue;
5987     for (f = 0; f < numFields; ++f) {
5988       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
5989       coffsets[f+1] += fdof;
5990     }
5991     numCIndices += dof;
5992   }
5993   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
5994   /* Row indices */
5995   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
5996   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
5997   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
5998   for (r = 0, q = 0; r < numSubcells; ++r) {
5999     /* TODO Map from coarse to fine cells */
6000     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6001     /* Compress out points not in the section */
6002     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6003     for (p = 0; p < numFPoints*2; p += 2) {
6004       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6005         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6006         if (!dof) continue;
6007         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6008         if (s < q) continue;
6009         ftotpoints[q*2]   = fpoints[p];
6010         ftotpoints[q*2+1] = fpoints[p+1];
6011         ++q;
6012       }
6013     }
6014     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6015   }
6016   numFPoints = q;
6017   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6018     PetscInt fdof;
6019 
6020     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6021     if (!dof) continue;
6022     for (f = 0; f < numFields; ++f) {
6023       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6024       foffsets[f+1] += fdof;
6025     }
6026     numFIndices += dof;
6027   }
6028   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6029 
6030   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6031   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6032   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6033   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6034   if (numFields) {
6035     const PetscInt **permsF[32] = {NULL};
6036     const PetscInt **permsC[32] = {NULL};
6037 
6038     for (f = 0; f < numFields; f++) {
6039       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6040       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6041     }
6042     for (p = 0; p < numFPoints; p++) {
6043       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6044       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6045     }
6046     for (p = 0; p < numCPoints; p++) {
6047       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6048       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6049     }
6050     for (f = 0; f < numFields; f++) {
6051       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6052       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6053     }
6054   } else {
6055     const PetscInt **permsF = NULL;
6056     const PetscInt **permsC = NULL;
6057 
6058     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6059     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6060     for (p = 0, off = 0; p < numFPoints; p++) {
6061       const PetscInt *perm = permsF ? permsF[p] : NULL;
6062 
6063       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6064       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6065     }
6066     for (p = 0, off = 0; p < numCPoints; p++) {
6067       const PetscInt *perm = permsC ? permsC[p] : NULL;
6068 
6069       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6070       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6071     }
6072     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6073     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6074   }
6075   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6076   /* TODO: flips */
6077   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6078   if (ierr) {
6079     PetscMPIInt    rank;
6080     PetscErrorCode ierr2;
6081 
6082     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6083     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6084     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6085     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6086     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6087     CHKERRQ(ierr);
6088   }
6089   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6090   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6091   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6092   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6093   PetscFunctionReturn(0);
6094 }
6095 
6096 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6097 {
6098   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6099   PetscInt      *cpoints = NULL;
6100   PetscInt       foffsets[32], coffsets[32];
6101   const PetscInt *fclperm, *cclperm;
6102   CellRefiner    cellRefiner;
6103   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6104   PetscErrorCode ierr;
6105 
6106   PetscFunctionBegin;
6107   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6108   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6109   if (!fsection) {ierr = DMGetSection(dmf, &fsection);CHKERRQ(ierr);}
6110   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6111   if (!csection) {ierr = DMGetSection(dmc, &csection);CHKERRQ(ierr);}
6112   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6113   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6114   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6115   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6116   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6117   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6118   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6119   ierr = PetscMemzero(foffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
6120   ierr = PetscMemzero(coffsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
6121   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6122   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6123   /* Column indices */
6124   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6125   maxFPoints = numCPoints;
6126   /* Compress out points not in the section */
6127   /*   TODO: Squeeze out points with 0 dof as well */
6128   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6129   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6130     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6131       cpoints[q*2]   = cpoints[p];
6132       cpoints[q*2+1] = cpoints[p+1];
6133       ++q;
6134     }
6135   }
6136   numCPoints = q;
6137   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6138     PetscInt fdof;
6139 
6140     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6141     if (!dof) continue;
6142     for (f = 0; f < numFields; ++f) {
6143       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6144       coffsets[f+1] += fdof;
6145     }
6146     numCIndices += dof;
6147   }
6148   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6149   /* Row indices */
6150   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6151   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6152   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6153   for (r = 0, q = 0; r < numSubcells; ++r) {
6154     /* TODO Map from coarse to fine cells */
6155     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6156     /* Compress out points not in the section */
6157     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6158     for (p = 0; p < numFPoints*2; p += 2) {
6159       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6160         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6161         if (!dof) continue;
6162         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6163         if (s < q) continue;
6164         ftotpoints[q*2]   = fpoints[p];
6165         ftotpoints[q*2+1] = fpoints[p+1];
6166         ++q;
6167       }
6168     }
6169     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6170   }
6171   numFPoints = q;
6172   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6173     PetscInt fdof;
6174 
6175     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6176     if (!dof) continue;
6177     for (f = 0; f < numFields; ++f) {
6178       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6179       foffsets[f+1] += fdof;
6180     }
6181     numFIndices += dof;
6182   }
6183   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6184 
6185   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6186   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6187   if (numFields) {
6188     const PetscInt **permsF[32] = {NULL};
6189     const PetscInt **permsC[32] = {NULL};
6190 
6191     for (f = 0; f < numFields; f++) {
6192       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6193       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6194     }
6195     for (p = 0; p < numFPoints; p++) {
6196       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6197       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6198     }
6199     for (p = 0; p < numCPoints; p++) {
6200       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6201       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6202     }
6203     for (f = 0; f < numFields; f++) {
6204       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6205       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6206     }
6207   } else {
6208     const PetscInt **permsF = NULL;
6209     const PetscInt **permsC = NULL;
6210 
6211     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6212     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6213     for (p = 0, off = 0; p < numFPoints; p++) {
6214       const PetscInt *perm = permsF ? permsF[p] : NULL;
6215 
6216       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6217       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6218     }
6219     for (p = 0, off = 0; p < numCPoints; p++) {
6220       const PetscInt *perm = permsC ? permsC[p] : NULL;
6221 
6222       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6223       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6224     }
6225     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6226     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6227   }
6228   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6229   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6230   PetscFunctionReturn(0);
6231 }
6232 
6233 /*@
6234   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6235 
6236   Input Parameter:
6237 . dm - The DMPlex object
6238 
6239   Output Parameters:
6240 + cMax - The first hybrid cell
6241 . fMax - The first hybrid face
6242 . eMax - The first hybrid edge
6243 - vMax - The first hybrid vertex
6244 
6245   Level: developer
6246 
6247 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6248 @*/
6249 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6250 {
6251   DM_Plex       *mesh = (DM_Plex*) dm->data;
6252   PetscInt       dim;
6253   PetscErrorCode ierr;
6254 
6255   PetscFunctionBegin;
6256   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6257   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6258   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6259   if (cMax) *cMax = mesh->hybridPointMax[dim];
6260   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6261   if (eMax) *eMax = mesh->hybridPointMax[1];
6262   if (vMax) *vMax = mesh->hybridPointMax[0];
6263   PetscFunctionReturn(0);
6264 }
6265 
6266 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6267 {
6268   IS             is, his;
6269   PetscInt       first = 0, stride;
6270   PetscBool      isStride;
6271   PetscErrorCode ierr;
6272 
6273   PetscFunctionBegin;
6274   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6275   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6276   if (isStride) {
6277     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6278   }
6279   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6280   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6281   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6282   ierr = ISDestroy(&his);CHKERRQ(ierr);
6283   ierr = ISDestroy(&is);CHKERRQ(ierr);
6284   PetscFunctionReturn(0);
6285 }
6286 
6287 /*@
6288   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6289 
6290   Input Parameters:
6291 . dm   - The DMPlex object
6292 . cMax - The first hybrid cell
6293 . fMax - The first hybrid face
6294 . eMax - The first hybrid edge
6295 - vMax - The first hybrid vertex
6296 
6297   Level: developer
6298 
6299 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6300 @*/
6301 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6302 {
6303   DM_Plex       *mesh = (DM_Plex*) dm->data;
6304   PetscInt       dim;
6305   PetscErrorCode ierr;
6306 
6307   PetscFunctionBegin;
6308   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6309   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6310   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6311   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6312   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6313   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6314   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6315   PetscFunctionReturn(0);
6316 }
6317 
6318 /*@C
6319   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6320 
6321   Input Parameter:
6322 . dm   - The DMPlex object
6323 
6324   Output Parameter:
6325 . cellHeight - The height of a cell
6326 
6327   Level: developer
6328 
6329 .seealso DMPlexSetVTKCellHeight()
6330 @*/
6331 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6332 {
6333   DM_Plex *mesh = (DM_Plex*) dm->data;
6334 
6335   PetscFunctionBegin;
6336   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6337   PetscValidPointer(cellHeight, 2);
6338   *cellHeight = mesh->vtkCellHeight;
6339   PetscFunctionReturn(0);
6340 }
6341 
6342 /*@C
6343   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6344 
6345   Input Parameters:
6346 + dm   - The DMPlex object
6347 - cellHeight - The height of a cell
6348 
6349   Level: developer
6350 
6351 .seealso DMPlexGetVTKCellHeight()
6352 @*/
6353 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6354 {
6355   DM_Plex *mesh = (DM_Plex*) dm->data;
6356 
6357   PetscFunctionBegin;
6358   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6359   mesh->vtkCellHeight = cellHeight;
6360   PetscFunctionReturn(0);
6361 }
6362 
6363 /* We can easily have a form that takes an IS instead */
6364 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6365 {
6366   PetscSection   section, globalSection;
6367   PetscInt      *numbers, p;
6368   PetscErrorCode ierr;
6369 
6370   PetscFunctionBegin;
6371   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6372   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6373   for (p = pStart; p < pEnd; ++p) {
6374     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6375   }
6376   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6377   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6378   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6379   for (p = pStart; p < pEnd; ++p) {
6380     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6381     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6382     else                       numbers[p-pStart] += shift;
6383   }
6384   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6385   if (globalSize) {
6386     PetscLayout layout;
6387     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6388     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6389     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6390   }
6391   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6392   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6393   PetscFunctionReturn(0);
6394 }
6395 
6396 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6397 {
6398   PetscInt       cellHeight, cStart, cEnd, cMax;
6399   PetscErrorCode ierr;
6400 
6401   PetscFunctionBegin;
6402   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6403   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6404   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6405   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6406   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6407   PetscFunctionReturn(0);
6408 }
6409 
6410 /*@
6411   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6412 
6413   Input Parameter:
6414 . dm   - The DMPlex object
6415 
6416   Output Parameter:
6417 . globalCellNumbers - Global cell numbers for all cells on this process
6418 
6419   Level: developer
6420 
6421 .seealso DMPlexGetVertexNumbering()
6422 @*/
6423 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6424 {
6425   DM_Plex       *mesh = (DM_Plex*) dm->data;
6426   PetscErrorCode ierr;
6427 
6428   PetscFunctionBegin;
6429   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6430   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6431   *globalCellNumbers = mesh->globalCellNumbers;
6432   PetscFunctionReturn(0);
6433 }
6434 
6435 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6436 {
6437   PetscInt       vStart, vEnd, vMax;
6438   PetscErrorCode ierr;
6439 
6440   PetscFunctionBegin;
6441   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6442   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6443   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6444   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6445   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6446   PetscFunctionReturn(0);
6447 }
6448 
6449 /*@
6450   DMPlexGetVertexNumbering - Get a global certex numbering for all vertices on this process
6451 
6452   Input Parameter:
6453 . dm   - The DMPlex object
6454 
6455   Output Parameter:
6456 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6457 
6458   Level: developer
6459 
6460 .seealso DMPlexGetCellNumbering()
6461 @*/
6462 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6463 {
6464   DM_Plex       *mesh = (DM_Plex*) dm->data;
6465   PetscErrorCode ierr;
6466 
6467   PetscFunctionBegin;
6468   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6469   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6470   *globalVertexNumbers = mesh->globalVertexNumbers;
6471   PetscFunctionReturn(0);
6472 }
6473 
6474 /*@
6475   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6476 
6477   Input Parameter:
6478 . dm   - The DMPlex object
6479 
6480   Output Parameter:
6481 . globalPointNumbers - Global numbers for all points on this process
6482 
6483   Level: developer
6484 
6485 .seealso DMPlexGetCellNumbering()
6486 @*/
6487 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6488 {
6489   IS             nums[4];
6490   PetscInt       depths[4], gdepths[4], starts[4];
6491   PetscInt       depth, d, shift = 0;
6492   PetscErrorCode ierr;
6493 
6494   PetscFunctionBegin;
6495   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6496   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6497   /* For unstratified meshes use dim instead of depth */
6498   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6499   for (d = 0; d <= depth; ++d) {
6500     PetscInt end;
6501 
6502     depths[d] = depth-d;
6503     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6504     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6505   }
6506   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6507   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6508   for (d = 0; d <= depth; ++d) {
6509     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6510   }
6511   for (d = 0; d <= depth; ++d) {
6512     PetscInt pStart, pEnd, gsize;
6513 
6514     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6515     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6516     shift += gsize;
6517   }
6518   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6519   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6520   PetscFunctionReturn(0);
6521 }
6522 
6523 
6524 /*@
6525   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6526 
6527   Input Parameter:
6528 . dm - The DMPlex object
6529 
6530   Output Parameter:
6531 . ranks - The rank field
6532 
6533   Options Database Keys:
6534 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6535 
6536   Level: intermediate
6537 
6538 .seealso: DMView()
6539 @*/
6540 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6541 {
6542   DM             rdm;
6543   PetscFE        fe;
6544   PetscScalar   *r;
6545   PetscMPIInt    rank;
6546   PetscInt       dim, cStart, cEnd, c;
6547   PetscErrorCode ierr;
6548 
6549   PetscFunctionBeginUser;
6550   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6551   PetscValidPointer(ranks, 2);
6552   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6553   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6554   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6555   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6556   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6557   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6558   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6559   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6560   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6561   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6562   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6563   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6564   for (c = cStart; c < cEnd; ++c) {
6565     PetscScalar *lr;
6566 
6567     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6568     *lr = rank;
6569   }
6570   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6571   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6572   PetscFunctionReturn(0);
6573 }
6574 
6575 /*@
6576   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6577 
6578   Input Parameters:
6579 + dm    - The DMPlex
6580 - label - The DMLabel
6581 
6582   Output Parameter:
6583 . val - The label value field
6584 
6585   Options Database Keys:
6586 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6587 
6588   Level: intermediate
6589 
6590 .seealso: DMView()
6591 @*/
6592 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6593 {
6594   DM             rdm;
6595   PetscFE        fe;
6596   PetscScalar   *v;
6597   PetscInt       dim, cStart, cEnd, c;
6598   PetscErrorCode ierr;
6599 
6600   PetscFunctionBeginUser;
6601   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6602   PetscValidPointer(label, 2);
6603   PetscValidPointer(val, 3);
6604   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6605   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6606   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6607   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6608   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6609   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6610   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6611   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6612   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6613   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6614   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6615   for (c = cStart; c < cEnd; ++c) {
6616     PetscScalar *lv;
6617     PetscInt     cval;
6618 
6619     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6620     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6621     *lv = cval;
6622   }
6623   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6624   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6625   PetscFunctionReturn(0);
6626 }
6627 
6628 /*@
6629   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6630 
6631   Input Parameter:
6632 . dm - The DMPlex object
6633 
6634   Note: This is a useful diagnostic when creating meshes programmatically.
6635 
6636   Level: developer
6637 
6638 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6639 @*/
6640 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6641 {
6642   PetscSection    coneSection, supportSection;
6643   const PetscInt *cone, *support;
6644   PetscInt        coneSize, c, supportSize, s;
6645   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6646   PetscBool       storagecheck = PETSC_TRUE;
6647   PetscErrorCode  ierr;
6648 
6649   PetscFunctionBegin;
6650   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6651   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6652   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6653   /* Check that point p is found in the support of its cone points, and vice versa */
6654   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6655   for (p = pStart; p < pEnd; ++p) {
6656     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6657     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6658     for (c = 0; c < coneSize; ++c) {
6659       PetscBool dup = PETSC_FALSE;
6660       PetscInt  d;
6661       for (d = c-1; d >= 0; --d) {
6662         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6663       }
6664       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6665       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6666       for (s = 0; s < supportSize; ++s) {
6667         if (support[s] == p) break;
6668       }
6669       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6670         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6671         for (s = 0; s < coneSize; ++s) {
6672           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6673         }
6674         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6675         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6676         for (s = 0; s < supportSize; ++s) {
6677           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6678         }
6679         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6680         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6681         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6682       }
6683     }
6684     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6685     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6686     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6687     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6688     for (s = 0; s < supportSize; ++s) {
6689       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6690       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6691       for (c = 0; c < coneSize; ++c) {
6692         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6693         if (cone[c] != pp) { c = 0; break; }
6694         if (cone[c] == p) break;
6695       }
6696       if (c >= coneSize) {
6697         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6698         for (c = 0; c < supportSize; ++c) {
6699           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6700         }
6701         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6702         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6703         for (c = 0; c < coneSize; ++c) {
6704           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6705         }
6706         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6707         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6708       }
6709     }
6710   }
6711   if (storagecheck) {
6712     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6713     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6714     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6715   }
6716   PetscFunctionReturn(0);
6717 }
6718 
6719 /*@
6720   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6721 
6722   Input Parameters:
6723 + dm - The DMPlex object
6724 - cellHeight - Normally 0
6725 
6726   Note: This is a useful diagnostic when creating meshes programmatically.
6727   Currently applicable only to homogeneous simplex or tensor meshes.
6728 
6729   Level: developer
6730 
6731 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6732 @*/
6733 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6734 {
6735   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6736   PetscBool      isSimplex = PETSC_FALSE;
6737   PetscErrorCode ierr;
6738 
6739   PetscFunctionBegin;
6740   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6741   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6742   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6743   if (cStart < cEnd) {
6744     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6745     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6746   }
6747   switch (dim) {
6748   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6749   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6750   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6751   default:
6752     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6753   }
6754   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6755   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6756   cMax = cMax >= 0 ? cMax : cEnd;
6757   for (c = cStart; c < cMax; ++c) {
6758     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6759 
6760     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6761     for (cl = 0; cl < closureSize*2; cl += 2) {
6762       const PetscInt p = closure[cl];
6763       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6764     }
6765     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6766     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6767   }
6768   for (c = cMax; c < cEnd; ++c) {
6769     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6770 
6771     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6772     for (cl = 0; cl < closureSize*2; cl += 2) {
6773       const PetscInt p = closure[cl];
6774       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6775     }
6776     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6777     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6778   }
6779   PetscFunctionReturn(0);
6780 }
6781 
6782 /*@
6783   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6784 
6785   Input Parameters:
6786 + dm - The DMPlex object
6787 - cellHeight - Normally 0
6788 
6789   Note: This is a useful diagnostic when creating meshes programmatically.
6790 
6791   Level: developer
6792 
6793 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6794 @*/
6795 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6796 {
6797   PetscInt       pMax[4];
6798   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6799   PetscErrorCode ierr;
6800 
6801   PetscFunctionBegin;
6802   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6803   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6804   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6805   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6806   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6807   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6808     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6809     for (c = cStart; c < cEnd; ++c) {
6810       const PetscInt *cone, *ornt, *faces;
6811       PetscInt        numFaces, faceSize, coneSize,f;
6812       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6813 
6814       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6815       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6816       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6817       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6818       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6819       for (cl = 0; cl < closureSize*2; cl += 2) {
6820         const PetscInt p = closure[cl];
6821         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6822       }
6823       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6824       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6825       for (f = 0; f < numFaces; ++f) {
6826         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6827 
6828         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6829         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6830           const PetscInt p = fclosure[cl];
6831           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6832         }
6833         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6834         for (v = 0; v < fnumCorners; ++v) {
6835           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6836         }
6837         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6838       }
6839       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6840       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6841     }
6842   }
6843   PetscFunctionReturn(0);
6844 }
6845 
6846 /*@
6847   DMPlexCheckGeometry - Check the geometry of mesh cells
6848 
6849   Input Parameter:
6850 . dm - The DMPlex object
6851 
6852   Note: This is a useful diagnostic when creating meshes programmatically.
6853 
6854   Level: developer
6855 
6856 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
6857 @*/
6858 PetscErrorCode DMPlexCheckGeometry(DM dm)
6859 {
6860   PetscReal      detJ, J[9], refVol = 1.0;
6861   PetscReal      vol;
6862   PetscInt       dim, depth, d, cStart, cEnd, c;
6863   PetscErrorCode ierr;
6864 
6865   PetscFunctionBegin;
6866   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6867   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6868   for (d = 0; d < dim; ++d) refVol *= 2.0;
6869   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6870   for (c = cStart; c < cEnd; ++c) {
6871     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
6872     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
6873     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
6874     if (depth > 1) {
6875       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
6876       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
6877       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
6878     }
6879   }
6880   PetscFunctionReturn(0);
6881 }
6882 
6883 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
6884 {
6885   PetscInt i,l,n;
6886   const PetscInt *cone;
6887   PetscErrorCode ierr;
6888 
6889   PetscFunctionBegin;
6890   *missingPoint = -1;
6891   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
6892   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6893   for (i=0; i<n; i++) {
6894     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
6895     if (l < 0) {
6896       *missingPoint = cone[i];
6897       break;
6898     }
6899   }
6900   PetscFunctionReturn(0);
6901 }
6902 
6903 /*@
6904   DMPlexCheckPointSF - Check that several sufficient conditions are met for the point SF of this plex.
6905 
6906   Input Parameters:
6907 . dm - The DMPlex object
6908 
6909   Note: This is mainly intended for debugging/testing purposes.
6910 
6911   Level: developer
6912 
6913 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6914 @*/
6915 PetscErrorCode DMPlexCheckPointSF(DM dm)
6916 {
6917   PetscSF sf;
6918   PetscInt d,depth,i,nleaves,p,plo,phi,missingPoint;
6919   const PetscInt *locals;
6920   PetscErrorCode ierr;
6921 
6922   PetscFunctionBegin;
6923   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6924   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6925   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
6926   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
6927 
6928   /* 1) check there are no faces in 2D, cells in 3D, in interface */
6929   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
6930   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
6931   for (i=0; i<nleaves; i++) {
6932     p = locals[i];
6933     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
6934   }
6935 
6936   /* 2) if some point is in interface, then all its cone points must be also in interface  */
6937   for (i=0; i<nleaves; i++) {
6938     p = locals[i];
6939     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
6940     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
6941   }
6942   PetscFunctionReturn(0);
6943 }
6944 
6945 typedef struct cell_stats
6946 {
6947   PetscReal min, max, sum, squaresum;
6948   PetscInt  count;
6949 } cell_stats_t;
6950 
6951 static void cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
6952 {
6953   PetscInt i, N = *len;
6954 
6955   for (i = 0; i < N; i++) {
6956     cell_stats_t *A = (cell_stats_t *) a;
6957     cell_stats_t *B = (cell_stats_t *) b;
6958 
6959     B->min = PetscMin(A->min,B->min);
6960     B->max = PetscMax(A->max,B->max);
6961     B->sum += A->sum;
6962     B->squaresum += A->squaresum;
6963     B->count += A->count;
6964   }
6965 }
6966 
6967 /*@
6968   DMPlexCheckCellShape - Checks the Jacobian of the mapping and computes some minimal statistics.
6969 
6970   Input Parameters:
6971 + dm - The DMPlex object
6972 - output - If true, statistics will be displayed on stdout
6973 
6974   Note: This is mainly intended for debugging/testing purposes.
6975 
6976   Level: developer
6977 
6978 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6979 @*/
6980 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output)
6981 {
6982   PetscMPIInt    rank,size;
6983   PetscInt       dim, c, cStart, cEnd, cMax, count = 0;
6984   cell_stats_t   stats, globalStats;
6985   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
6986   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
6987   DM             dmCoarse;
6988   PetscErrorCode ierr;
6989 
6990   PetscFunctionBegin;
6991   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6992   stats.min   = PETSC_MAX_REAL;
6993   stats.max   = PETSC_MIN_REAL;
6994   stats.sum   = stats.squaresum = 0.;
6995   stats.count = 0;
6996 
6997   ierr = DMGetCoordinateDim(dm,&dim);CHKERRQ(ierr);
6998   ierr = PetscMalloc2(dim * dim, &J, dim * dim, &invJ);CHKERRQ(ierr);
6999   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7000   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7001   cMax = cMax < 0 ? cEnd : cMax;
7002   for (c = cStart; c < cMax; c++) {
7003     PetscInt  i;
7004     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7005 
7006     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7007     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7008     for (i = 0; i < dim * dim; i++) {
7009       frobJ    += J[i] * J[i];
7010       frobInvJ += invJ[i] * invJ[i];
7011     }
7012     cond2 = frobJ * frobInvJ;
7013     cond  = PetscSqrtReal(cond2);
7014 
7015     stats.min        = PetscMin(stats.min,cond);
7016     stats.max        = PetscMax(stats.max,cond);
7017     stats.sum       += cond;
7018     stats.squaresum += cond2;
7019     stats.count++;
7020   }
7021 
7022   ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
7023   if (size > 1) {
7024     PetscMPIInt   blockLengths[2] = {4,1};
7025     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7026     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7027     MPI_Op        statReduce;
7028 
7029     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7030     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7031     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7032     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7033     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7034     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7035   } else {
7036     ierr = PetscMemcpy(&globalStats,&stats,sizeof(stats));CHKERRQ(ierr);
7037   }
7038 
7039   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
7040   if (!rank) {
7041     count = globalStats.count;
7042     min   = globalStats.min;
7043     max   = globalStats.max;
7044     mean  = globalStats.sum / globalStats.count;
7045     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7046   }
7047 
7048   if (output) {
7049     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7050   }
7051   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7052 
7053   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7054   if (dmCoarse) {
7055     PetscBool isplex;
7056 
7057     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7058     if (isplex) {
7059       ierr = DMPlexCheckCellShape(dmCoarse,output);CHKERRQ(ierr);
7060     }
7061   }
7062   PetscFunctionReturn(0);
7063 }
7064 
7065 /* Pointwise interpolation
7066      Just code FEM for now
7067      u^f = I u^c
7068      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7069      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7070      I_{ij} = psi^f_i phi^c_j
7071 */
7072 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7073 {
7074   PetscSection   gsc, gsf;
7075   PetscInt       m, n;
7076   void          *ctx;
7077   DM             cdm;
7078   PetscBool      regular, ismatis;
7079   PetscErrorCode ierr;
7080 
7081   PetscFunctionBegin;
7082   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7083   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7084   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7085   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7086 
7087   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7088   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7089   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7090   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7091   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7092 
7093   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7094   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7095   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7096   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7097   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7098   if (scaling) {
7099     /* Use naive scaling */
7100     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7101   }
7102   PetscFunctionReturn(0);
7103 }
7104 
7105 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7106 {
7107   PetscErrorCode ierr;
7108   VecScatter     ctx;
7109 
7110   PetscFunctionBegin;
7111   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7112   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7113   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7114   PetscFunctionReturn(0);
7115 }
7116 
7117 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7118 {
7119   PetscSection   gsc, gsf;
7120   PetscInt       m, n;
7121   void          *ctx;
7122   DM             cdm;
7123   PetscBool      regular;
7124   PetscErrorCode ierr;
7125 
7126   PetscFunctionBegin;
7127   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7128   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7129   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7130   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7131 
7132   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7133   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7134   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7135   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7136 
7137   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7138   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7139   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7140   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7141   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7142   PetscFunctionReturn(0);
7143 }
7144 
7145 /*@
7146   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7147 
7148   Input Parameter:
7149 . dm - The DMPlex object
7150 
7151   Output Parameter:
7152 . regular - The flag
7153 
7154   Level: intermediate
7155 
7156 .seealso: DMPlexSetRegularRefinement()
7157 @*/
7158 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7159 {
7160   PetscFunctionBegin;
7161   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7162   PetscValidPointer(regular, 2);
7163   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7164   PetscFunctionReturn(0);
7165 }
7166 
7167 /*@
7168   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7169 
7170   Input Parameters:
7171 + dm - The DMPlex object
7172 - regular - The flag
7173 
7174   Level: intermediate
7175 
7176 .seealso: DMPlexGetRegularRefinement()
7177 @*/
7178 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7179 {
7180   PetscFunctionBegin;
7181   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7182   ((DM_Plex *) dm->data)->regularRefinement = regular;
7183   PetscFunctionReturn(0);
7184 }
7185 
7186 /* anchors */
7187 /*@
7188   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7189   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7190 
7191   not collective
7192 
7193   Input Parameters:
7194 . dm - The DMPlex object
7195 
7196   Output Parameters:
7197 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7198 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7199 
7200 
7201   Level: intermediate
7202 
7203 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7204 @*/
7205 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7206 {
7207   DM_Plex *plex = (DM_Plex *)dm->data;
7208   PetscErrorCode ierr;
7209 
7210   PetscFunctionBegin;
7211   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7212   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7213   if (anchorSection) *anchorSection = plex->anchorSection;
7214   if (anchorIS) *anchorIS = plex->anchorIS;
7215   PetscFunctionReturn(0);
7216 }
7217 
7218 /*@
7219   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7220   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7221   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7222 
7223   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7224   DMGetConstraints() and filling in the entries in the constraint matrix.
7225 
7226   collective on dm
7227 
7228   Input Parameters:
7229 + dm - The DMPlex object
7230 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7231 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7232 
7233   The reference counts of anchorSection and anchorIS are incremented.
7234 
7235   Level: intermediate
7236 
7237 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7238 @*/
7239 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7240 {
7241   DM_Plex        *plex = (DM_Plex *)dm->data;
7242   PetscMPIInt    result;
7243   PetscErrorCode ierr;
7244 
7245   PetscFunctionBegin;
7246   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7247   if (anchorSection) {
7248     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7249     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7250     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7251   }
7252   if (anchorIS) {
7253     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7254     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7255     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7256   }
7257 
7258   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7259   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7260   plex->anchorSection = anchorSection;
7261 
7262   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7263   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7264   plex->anchorIS = anchorIS;
7265 
7266 #if defined(PETSC_USE_DEBUG)
7267   if (anchorIS && anchorSection) {
7268     PetscInt size, a, pStart, pEnd;
7269     const PetscInt *anchors;
7270 
7271     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7272     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7273     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7274     for (a = 0; a < size; a++) {
7275       PetscInt p;
7276 
7277       p = anchors[a];
7278       if (p >= pStart && p < pEnd) {
7279         PetscInt dof;
7280 
7281         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7282         if (dof) {
7283           PetscErrorCode ierr2;
7284 
7285           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7286           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7287         }
7288       }
7289     }
7290     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7291   }
7292 #endif
7293   /* reset the generic constraints */
7294   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7295   PetscFunctionReturn(0);
7296 }
7297 
7298 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7299 {
7300   PetscSection anchorSection;
7301   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7302   PetscErrorCode ierr;
7303 
7304   PetscFunctionBegin;
7305   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7306   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7307   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7308   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7309   if (numFields) {
7310     PetscInt f;
7311     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7312 
7313     for (f = 0; f < numFields; f++) {
7314       PetscInt numComp;
7315 
7316       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7317       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7318     }
7319   }
7320   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7321   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7322   pStart = PetscMax(pStart,sStart);
7323   pEnd   = PetscMin(pEnd,sEnd);
7324   pEnd   = PetscMax(pStart,pEnd);
7325   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7326   for (p = pStart; p < pEnd; p++) {
7327     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7328     if (dof) {
7329       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7330       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7331       for (f = 0; f < numFields; f++) {
7332         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7333         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7334       }
7335     }
7336   }
7337   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7338   PetscFunctionReturn(0);
7339 }
7340 
7341 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7342 {
7343   PetscSection aSec;
7344   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7345   const PetscInt *anchors;
7346   PetscInt numFields, f;
7347   IS aIS;
7348   PetscErrorCode ierr;
7349 
7350   PetscFunctionBegin;
7351   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7352   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7353   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7354   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7355   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7356   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7357   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7358   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7359   /* cSec will be a subset of aSec and section */
7360   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7361   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7362   i[0] = 0;
7363   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7364   for (p = pStart; p < pEnd; p++) {
7365     PetscInt rDof, rOff, r;
7366 
7367     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7368     if (!rDof) continue;
7369     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7370     if (numFields) {
7371       for (f = 0; f < numFields; f++) {
7372         annz = 0;
7373         for (r = 0; r < rDof; r++) {
7374           a = anchors[rOff + r];
7375           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7376           annz += aDof;
7377         }
7378         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7379         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7380         for (q = 0; q < dof; q++) {
7381           i[off + q + 1] = i[off + q] + annz;
7382         }
7383       }
7384     }
7385     else {
7386       annz = 0;
7387       for (q = 0; q < dof; q++) {
7388         a = anchors[off + q];
7389         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7390         annz += aDof;
7391       }
7392       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7393       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7394       for (q = 0; q < dof; q++) {
7395         i[off + q + 1] = i[off + q] + annz;
7396       }
7397     }
7398   }
7399   nnz = i[m];
7400   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7401   offset = 0;
7402   for (p = pStart; p < pEnd; p++) {
7403     if (numFields) {
7404       for (f = 0; f < numFields; f++) {
7405         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7406         for (q = 0; q < dof; q++) {
7407           PetscInt rDof, rOff, r;
7408           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7409           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7410           for (r = 0; r < rDof; r++) {
7411             PetscInt s;
7412 
7413             a = anchors[rOff + r];
7414             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7415             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7416             for (s = 0; s < aDof; s++) {
7417               j[offset++] = aOff + s;
7418             }
7419           }
7420         }
7421       }
7422     }
7423     else {
7424       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7425       for (q = 0; q < dof; q++) {
7426         PetscInt rDof, rOff, r;
7427         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7428         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7429         for (r = 0; r < rDof; r++) {
7430           PetscInt s;
7431 
7432           a = anchors[rOff + r];
7433           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7434           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7435           for (s = 0; s < aDof; s++) {
7436             j[offset++] = aOff + s;
7437           }
7438         }
7439       }
7440     }
7441   }
7442   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7443   ierr = PetscFree(i);CHKERRQ(ierr);
7444   ierr = PetscFree(j);CHKERRQ(ierr);
7445   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7446   PetscFunctionReturn(0);
7447 }
7448 
7449 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7450 {
7451   DM_Plex        *plex = (DM_Plex *)dm->data;
7452   PetscSection   anchorSection, section, cSec;
7453   Mat            cMat;
7454   PetscErrorCode ierr;
7455 
7456   PetscFunctionBegin;
7457   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7458   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7459   if (anchorSection) {
7460     PetscInt Nf;
7461 
7462     ierr = DMGetSection(dm,&section);CHKERRQ(ierr);
7463     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7464     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7465     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7466     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7467     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7468     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7469     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7470   }
7471   PetscFunctionReturn(0);
7472 }
7473 
7474 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7475 {
7476   IS             subis;
7477   PetscSection   section, subsection;
7478   PetscErrorCode ierr;
7479 
7480   PetscFunctionBegin;
7481   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
7482   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7483   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7484   /* Create subdomain */
7485   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7486   /* Create submodel */
7487   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7488   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7489   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7490   ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
7491   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7492   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7493   /* Create map from submodel to global model */
7494   if (is) {
7495     PetscSection    sectionGlobal, subsectionGlobal;
7496     IS              spIS;
7497     const PetscInt *spmap;
7498     PetscInt       *subIndices;
7499     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7500     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7501 
7502     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7503     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7504     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7505     ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7506     ierr = DMGetDefaultGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7507     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7508     for (p = pStart; p < pEnd; ++p) {
7509       PetscInt gdof, pSubSize  = 0;
7510 
7511       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7512       if (gdof > 0) {
7513         for (f = 0; f < Nf; ++f) {
7514           PetscInt fdof, fcdof;
7515 
7516           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7517           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7518           pSubSize += fdof-fcdof;
7519         }
7520         subSize += pSubSize;
7521         if (pSubSize) {
7522           if (bs < 0) {
7523             bs = pSubSize;
7524           } else if (bs != pSubSize) {
7525             /* Layout does not admit a pointwise block size */
7526             bs = 1;
7527           }
7528         }
7529       }
7530     }
7531     /* Must have same blocksize on all procs (some might have no points) */
7532     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7533     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7534     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7535     else                            {bs = bsMinMax[0];}
7536     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7537     for (p = pStart; p < pEnd; ++p) {
7538       PetscInt gdof, goff;
7539 
7540       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7541       if (gdof > 0) {
7542         const PetscInt point = spmap[p];
7543 
7544         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7545         for (f = 0; f < Nf; ++f) {
7546           PetscInt fdof, fcdof, fc, f2, poff = 0;
7547 
7548           /* Can get rid of this loop by storing field information in the global section */
7549           for (f2 = 0; f2 < f; ++f2) {
7550             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7551             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7552             poff += fdof-fcdof;
7553           }
7554           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7555           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7556           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7557             subIndices[subOff] = goff+poff+fc;
7558           }
7559         }
7560       }
7561     }
7562     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7563     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7564     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7565     if (bs > 1) {
7566       /* We need to check that the block size does not come from non-contiguous fields */
7567       PetscInt i, j, set = 1;
7568       for (i = 0; i < subSize; i += bs) {
7569         for (j = 0; j < bs; ++j) {
7570           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7571         }
7572       }
7573       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7574     }
7575     /* Attach nullspace */
7576     for (f = 0; f < Nf; ++f) {
7577       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7578       if ((*subdm)->nullspaceConstructors[f]) break;
7579     }
7580     if (f < Nf) {
7581       MatNullSpace nullSpace;
7582 
7583       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7584       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7585       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7586     }
7587   }
7588   PetscFunctionReturn(0);
7589 }
7590