xref: /petsc/src/dm/impls/plex/plex.c (revision 0040bde18f069f966ff342adc5a53be3404b34fd)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     char         lname[PETSC_MAX_PATH_LEN];
675     PetscReal    scale         = 2.0;
676     PetscReal    tikzscale     = 1.0;
677     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
678     double       tcoords[3];
679     PetscScalar *coords;
680     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
681     PetscMPIInt  rank, size;
682     char         **names, **colors, **lcolors;
683     PetscBool    plotEdges, flg, lflg;
684     PetscBT      wp = NULL;
685     PetscInt     pEnd, pStart;
686 
687     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
688     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
689     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
690     numLabels  = PetscMax(numLabels, 10);
691     numColors  = 10;
692     numLColors = 10;
693     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
694     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
695     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
696     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
697     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
698     if (!useLabels) numLabels = 0;
699     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
700     if (!useColors) {
701       numColors = 3;
702       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
703     }
704     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
705     if (!useColors) {
706       numLColors = 4;
707       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
708     }
709     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
710     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
711     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
712     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
713     if (depth < dim) plotEdges = PETSC_FALSE;
714 
715     /* filter points with labelvalue != labeldefaultvalue */
716     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
717     if (lflg) {
718       DMLabel lbl;
719 
720       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
721       if (lbl) {
722         PetscInt val, defval;
723 
724         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
725         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
726         for (c = pStart;  c < pEnd; c++) {
727           PetscInt *closure = NULL;
728           PetscInt  closureSize;
729 
730           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
731           if (val == defval) continue;
732 
733           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
734           for (p = 0; p < closureSize*2; p += 2) {
735             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
736           }
737           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
738         }
739       }
740     }
741 
742     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
743     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
744     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
745     ierr = PetscViewerASCIIPrintf(viewer, "\
746 \\documentclass[tikz]{standalone}\n\n\
747 \\usepackage{pgflibraryshapes}\n\
748 \\usetikzlibrary{backgrounds}\n\
749 \\usetikzlibrary{arrows}\n\
750 \\begin{document}\n");CHKERRQ(ierr);
751     if (size > 1) {
752       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
753       for (p = 0; p < size; ++p) {
754         if (p > 0 && p == size-1) {
755           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
756         } else if (p > 0) {
757           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
758         }
759         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
760       }
761       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
762     }
763     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
764 
765     /* Plot vertices */
766     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
767     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
768     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
769     for (v = vStart; v < vEnd; ++v) {
770       PetscInt  off, dof, d;
771       PetscBool isLabeled = PETSC_FALSE;
772 
773       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
774       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
775       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
776       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
777       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
778       for (d = 0; d < dof; ++d) {
779         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
780         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
781       }
782       /* Rotate coordinates since PGF makes z point out of the page instead of up */
783       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
784       for (d = 0; d < dof; ++d) {
785         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
786         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
787       }
788       color = colors[rank%numColors];
789       for (l = 0; l < numLabels; ++l) {
790         PetscInt val;
791         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
792         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
793       }
794       if (useNumbers) {
795         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
796       } else {
797         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
798       }
799     }
800     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
801     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
802     /* Plot cells */
803     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
804     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
805     if (dim == 3 || !useNumbers) {
806       for (e = eStart; e < eEnd; ++e) {
807         const PetscInt *cone;
808 
809         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
810         color = colors[rank%numColors];
811         for (l = 0; l < numLabels; ++l) {
812           PetscInt val;
813           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
814           if (val >= 0) {color = lcolors[l%numLColors]; break;}
815         }
816         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
817         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
818       }
819     } else {
820       for (c = cStart; c < cEnd; ++c) {
821         PetscInt *closure = NULL;
822         PetscInt  closureSize, firstPoint = -1;
823 
824         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
825         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
826         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
827         for (p = 0; p < closureSize*2; p += 2) {
828           const PetscInt point = closure[p];
829 
830           if ((point < vStart) || (point >= vEnd)) continue;
831           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
832           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
833           if (firstPoint < 0) firstPoint = point;
834         }
835         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
836         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
837         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
838       }
839     }
840     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
841     for (c = cStart; c < cEnd; ++c) {
842       double    ccoords[3] = {0.0, 0.0, 0.0};
843       PetscBool isLabeled  = PETSC_FALSE;
844       PetscInt *closure    = NULL;
845       PetscInt  closureSize, dof, d, n = 0;
846 
847       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
848       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
849       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
850       for (p = 0; p < closureSize*2; p += 2) {
851         const PetscInt point = closure[p];
852         PetscInt       off;
853 
854         if ((point < vStart) || (point >= vEnd)) continue;
855         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
856         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
857         for (d = 0; d < dof; ++d) {
858           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
859           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
860         }
861         /* Rotate coordinates since PGF makes z point out of the page instead of up */
862         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
863         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
864         ++n;
865       }
866       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
867       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
868       for (d = 0; d < dof; ++d) {
869         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
870         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
871       }
872       color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (useNumbers) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
880       } else {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       }
883     }
884     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
885     /* Plot edges */
886     if (plotEdges) {
887       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
888       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
889       for (e = eStart; e < eEnd; ++e) {
890         const PetscInt *cone;
891         PetscInt        coneSize, offA, offB, dof, d;
892 
893         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
894         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
895         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
896         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
897         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
898         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
899         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
900         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
901         for (d = 0; d < dof; ++d) {
902           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
903           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
904         }
905         /* Rotate coordinates since PGF makes z point out of the page instead of up */
906         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
907         for (d = 0; d < dof; ++d) {
908           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
909           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
910         }
911         color = colors[rank%numColors];
912         for (l = 0; l < numLabels; ++l) {
913           PetscInt val;
914           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
915           if (val >= 0) {color = lcolors[l%numLColors]; break;}
916         }
917         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
918       }
919       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
920       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
921       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
922     }
923     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
924     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
925     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
926     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
927     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
928     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
929     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
930     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
931     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
932   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
933     Vec                    cown,acown;
934     VecScatter             sct;
935     ISLocalToGlobalMapping g2l;
936     IS                     gid,acis;
937     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
938     MPI_Group              ggroup,ngroup;
939     PetscScalar            *array,nid;
940     const PetscInt         *idxs;
941     PetscInt               *idxs2,*start,*adjacency,*work;
942     PetscInt64             lm[3],gm[3];
943     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
944     PetscMPIInt            d1,d2,rank;
945 
946     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
947     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
948 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
949     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
950 #endif
951     if (ncomm != MPI_COMM_NULL) {
952       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
953       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
954       d1   = 0;
955       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
956       nid  = d2;
957       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
958       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
959       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
960     } else nid = 0.0;
961 
962     /* Get connectivity */
963     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
964     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
965 
966     /* filter overlapped local cells */
967     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
968     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
969     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
970     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
971     for (c = cStart, cum = 0; c < cEnd; c++) {
972       if (idxs[c-cStart] < 0) continue;
973       idxs2[cum++] = idxs[c-cStart];
974     }
975     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
976     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
977     ierr = ISDestroy(&gid);CHKERRQ(ierr);
978     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
979 
980     /* support for node-aware cell locality */
981     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
982     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
983     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
984     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
985     for (c = 0; c < numVertices; c++) array[c] = nid;
986     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
987     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
988     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
989     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
990     ierr = ISDestroy(&acis);CHKERRQ(ierr);
991     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
992     ierr = VecDestroy(&cown);CHKERRQ(ierr);
993 
994     /* compute edgeCut */
995     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
996     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
997     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
998     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
999     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1000     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1001     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1002       PetscInt totl;
1003 
1004       totl = start[c+1]-start[c];
1005       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1006       for (i = 0; i < totl; i++) {
1007         if (work[i] < 0) {
1008           ect  += 1;
1009           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1010         }
1011       }
1012     }
1013     ierr  = PetscFree(work);CHKERRQ(ierr);
1014     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1015     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1016     lm[1] = -numVertices;
1017     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1018     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1019     lm[0] = ect; /* edgeCut */
1020     lm[1] = ectn; /* node-aware edgeCut */
1021     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1022     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1023     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1024 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1025     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1026 #else
1027     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1028 #endif
1029     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1030     ierr  = PetscFree(start);CHKERRQ(ierr);
1031     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1032     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1033   } else {
1034     MPI_Comm    comm;
1035     PetscInt   *sizes, *hybsizes;
1036     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
1037     PetscInt    pStart, pEnd, p;
1038     PetscInt    numLabels, l;
1039     const char *name;
1040     PetscMPIInt size;
1041 
1042     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1043     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1044     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1045     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1046     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1047     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1048     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1049     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1050     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1051     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1052     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1053     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1054     if (depth == 1) {
1055       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1056       pEnd = pEnd - pStart;
1057       pMax[0] -= pStart;
1058       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1059       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1060       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1061       for (p = 0; p < size; ++p) {
1062         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1063         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1064       }
1065       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1066       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1067       pEnd = pEnd - pStart;
1068       pMax[depth] -= pStart;
1069       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1070       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1071       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1072       for (p = 0; p < size; ++p) {
1073         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1074         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1075       }
1076       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1077     } else {
1078       PetscMPIInt rank;
1079       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1080       for (d = 0; d <= dim; d++) {
1081         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1082         pEnd    -= pStart;
1083         pMax[d] -= pStart;
1084         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1085         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1086         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1087         for (p = 0; p < size; ++p) {
1088           if (!rank) {
1089             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1090             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1091           }
1092         }
1093         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1094       }
1095     }
1096     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1097     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1098     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1099     for (l = 0; l < numLabels; ++l) {
1100       DMLabel         label;
1101       const char     *name;
1102       IS              valueIS;
1103       const PetscInt *values;
1104       PetscInt        numValues, v;
1105 
1106       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1107       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1108       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1109       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1110       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1111       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1112       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1113       for (v = 0; v < numValues; ++v) {
1114         PetscInt size;
1115 
1116         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1117         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1118         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1119       }
1120       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1122       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1123       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1124     }
1125     /* If no fields are specified, people do not want to see adjacency */
1126     if (dm->Nf) {
1127       PetscInt f;
1128 
1129       for (f = 0; f < dm->Nf; ++f) {
1130         const char *name;
1131 
1132         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1133         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1134         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1135         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1136         if (dm->fields[f].adjacency[0]) {
1137           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1138           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1139         } else {
1140           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1141           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1142         }
1143         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1144       }
1145     }
1146     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1147     if (cdm) {
1148       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1149       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1150       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1151     }
1152   }
1153   PetscFunctionReturn(0);
1154 }
1155 
1156 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1157 {
1158   PetscDraw          draw;
1159   DM                 cdm;
1160   PetscSection       coordSection;
1161   Vec                coordinates;
1162   const PetscScalar *coords;
1163   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1164   PetscBool          isnull;
1165   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1166   PetscMPIInt        rank;
1167   PetscErrorCode     ierr;
1168 
1169   PetscFunctionBegin;
1170   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1171   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1172   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1173   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1174   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1175   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1176   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1177 
1178   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1179   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1180   if (isnull) PetscFunctionReturn(0);
1181   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1182 
1183   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1184   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1185   for (c = 0; c < N; c += dim) {
1186     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1187     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1188   }
1189   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1190   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1191   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1192   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1193   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1194 
1195   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1196   for (c = cStart; c < cEnd; ++c) {
1197     PetscScalar *coords = NULL;
1198     PetscInt     numCoords,coneSize;
1199 
1200     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1201     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202     switch (coneSize) {
1203     case 3:
1204       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1205                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1206                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1207                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1208       break;
1209     case 4:
1210       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1211                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1212                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1213                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1214                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1215       break;
1216     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1217     }
1218     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1219   }
1220   for (c = cStart; c < cEnd; ++c) {
1221     PetscScalar *coords = NULL;
1222     PetscInt     numCoords,coneSize;
1223 
1224     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1225     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1226     switch (coneSize) {
1227     case 3:
1228       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1229       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1230       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1231       break;
1232     case 4:
1233       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1234       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1235       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1236       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237       break;
1238     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1239     }
1240     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1241   }
1242   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1243   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1244   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1249 {
1250   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1251   char           name[PETSC_MAX_PATH_LEN];
1252   PetscErrorCode ierr;
1253 
1254   PetscFunctionBegin;
1255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1256   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1257   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1258   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1259   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1260   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1261   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1262   if (iascii) {
1263     PetscViewerFormat format;
1264     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1265     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1266       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1267     } else {
1268       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1269     }
1270   } else if (ishdf5) {
1271 #if defined(PETSC_HAVE_HDF5)
1272     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1273 #else
1274     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1275 #endif
1276   } else if (isvtk) {
1277     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1278   } else if (isdraw) {
1279     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1280   } else if (isglvis) {
1281     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1282   } else {
1283     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1284   }
1285   /* Optionally view the partition */
1286   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1287   if (flg) {
1288     Vec ranks;
1289     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1290     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1291     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1292   }
1293   /* Optionally view a label */
1294   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1295   if (flg) {
1296     DMLabel label;
1297     Vec     val;
1298 
1299     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1300     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1301     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1302     ierr = VecView(val, viewer);CHKERRQ(ierr);
1303     ierr = VecDestroy(&val);CHKERRQ(ierr);
1304   }
1305   PetscFunctionReturn(0);
1306 }
1307 
1308 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1309 {
1310   PetscBool      ishdf5;
1311   PetscErrorCode ierr;
1312 
1313   PetscFunctionBegin;
1314   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1315   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1316   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1317   if (ishdf5) {
1318 #if defined(PETSC_HAVE_HDF5)
1319     PetscViewerFormat format;
1320     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1321     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1322       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1323     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1324       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1325     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1326 #else
1327     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1328 #endif
1329   } else {
1330     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1331   }
1332   PetscFunctionReturn(0);
1333 }
1334 
1335 PetscErrorCode DMDestroy_Plex(DM dm)
1336 {
1337   DM_Plex       *mesh = (DM_Plex*) dm->data;
1338   PetscErrorCode ierr;
1339 
1340   PetscFunctionBegin;
1341   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1342   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1343   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
1344   if (--mesh->refct > 0) PetscFunctionReturn(0);
1345   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1346   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1347   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1348   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1349   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1350   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1351   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1352   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1353   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1354   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1355   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1356   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1357   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1358   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1359   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1360   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1361   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1362   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1363   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1364   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1365   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1366   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1367   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1368   ierr = PetscFree(mesh);CHKERRQ(ierr);
1369   PetscFunctionReturn(0);
1370 }
1371 
1372 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1373 {
1374   PetscSection           sectionGlobal;
1375   PetscInt               bs = -1, mbs;
1376   PetscInt               localSize;
1377   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1378   PetscErrorCode         ierr;
1379   MatType                mtype;
1380   ISLocalToGlobalMapping ltog;
1381 
1382   PetscFunctionBegin;
1383   ierr = MatInitializePackage();CHKERRQ(ierr);
1384   mtype = dm->mattype;
1385   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1386   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1387   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1388   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1389   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1390   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1391   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1392   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1393   if (mbs > 1) bs = mbs;
1394   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1395   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1396   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1397   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1398   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1399   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1400   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1401   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1402   if (!isShell) {
1403     PetscSection subSection;
1404     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1405     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1406     PetscInt     pStart, pEnd, p, dof, cdof;
1407 
1408     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1409     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1410       PetscSection section;
1411       PetscInt     size;
1412 
1413       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1414       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1415       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1416       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1417     } else {
1418       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1419     }
1420     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1421     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1422       PetscInt bdof;
1423 
1424       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1425       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1426       dof  = dof < 0 ? -(dof+1) : dof;
1427       bdof = cdof && (dof-cdof) ? 1 : dof;
1428       if (dof) {
1429         if (bs < 0)          {bs = bdof;}
1430         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1431       }
1432       if (isMatIS) {
1433         PetscInt loff,c,off;
1434         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1435         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1436         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1437       }
1438     }
1439     /* Must have same blocksize on all procs (some might have no points) */
1440     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1441     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1442     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1443     else                            {bs = bsMinMax[0];}
1444     bs = PetscMax(1,bs);
1445     if (isMatIS) { /* Must reduce indices by blocksize */
1446       PetscInt l;
1447 
1448       lsize = lsize/bs;
1449       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1450       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1451     }
1452     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1453     if (isMatIS) {
1454       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1455     }
1456     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1457     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1458     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1459   }
1460   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1461   PetscFunctionReturn(0);
1462 }
1463 
1464 /*@
1465   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1466 
1467   Not collective
1468 
1469   Input Parameter:
1470 . mesh - The DMPlex
1471 
1472   Output Parameters:
1473 . subsection - The subdomain section
1474 
1475   Level: developer
1476 
1477 .seealso:
1478 @*/
1479 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1480 {
1481   DM_Plex       *mesh = (DM_Plex*) dm->data;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   if (!mesh->subdomainSection) {
1487     PetscSection section;
1488     PetscSF      sf;
1489 
1490     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1491     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1492     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1493     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1494   }
1495   *subsection = mesh->subdomainSection;
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 /*@
1500   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1501 
1502   Not collective
1503 
1504   Input Parameter:
1505 . mesh - The DMPlex
1506 
1507   Output Parameters:
1508 + pStart - The first mesh point
1509 - pEnd   - The upper bound for mesh points
1510 
1511   Level: beginner
1512 
1513 .seealso: DMPlexCreate(), DMPlexSetChart()
1514 @*/
1515 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1516 {
1517   DM_Plex       *mesh = (DM_Plex*) dm->data;
1518   PetscErrorCode ierr;
1519 
1520   PetscFunctionBegin;
1521   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1522   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1523   PetscFunctionReturn(0);
1524 }
1525 
1526 /*@
1527   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1528 
1529   Not collective
1530 
1531   Input Parameters:
1532 + mesh - The DMPlex
1533 . pStart - The first mesh point
1534 - pEnd   - The upper bound for mesh points
1535 
1536   Output Parameters:
1537 
1538   Level: beginner
1539 
1540 .seealso: DMPlexCreate(), DMPlexGetChart()
1541 @*/
1542 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1543 {
1544   DM_Plex       *mesh = (DM_Plex*) dm->data;
1545   PetscErrorCode ierr;
1546 
1547   PetscFunctionBegin;
1548   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1549   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1550   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1551   PetscFunctionReturn(0);
1552 }
1553 
1554 /*@
1555   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1556 
1557   Not collective
1558 
1559   Input Parameters:
1560 + mesh - The DMPlex
1561 - p - The point, which must lie in the chart set with DMPlexSetChart()
1562 
1563   Output Parameter:
1564 . size - The cone size for point p
1565 
1566   Level: beginner
1567 
1568 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1569 @*/
1570 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1571 {
1572   DM_Plex       *mesh = (DM_Plex*) dm->data;
1573   PetscErrorCode ierr;
1574 
1575   PetscFunctionBegin;
1576   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1577   PetscValidPointer(size, 3);
1578   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1579   PetscFunctionReturn(0);
1580 }
1581 
1582 /*@
1583   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1584 
1585   Not collective
1586 
1587   Input Parameters:
1588 + mesh - The DMPlex
1589 . p - The point, which must lie in the chart set with DMPlexSetChart()
1590 - size - The cone size for point p
1591 
1592   Output Parameter:
1593 
1594   Note:
1595   This should be called after DMPlexSetChart().
1596 
1597   Level: beginner
1598 
1599 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1600 @*/
1601 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1602 {
1603   DM_Plex       *mesh = (DM_Plex*) dm->data;
1604   PetscErrorCode ierr;
1605 
1606   PetscFunctionBegin;
1607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1608   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1609 
1610   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1611   PetscFunctionReturn(0);
1612 }
1613 
1614 /*@
1615   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1616 
1617   Not collective
1618 
1619   Input Parameters:
1620 + mesh - The DMPlex
1621 . p - The point, which must lie in the chart set with DMPlexSetChart()
1622 - size - The additional cone size for point p
1623 
1624   Output Parameter:
1625 
1626   Note:
1627   This should be called after DMPlexSetChart().
1628 
1629   Level: beginner
1630 
1631 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1632 @*/
1633 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1634 {
1635   DM_Plex       *mesh = (DM_Plex*) dm->data;
1636   PetscInt       csize;
1637   PetscErrorCode ierr;
1638 
1639   PetscFunctionBegin;
1640   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1641   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1642   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1643 
1644   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1645   PetscFunctionReturn(0);
1646 }
1647 
1648 /*@C
1649   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1650 
1651   Not collective
1652 
1653   Input Parameters:
1654 + dm - The DMPlex
1655 - p - The point, which must lie in the chart set with DMPlexSetChart()
1656 
1657   Output Parameter:
1658 . cone - An array of points which are on the in-edges for point p
1659 
1660   Level: beginner
1661 
1662   Fortran Notes:
1663   Since it returns an array, this routine is only available in Fortran 90, and you must
1664   include petsc.h90 in your code.
1665   You must also call DMPlexRestoreCone() after you finish using the returned array.
1666   DMPlexRestoreCone() is not needed/available in C.
1667 
1668 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1669 @*/
1670 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1671 {
1672   DM_Plex       *mesh = (DM_Plex*) dm->data;
1673   PetscInt       off;
1674   PetscErrorCode ierr;
1675 
1676   PetscFunctionBegin;
1677   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1678   PetscValidPointer(cone, 3);
1679   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1680   *cone = &mesh->cones[off];
1681   PetscFunctionReturn(0);
1682 }
1683 
1684 /*@C
1685   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1686 
1687   Not collective
1688 
1689   Input Parameters:
1690 + dm - The DMPlex
1691 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1692 
1693   Output Parameter:
1694 + pConesSection - PetscSection describing the layout of pCones
1695 - pCones - An array of points which are on the in-edges for the point set p
1696 
1697   Level: intermediate
1698 
1699 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1700 @*/
1701 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1702 {
1703   PetscSection        cs, newcs;
1704   PetscInt            *cones;
1705   PetscInt            *newarr=NULL;
1706   PetscInt            n;
1707   PetscErrorCode      ierr;
1708 
1709   PetscFunctionBegin;
1710   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1711   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1712   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1713   if (pConesSection) *pConesSection = newcs;
1714   if (pCones) {
1715     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1716     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1717   }
1718   PetscFunctionReturn(0);
1719 }
1720 
1721 /*@
1722   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1723 
1724   Not collective
1725 
1726   Input Parameters:
1727 + dm - The DMPlex
1728 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1729 
1730   Output Parameter:
1731 . expandedPoints - An array of vertices recursively expanded from input points
1732 
1733   Level: advanced
1734 
1735   Notes:
1736   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1737   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1738 
1739 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1740 @*/
1741 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1742 {
1743   IS                  *expandedPointsAll;
1744   PetscInt            depth;
1745   PetscErrorCode      ierr;
1746 
1747   PetscFunctionBegin;
1748   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1749   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1750   PetscValidPointer(expandedPoints, 3);
1751   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1752   *expandedPoints = expandedPointsAll[0];
1753   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1754   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1755   PetscFunctionReturn(0);
1756 }
1757 
1758 /*@
1759   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1760 
1761   Not collective
1762 
1763   Input Parameters:
1764 + dm - The DMPlex
1765 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1766 
1767   Output Parameter:
1768 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1769 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1770 - sections - (optional) An array of sections which describe mappings from points to their cone points
1771 
1772   Level: advanced
1773 
1774   Notes:
1775   Like DMPlexGetConeTuple() but recursive.
1776 
1777   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1778   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1779 
1780   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1781   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1782   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1783 
1784 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1785 @*/
1786 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1787 {
1788   const PetscInt      *arr0=NULL, *cone=NULL;
1789   PetscInt            *arr=NULL, *newarr=NULL;
1790   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1791   IS                  *expandedPoints_;
1792   PetscSection        *sections_;
1793   PetscErrorCode      ierr;
1794 
1795   PetscFunctionBegin;
1796   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1797   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1798   if (depth) PetscValidIntPointer(depth, 3);
1799   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1800   if (sections) PetscValidPointer(sections, 5);
1801   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1802   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1803   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1804   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1805   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1806   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1807   for (d=depth_-1; d>=0; d--) {
1808     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1809     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1810     for (i=0; i<n; i++) {
1811       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1812       if (arr[i] >= start && arr[i] < end) {
1813         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1814         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1815       } else {
1816         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1817       }
1818     }
1819     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1820     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1821     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1822     for (i=0; i<n; i++) {
1823       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1824       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1825       if (cn > 1) {
1826         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1827         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1828       } else {
1829         newarr[co] = arr[i];
1830       }
1831     }
1832     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1833     arr = newarr;
1834     n = newn;
1835   }
1836   *depth = depth_;
1837   if (expandedPoints) *expandedPoints = expandedPoints_;
1838   else {
1839     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1840     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1841   }
1842   if (sections) *sections = sections_;
1843   else {
1844     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1845     ierr = PetscFree(sections_);CHKERRQ(ierr);
1846   }
1847   PetscFunctionReturn(0);
1848 }
1849 
1850 /*@
1851   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1852 
1853   Not collective
1854 
1855   Input Parameters:
1856 + dm - The DMPlex
1857 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1858 
1859   Output Parameter:
1860 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1861 . expandedPoints - (optional) An array of recursively expanded cones
1862 - sections - (optional) An array of sections which describe mappings from points to their cone points
1863 
1864   Level: advanced
1865 
1866   Notes:
1867   See DMPlexGetConeRecursive() for details.
1868 
1869 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1870 @*/
1871 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1872 {
1873   PetscInt            d, depth_;
1874   PetscErrorCode      ierr;
1875 
1876   PetscFunctionBegin;
1877   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1878   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1879   if (depth) *depth = 0;
1880   if (expandedPoints) {
1881     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1882     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1883   }
1884   if (sections)  {
1885     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1886     ierr = PetscFree(*sections);CHKERRQ(ierr);
1887   }
1888   PetscFunctionReturn(0);
1889 }
1890 
1891 /*@
1892   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1893 
1894   Not collective
1895 
1896   Input Parameters:
1897 + mesh - The DMPlex
1898 . p - The point, which must lie in the chart set with DMPlexSetChart()
1899 - cone - An array of points which are on the in-edges for point p
1900 
1901   Output Parameter:
1902 
1903   Note:
1904   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1905 
1906   Developer Note: Why not call this DMPlexSetCover()
1907 
1908   Level: beginner
1909 
1910 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1911 @*/
1912 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1913 {
1914   DM_Plex       *mesh = (DM_Plex*) dm->data;
1915   PetscInt       pStart, pEnd;
1916   PetscInt       dof, off, c;
1917   PetscErrorCode ierr;
1918 
1919   PetscFunctionBegin;
1920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1921   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1922   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1923   if (dof) PetscValidPointer(cone, 3);
1924   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1925   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1926   for (c = 0; c < dof; ++c) {
1927     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1928     mesh->cones[off+c] = cone[c];
1929   }
1930   PetscFunctionReturn(0);
1931 }
1932 
1933 /*@C
1934   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1935 
1936   Not collective
1937 
1938   Input Parameters:
1939 + mesh - The DMPlex
1940 - p - The point, which must lie in the chart set with DMPlexSetChart()
1941 
1942   Output Parameter:
1943 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1944                     integer giving the prescription for cone traversal. If it is negative, the cone is
1945                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1946                     the index of the cone point on which to start.
1947 
1948   Level: beginner
1949 
1950   Fortran Notes:
1951   Since it returns an array, this routine is only available in Fortran 90, and you must
1952   include petsc.h90 in your code.
1953   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1954   DMPlexRestoreConeOrientation() is not needed/available in C.
1955 
1956 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1957 @*/
1958 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1959 {
1960   DM_Plex       *mesh = (DM_Plex*) dm->data;
1961   PetscInt       off;
1962   PetscErrorCode ierr;
1963 
1964   PetscFunctionBegin;
1965   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1966 #if defined(PETSC_USE_DEBUG)
1967   {
1968     PetscInt dof;
1969     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1970     if (dof) PetscValidPointer(coneOrientation, 3);
1971   }
1972 #endif
1973   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1974 
1975   *coneOrientation = &mesh->coneOrientations[off];
1976   PetscFunctionReturn(0);
1977 }
1978 
1979 /*@
1980   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1981 
1982   Not collective
1983 
1984   Input Parameters:
1985 + mesh - The DMPlex
1986 . p - The point, which must lie in the chart set with DMPlexSetChart()
1987 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1988                     integer giving the prescription for cone traversal. If it is negative, the cone is
1989                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1990                     the index of the cone point on which to start.
1991 
1992   Output Parameter:
1993 
1994   Note:
1995   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1996 
1997   Level: beginner
1998 
1999 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2000 @*/
2001 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2002 {
2003   DM_Plex       *mesh = (DM_Plex*) dm->data;
2004   PetscInt       pStart, pEnd;
2005   PetscInt       dof, off, c;
2006   PetscErrorCode ierr;
2007 
2008   PetscFunctionBegin;
2009   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2010   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2011   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2012   if (dof) PetscValidPointer(coneOrientation, 3);
2013   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2014   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2015   for (c = 0; c < dof; ++c) {
2016     PetscInt cdof, o = coneOrientation[c];
2017 
2018     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2019     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2020     mesh->coneOrientations[off+c] = o;
2021   }
2022   PetscFunctionReturn(0);
2023 }
2024 
2025 /*@
2026   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2027 
2028   Not collective
2029 
2030   Input Parameters:
2031 + mesh - The DMPlex
2032 . p - The point, which must lie in the chart set with DMPlexSetChart()
2033 . conePos - The local index in the cone where the point should be put
2034 - conePoint - The mesh point to insert
2035 
2036   Level: beginner
2037 
2038 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2039 @*/
2040 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2041 {
2042   DM_Plex       *mesh = (DM_Plex*) dm->data;
2043   PetscInt       pStart, pEnd;
2044   PetscInt       dof, off;
2045   PetscErrorCode ierr;
2046 
2047   PetscFunctionBegin;
2048   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2049   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2050   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2051   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2052   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2053   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2054   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2055   mesh->cones[off+conePos] = conePoint;
2056   PetscFunctionReturn(0);
2057 }
2058 
2059 /*@
2060   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2061 
2062   Not collective
2063 
2064   Input Parameters:
2065 + mesh - The DMPlex
2066 . p - The point, which must lie in the chart set with DMPlexSetChart()
2067 . conePos - The local index in the cone where the point should be put
2068 - coneOrientation - The point orientation to insert
2069 
2070   Level: beginner
2071 
2072 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2073 @*/
2074 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2075 {
2076   DM_Plex       *mesh = (DM_Plex*) dm->data;
2077   PetscInt       pStart, pEnd;
2078   PetscInt       dof, off;
2079   PetscErrorCode ierr;
2080 
2081   PetscFunctionBegin;
2082   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2083   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2084   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2085   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2086   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2087   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2088   mesh->coneOrientations[off+conePos] = coneOrientation;
2089   PetscFunctionReturn(0);
2090 }
2091 
2092 /*@
2093   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2094 
2095   Not collective
2096 
2097   Input Parameters:
2098 + mesh - The DMPlex
2099 - p - The point, which must lie in the chart set with DMPlexSetChart()
2100 
2101   Output Parameter:
2102 . size - The support size for point p
2103 
2104   Level: beginner
2105 
2106 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2107 @*/
2108 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2109 {
2110   DM_Plex       *mesh = (DM_Plex*) dm->data;
2111   PetscErrorCode ierr;
2112 
2113   PetscFunctionBegin;
2114   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2115   PetscValidPointer(size, 3);
2116   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2117   PetscFunctionReturn(0);
2118 }
2119 
2120 /*@
2121   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2122 
2123   Not collective
2124 
2125   Input Parameters:
2126 + mesh - The DMPlex
2127 . p - The point, which must lie in the chart set with DMPlexSetChart()
2128 - size - The support size for point p
2129 
2130   Output Parameter:
2131 
2132   Note:
2133   This should be called after DMPlexSetChart().
2134 
2135   Level: beginner
2136 
2137 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2138 @*/
2139 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2140 {
2141   DM_Plex       *mesh = (DM_Plex*) dm->data;
2142   PetscErrorCode ierr;
2143 
2144   PetscFunctionBegin;
2145   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2146   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2147 
2148   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2149   PetscFunctionReturn(0);
2150 }
2151 
2152 /*@C
2153   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2154 
2155   Not collective
2156 
2157   Input Parameters:
2158 + mesh - The DMPlex
2159 - p - The point, which must lie in the chart set with DMPlexSetChart()
2160 
2161   Output Parameter:
2162 . support - An array of points which are on the out-edges for point p
2163 
2164   Level: beginner
2165 
2166   Fortran Notes:
2167   Since it returns an array, this routine is only available in Fortran 90, and you must
2168   include petsc.h90 in your code.
2169   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2170   DMPlexRestoreSupport() is not needed/available in C.
2171 
2172 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2173 @*/
2174 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2175 {
2176   DM_Plex       *mesh = (DM_Plex*) dm->data;
2177   PetscInt       off;
2178   PetscErrorCode ierr;
2179 
2180   PetscFunctionBegin;
2181   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2182   PetscValidPointer(support, 3);
2183   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2184   *support = &mesh->supports[off];
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 /*@
2189   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2190 
2191   Not collective
2192 
2193   Input Parameters:
2194 + mesh - The DMPlex
2195 . p - The point, which must lie in the chart set with DMPlexSetChart()
2196 - support - An array of points which are on the out-edges for point p
2197 
2198   Output Parameter:
2199 
2200   Note:
2201   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2202 
2203   Level: beginner
2204 
2205 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2206 @*/
2207 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2208 {
2209   DM_Plex       *mesh = (DM_Plex*) dm->data;
2210   PetscInt       pStart, pEnd;
2211   PetscInt       dof, off, c;
2212   PetscErrorCode ierr;
2213 
2214   PetscFunctionBegin;
2215   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2216   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2217   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2218   if (dof) PetscValidPointer(support, 3);
2219   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2220   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2221   for (c = 0; c < dof; ++c) {
2222     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2223     mesh->supports[off+c] = support[c];
2224   }
2225   PetscFunctionReturn(0);
2226 }
2227 
2228 /*@
2229   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2230 
2231   Not collective
2232 
2233   Input Parameters:
2234 + mesh - The DMPlex
2235 . p - The point, which must lie in the chart set with DMPlexSetChart()
2236 . supportPos - The local index in the cone where the point should be put
2237 - supportPoint - The mesh point to insert
2238 
2239   Level: beginner
2240 
2241 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2242 @*/
2243 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2244 {
2245   DM_Plex       *mesh = (DM_Plex*) dm->data;
2246   PetscInt       pStart, pEnd;
2247   PetscInt       dof, off;
2248   PetscErrorCode ierr;
2249 
2250   PetscFunctionBegin;
2251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2252   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2253   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2254   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2255   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2256   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2257   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2258   mesh->supports[off+supportPos] = supportPoint;
2259   PetscFunctionReturn(0);
2260 }
2261 
2262 /*@C
2263   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2264 
2265   Not collective
2266 
2267   Input Parameters:
2268 + mesh - The DMPlex
2269 . p - The point, which must lie in the chart set with DMPlexSetChart()
2270 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2271 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2272 
2273   Output Parameters:
2274 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2275 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2276 
2277   Note:
2278   If using internal storage (points is NULL on input), each call overwrites the last output.
2279 
2280   Fortran Notes:
2281   Since it returns an array, this routine is only available in Fortran 90, and you must
2282   include petsc.h90 in your code.
2283 
2284   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2285 
2286   Level: beginner
2287 
2288 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2289 @*/
2290 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2291 {
2292   DM_Plex        *mesh = (DM_Plex*) dm->data;
2293   PetscInt       *closure, *fifo;
2294   const PetscInt *tmp = NULL, *tmpO = NULL;
2295   PetscInt        tmpSize, t;
2296   PetscInt        depth       = 0, maxSize;
2297   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2298   PetscErrorCode  ierr;
2299 
2300   PetscFunctionBegin;
2301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2302   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2303   /* This is only 1-level */
2304   if (useCone) {
2305     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2306     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2307     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2308   } else {
2309     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2310     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2311   }
2312   if (depth == 1) {
2313     if (*points) {
2314       closure = *points;
2315     } else {
2316       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2317       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2318     }
2319     closure[0] = p; closure[1] = 0;
2320     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2321       closure[closureSize]   = tmp[t];
2322       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2323     }
2324     if (numPoints) *numPoints = closureSize/2;
2325     if (points)    *points    = closure;
2326     PetscFunctionReturn(0);
2327   }
2328   {
2329     PetscInt c, coneSeries, s,supportSeries;
2330 
2331     c = mesh->maxConeSize;
2332     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2333     s = mesh->maxSupportSize;
2334     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2335     maxSize = 2*PetscMax(coneSeries,supportSeries);
2336   }
2337   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2338   if (*points) {
2339     closure = *points;
2340   } else {
2341     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2342   }
2343   closure[0] = p; closure[1] = 0;
2344   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2345     const PetscInt cp = tmp[t];
2346     const PetscInt co = tmpO ? tmpO[t] : 0;
2347 
2348     closure[closureSize]   = cp;
2349     closure[closureSize+1] = co;
2350     fifo[fifoSize]         = cp;
2351     fifo[fifoSize+1]       = co;
2352   }
2353   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2354   while (fifoSize - fifoStart) {
2355     const PetscInt q   = fifo[fifoStart];
2356     const PetscInt o   = fifo[fifoStart+1];
2357     const PetscInt rev = o >= 0 ? 0 : 1;
2358     const PetscInt off = rev ? -(o+1) : o;
2359 
2360     if (useCone) {
2361       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2362       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2363       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2364     } else {
2365       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2366       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2367       tmpO = NULL;
2368     }
2369     for (t = 0; t < tmpSize; ++t) {
2370       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2371       const PetscInt cp = tmp[i];
2372       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2373       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2374        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2375       PetscInt       co = tmpO ? tmpO[i] : 0;
2376       PetscInt       c;
2377 
2378       if (rev) {
2379         PetscInt childSize, coff;
2380         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2381         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2382         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2383       }
2384       /* Check for duplicate */
2385       for (c = 0; c < closureSize; c += 2) {
2386         if (closure[c] == cp) break;
2387       }
2388       if (c == closureSize) {
2389         closure[closureSize]   = cp;
2390         closure[closureSize+1] = co;
2391         fifo[fifoSize]         = cp;
2392         fifo[fifoSize+1]       = co;
2393         closureSize           += 2;
2394         fifoSize              += 2;
2395       }
2396     }
2397     fifoStart += 2;
2398   }
2399   if (numPoints) *numPoints = closureSize/2;
2400   if (points)    *points    = closure;
2401   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2402   PetscFunctionReturn(0);
2403 }
2404 
2405 /*@C
2406   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2407 
2408   Not collective
2409 
2410   Input Parameters:
2411 + mesh - The DMPlex
2412 . p - The point, which must lie in the chart set with DMPlexSetChart()
2413 . orientation - The orientation of the point
2414 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2415 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2416 
2417   Output Parameters:
2418 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2419 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2420 
2421   Note:
2422   If using internal storage (points is NULL on input), each call overwrites the last output.
2423 
2424   Fortran Notes:
2425   Since it returns an array, this routine is only available in Fortran 90, and you must
2426   include petsc.h90 in your code.
2427 
2428   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2429 
2430   Level: beginner
2431 
2432 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2433 @*/
2434 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2435 {
2436   DM_Plex        *mesh = (DM_Plex*) dm->data;
2437   PetscInt       *closure, *fifo;
2438   const PetscInt *tmp = NULL, *tmpO = NULL;
2439   PetscInt        tmpSize, t;
2440   PetscInt        depth       = 0, maxSize;
2441   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2442   PetscErrorCode  ierr;
2443 
2444   PetscFunctionBegin;
2445   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2446   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2447   /* This is only 1-level */
2448   if (useCone) {
2449     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2450     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2451     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2452   } else {
2453     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2454     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2455   }
2456   if (depth == 1) {
2457     if (*points) {
2458       closure = *points;
2459     } else {
2460       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2461       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2462     }
2463     closure[0] = p; closure[1] = ornt;
2464     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2465       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2466       closure[closureSize]   = tmp[i];
2467       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2468     }
2469     if (numPoints) *numPoints = closureSize/2;
2470     if (points)    *points    = closure;
2471     PetscFunctionReturn(0);
2472   }
2473   {
2474     PetscInt c, coneSeries, s,supportSeries;
2475 
2476     c = mesh->maxConeSize;
2477     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2478     s = mesh->maxSupportSize;
2479     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2480     maxSize = 2*PetscMax(coneSeries,supportSeries);
2481   }
2482   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2483   if (*points) {
2484     closure = *points;
2485   } else {
2486     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2487   }
2488   closure[0] = p; closure[1] = ornt;
2489   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2490     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2491     const PetscInt cp = tmp[i];
2492     PetscInt       co = tmpO ? tmpO[i] : 0;
2493 
2494     if (ornt < 0) {
2495       PetscInt childSize, coff;
2496       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2497       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2498       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2499     }
2500     closure[closureSize]   = cp;
2501     closure[closureSize+1] = co;
2502     fifo[fifoSize]         = cp;
2503     fifo[fifoSize+1]       = co;
2504   }
2505   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2506   while (fifoSize - fifoStart) {
2507     const PetscInt q   = fifo[fifoStart];
2508     const PetscInt o   = fifo[fifoStart+1];
2509     const PetscInt rev = o >= 0 ? 0 : 1;
2510     const PetscInt off = rev ? -(o+1) : o;
2511 
2512     if (useCone) {
2513       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2514       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2515       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2516     } else {
2517       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2518       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2519       tmpO = NULL;
2520     }
2521     for (t = 0; t < tmpSize; ++t) {
2522       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2523       const PetscInt cp = tmp[i];
2524       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2525       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2526        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2527       PetscInt       co = tmpO ? tmpO[i] : 0;
2528       PetscInt       c;
2529 
2530       if (rev) {
2531         PetscInt childSize, coff;
2532         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2533         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2534         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2535       }
2536       /* Check for duplicate */
2537       for (c = 0; c < closureSize; c += 2) {
2538         if (closure[c] == cp) break;
2539       }
2540       if (c == closureSize) {
2541         closure[closureSize]   = cp;
2542         closure[closureSize+1] = co;
2543         fifo[fifoSize]         = cp;
2544         fifo[fifoSize+1]       = co;
2545         closureSize           += 2;
2546         fifoSize              += 2;
2547       }
2548     }
2549     fifoStart += 2;
2550   }
2551   if (numPoints) *numPoints = closureSize/2;
2552   if (points)    *points    = closure;
2553   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2554   PetscFunctionReturn(0);
2555 }
2556 
2557 /*@C
2558   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2559 
2560   Not collective
2561 
2562   Input Parameters:
2563 + mesh - The DMPlex
2564 . p - The point, which must lie in the chart set with DMPlexSetChart()
2565 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2566 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2567 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2568 
2569   Note:
2570   If not using internal storage (points is not NULL on input), this call is unnecessary
2571 
2572   Fortran Notes:
2573   Since it returns an array, this routine is only available in Fortran 90, and you must
2574   include petsc.h90 in your code.
2575 
2576   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2577 
2578   Level: beginner
2579 
2580 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2581 @*/
2582 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2583 {
2584   PetscErrorCode ierr;
2585 
2586   PetscFunctionBegin;
2587   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2588   if (numPoints) PetscValidIntPointer(numPoints,4);
2589   if (points) PetscValidPointer(points,5);
2590   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2591   if (numPoints) *numPoints = 0;
2592   PetscFunctionReturn(0);
2593 }
2594 
2595 /*@
2596   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2597 
2598   Not collective
2599 
2600   Input Parameter:
2601 . mesh - The DMPlex
2602 
2603   Output Parameters:
2604 + maxConeSize - The maximum number of in-edges
2605 - maxSupportSize - The maximum number of out-edges
2606 
2607   Level: beginner
2608 
2609 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2610 @*/
2611 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2612 {
2613   DM_Plex *mesh = (DM_Plex*) dm->data;
2614 
2615   PetscFunctionBegin;
2616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2617   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2618   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2619   PetscFunctionReturn(0);
2620 }
2621 
2622 PetscErrorCode DMSetUp_Plex(DM dm)
2623 {
2624   DM_Plex       *mesh = (DM_Plex*) dm->data;
2625   PetscInt       size;
2626   PetscErrorCode ierr;
2627 
2628   PetscFunctionBegin;
2629   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2630   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2631   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2632   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2633   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2634   if (mesh->maxSupportSize) {
2635     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2636     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2637     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2638   }
2639   PetscFunctionReturn(0);
2640 }
2641 
2642 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2643 {
2644   PetscErrorCode ierr;
2645 
2646   PetscFunctionBegin;
2647   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2648   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2649   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2650   if (dm->useNatural && dm->sfMigration) {
2651     PetscSF        sfMigrationInv,sfNatural;
2652     PetscSection   section, sectionSeq;
2653 
2654     (*subdm)->sfMigration = dm->sfMigration;
2655     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2656     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2657     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2658     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2659     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2660 
2661     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2662     (*subdm)->sfNatural = sfNatural;
2663     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2664     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2665   }
2666   PetscFunctionReturn(0);
2667 }
2668 
2669 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2670 {
2671   PetscErrorCode ierr;
2672   PetscInt       i = 0;
2673 
2674   PetscFunctionBegin;
2675   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2676   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2677   (*superdm)->useNatural = PETSC_FALSE;
2678   for (i = 0; i < len; i++){
2679     if (dms[i]->useNatural && dms[i]->sfMigration) {
2680       PetscSF        sfMigrationInv,sfNatural;
2681       PetscSection   section, sectionSeq;
2682 
2683       (*superdm)->sfMigration = dms[i]->sfMigration;
2684       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2685       (*superdm)->useNatural = PETSC_TRUE;
2686       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2687       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2688       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2689       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2690 
2691       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2692       (*superdm)->sfNatural = sfNatural;
2693       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2694       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2695       break;
2696     }
2697   }
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 /*@
2702   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2703 
2704   Not collective
2705 
2706   Input Parameter:
2707 . mesh - The DMPlex
2708 
2709   Output Parameter:
2710 
2711   Note:
2712   This should be called after all calls to DMPlexSetCone()
2713 
2714   Level: beginner
2715 
2716 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2717 @*/
2718 PetscErrorCode DMPlexSymmetrize(DM dm)
2719 {
2720   DM_Plex       *mesh = (DM_Plex*) dm->data;
2721   PetscInt      *offsets;
2722   PetscInt       supportSize;
2723   PetscInt       pStart, pEnd, p;
2724   PetscErrorCode ierr;
2725 
2726   PetscFunctionBegin;
2727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2728   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2729   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2730   /* Calculate support sizes */
2731   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2732   for (p = pStart; p < pEnd; ++p) {
2733     PetscInt dof, off, c;
2734 
2735     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2736     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2737     for (c = off; c < off+dof; ++c) {
2738       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2739     }
2740   }
2741   for (p = pStart; p < pEnd; ++p) {
2742     PetscInt dof;
2743 
2744     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2745 
2746     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2747   }
2748   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2749   /* Calculate supports */
2750   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2751   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2752   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2753   for (p = pStart; p < pEnd; ++p) {
2754     PetscInt dof, off, c;
2755 
2756     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2757     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2758     for (c = off; c < off+dof; ++c) {
2759       const PetscInt q = mesh->cones[c];
2760       PetscInt       offS;
2761 
2762       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2763 
2764       mesh->supports[offS+offsets[q]] = p;
2765       ++offsets[q];
2766     }
2767   }
2768   ierr = PetscFree(offsets);CHKERRQ(ierr);
2769   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2770   PetscFunctionReturn(0);
2771 }
2772 
2773 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2774 {
2775   IS             stratumIS;
2776   PetscErrorCode ierr;
2777 
2778   PetscFunctionBegin;
2779   if (pStart >= pEnd) PetscFunctionReturn(0);
2780 #if defined(PETSC_USE_DEBUG)
2781   {
2782     PetscInt  qStart, qEnd, numLevels, level;
2783     PetscBool overlap = PETSC_FALSE;
2784     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2785     for (level = 0; level < numLevels; level++) {
2786       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2787       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2788     }
2789     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2790   }
2791 #endif
2792   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2793   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2794   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2795   PetscFunctionReturn(0);
2796 }
2797 
2798 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2799 
2800 /*@
2801   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2802   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2803   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2804   the DAG.
2805 
2806   Collective on dm
2807 
2808   Input Parameter:
2809 . mesh - The DMPlex
2810 
2811   Output Parameter:
2812 
2813   Notes:
2814   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2815   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2816   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2817   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2818 
2819   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2820 
2821   Level: beginner
2822 
2823 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2824 @*/
2825 PetscErrorCode DMPlexStratify(DM dm)
2826 {
2827   DM_Plex       *mesh = (DM_Plex*) dm->data;
2828   DMLabel        label;
2829   PetscInt       pStart, pEnd, p;
2830   PetscInt       numRoots = 0, numLeaves = 0;
2831   PetscInt       cMax, fMax, eMax, vMax;
2832   PetscErrorCode ierr;
2833 
2834   PetscFunctionBegin;
2835   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2836   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2837 
2838   /* Create depth label */
2839   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2840   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2841   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2842 
2843   {
2844     /* Initialize roots and count leaves */
2845     PetscInt sMin = PETSC_MAX_INT;
2846     PetscInt sMax = PETSC_MIN_INT;
2847     PetscInt coneSize, supportSize;
2848 
2849     for (p = pStart; p < pEnd; ++p) {
2850       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2851       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2852       if (!coneSize && supportSize) {
2853         sMin = PetscMin(p, sMin);
2854         sMax = PetscMax(p, sMax);
2855         ++numRoots;
2856       } else if (!supportSize && coneSize) {
2857         ++numLeaves;
2858       } else if (!supportSize && !coneSize) {
2859         /* Isolated points */
2860         sMin = PetscMin(p, sMin);
2861         sMax = PetscMax(p, sMax);
2862       }
2863     }
2864     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2865   }
2866 
2867   if (numRoots + numLeaves == (pEnd - pStart)) {
2868     PetscInt sMin = PETSC_MAX_INT;
2869     PetscInt sMax = PETSC_MIN_INT;
2870     PetscInt coneSize, supportSize;
2871 
2872     for (p = pStart; p < pEnd; ++p) {
2873       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2874       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2875       if (!supportSize && coneSize) {
2876         sMin = PetscMin(p, sMin);
2877         sMax = PetscMax(p, sMax);
2878       }
2879     }
2880     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2881   } else {
2882     PetscInt level = 0;
2883     PetscInt qStart, qEnd, q;
2884 
2885     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2886     while (qEnd > qStart) {
2887       PetscInt sMin = PETSC_MAX_INT;
2888       PetscInt sMax = PETSC_MIN_INT;
2889 
2890       for (q = qStart; q < qEnd; ++q) {
2891         const PetscInt *support;
2892         PetscInt        supportSize, s;
2893 
2894         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2895         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2896         for (s = 0; s < supportSize; ++s) {
2897           sMin = PetscMin(support[s], sMin);
2898           sMax = PetscMax(support[s], sMax);
2899         }
2900       }
2901       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2902       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2903       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2904     }
2905   }
2906   { /* just in case there is an empty process */
2907     PetscInt numValues, maxValues = 0, v;
2908 
2909     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2910     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2911     for (v = numValues; v < maxValues; v++) {
2912       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2913     }
2914   }
2915   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2916 
2917   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2918   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2919     PetscInt dim;
2920     DMLabel  dimLabel;
2921 
2922     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2923     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2924     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2925     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2926     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2927     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2928     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2929   }
2930   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2931   PetscFunctionReturn(0);
2932 }
2933 
2934 /*@C
2935   DMPlexGetJoin - Get an array for the join of the set of points
2936 
2937   Not Collective
2938 
2939   Input Parameters:
2940 + dm - The DMPlex object
2941 . numPoints - The number of input points for the join
2942 - points - The input points
2943 
2944   Output Parameters:
2945 + numCoveredPoints - The number of points in the join
2946 - coveredPoints - The points in the join
2947 
2948   Level: intermediate
2949 
2950   Note: Currently, this is restricted to a single level join
2951 
2952   Fortran Notes:
2953   Since it returns an array, this routine is only available in Fortran 90, and you must
2954   include petsc.h90 in your code.
2955 
2956   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2957 
2958 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2959 @*/
2960 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2961 {
2962   DM_Plex       *mesh = (DM_Plex*) dm->data;
2963   PetscInt      *join[2];
2964   PetscInt       joinSize, i = 0;
2965   PetscInt       dof, off, p, c, m;
2966   PetscErrorCode ierr;
2967 
2968   PetscFunctionBegin;
2969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2970   PetscValidIntPointer(points, 3);
2971   PetscValidIntPointer(numCoveredPoints, 4);
2972   PetscValidPointer(coveredPoints, 5);
2973   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2974   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2975   /* Copy in support of first point */
2976   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2977   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2978   for (joinSize = 0; joinSize < dof; ++joinSize) {
2979     join[i][joinSize] = mesh->supports[off+joinSize];
2980   }
2981   /* Check each successive support */
2982   for (p = 1; p < numPoints; ++p) {
2983     PetscInt newJoinSize = 0;
2984 
2985     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2986     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2987     for (c = 0; c < dof; ++c) {
2988       const PetscInt point = mesh->supports[off+c];
2989 
2990       for (m = 0; m < joinSize; ++m) {
2991         if (point == join[i][m]) {
2992           join[1-i][newJoinSize++] = point;
2993           break;
2994         }
2995       }
2996     }
2997     joinSize = newJoinSize;
2998     i        = 1-i;
2999   }
3000   *numCoveredPoints = joinSize;
3001   *coveredPoints    = join[i];
3002   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3003   PetscFunctionReturn(0);
3004 }
3005 
3006 /*@C
3007   DMPlexRestoreJoin - Restore an array for the join of the set of points
3008 
3009   Not Collective
3010 
3011   Input Parameters:
3012 + dm - The DMPlex object
3013 . numPoints - The number of input points for the join
3014 - points - The input points
3015 
3016   Output Parameters:
3017 + numCoveredPoints - The number of points in the join
3018 - coveredPoints - The points in the join
3019 
3020   Fortran Notes:
3021   Since it returns an array, this routine is only available in Fortran 90, and you must
3022   include petsc.h90 in your code.
3023 
3024   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3025 
3026   Level: intermediate
3027 
3028 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3029 @*/
3030 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3031 {
3032   PetscErrorCode ierr;
3033 
3034   PetscFunctionBegin;
3035   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3036   if (points) PetscValidIntPointer(points,3);
3037   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3038   PetscValidPointer(coveredPoints, 5);
3039   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3040   if (numCoveredPoints) *numCoveredPoints = 0;
3041   PetscFunctionReturn(0);
3042 }
3043 
3044 /*@C
3045   DMPlexGetFullJoin - Get an array for the join of the set of points
3046 
3047   Not Collective
3048 
3049   Input Parameters:
3050 + dm - The DMPlex object
3051 . numPoints - The number of input points for the join
3052 - points - The input points
3053 
3054   Output Parameters:
3055 + numCoveredPoints - The number of points in the join
3056 - coveredPoints - The points in the join
3057 
3058   Fortran Notes:
3059   Since it returns an array, this routine is only available in Fortran 90, and you must
3060   include petsc.h90 in your code.
3061 
3062   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3063 
3064   Level: intermediate
3065 
3066 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3067 @*/
3068 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3069 {
3070   DM_Plex       *mesh = (DM_Plex*) dm->data;
3071   PetscInt      *offsets, **closures;
3072   PetscInt      *join[2];
3073   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3074   PetscInt       p, d, c, m, ms;
3075   PetscErrorCode ierr;
3076 
3077   PetscFunctionBegin;
3078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3079   PetscValidIntPointer(points, 3);
3080   PetscValidIntPointer(numCoveredPoints, 4);
3081   PetscValidPointer(coveredPoints, 5);
3082 
3083   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3084   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3085   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3086   ms      = mesh->maxSupportSize;
3087   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3088   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3089   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3090 
3091   for (p = 0; p < numPoints; ++p) {
3092     PetscInt closureSize;
3093 
3094     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3095 
3096     offsets[p*(depth+2)+0] = 0;
3097     for (d = 0; d < depth+1; ++d) {
3098       PetscInt pStart, pEnd, i;
3099 
3100       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3101       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3102         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3103           offsets[p*(depth+2)+d+1] = i;
3104           break;
3105         }
3106       }
3107       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3108     }
3109     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3110   }
3111   for (d = 0; d < depth+1; ++d) {
3112     PetscInt dof;
3113 
3114     /* Copy in support of first point */
3115     dof = offsets[d+1] - offsets[d];
3116     for (joinSize = 0; joinSize < dof; ++joinSize) {
3117       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3118     }
3119     /* Check each successive cone */
3120     for (p = 1; p < numPoints && joinSize; ++p) {
3121       PetscInt newJoinSize = 0;
3122 
3123       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3124       for (c = 0; c < dof; ++c) {
3125         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3126 
3127         for (m = 0; m < joinSize; ++m) {
3128           if (point == join[i][m]) {
3129             join[1-i][newJoinSize++] = point;
3130             break;
3131           }
3132         }
3133       }
3134       joinSize = newJoinSize;
3135       i        = 1-i;
3136     }
3137     if (joinSize) break;
3138   }
3139   *numCoveredPoints = joinSize;
3140   *coveredPoints    = join[i];
3141   for (p = 0; p < numPoints; ++p) {
3142     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3143   }
3144   ierr = PetscFree(closures);CHKERRQ(ierr);
3145   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3146   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3147   PetscFunctionReturn(0);
3148 }
3149 
3150 /*@C
3151   DMPlexGetMeet - Get an array for the meet of the set of points
3152 
3153   Not Collective
3154 
3155   Input Parameters:
3156 + dm - The DMPlex object
3157 . numPoints - The number of input points for the meet
3158 - points - The input points
3159 
3160   Output Parameters:
3161 + numCoveredPoints - The number of points in the meet
3162 - coveredPoints - The points in the meet
3163 
3164   Level: intermediate
3165 
3166   Note: Currently, this is restricted to a single level meet
3167 
3168   Fortran Notes:
3169   Since it returns an array, this routine is only available in Fortran 90, and you must
3170   include petsc.h90 in your code.
3171 
3172   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3173 
3174 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3175 @*/
3176 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3177 {
3178   DM_Plex       *mesh = (DM_Plex*) dm->data;
3179   PetscInt      *meet[2];
3180   PetscInt       meetSize, i = 0;
3181   PetscInt       dof, off, p, c, m;
3182   PetscErrorCode ierr;
3183 
3184   PetscFunctionBegin;
3185   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3186   PetscValidPointer(points, 2);
3187   PetscValidPointer(numCoveringPoints, 3);
3188   PetscValidPointer(coveringPoints, 4);
3189   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3190   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3191   /* Copy in cone of first point */
3192   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3193   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3194   for (meetSize = 0; meetSize < dof; ++meetSize) {
3195     meet[i][meetSize] = mesh->cones[off+meetSize];
3196   }
3197   /* Check each successive cone */
3198   for (p = 1; p < numPoints; ++p) {
3199     PetscInt newMeetSize = 0;
3200 
3201     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3202     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3203     for (c = 0; c < dof; ++c) {
3204       const PetscInt point = mesh->cones[off+c];
3205 
3206       for (m = 0; m < meetSize; ++m) {
3207         if (point == meet[i][m]) {
3208           meet[1-i][newMeetSize++] = point;
3209           break;
3210         }
3211       }
3212     }
3213     meetSize = newMeetSize;
3214     i        = 1-i;
3215   }
3216   *numCoveringPoints = meetSize;
3217   *coveringPoints    = meet[i];
3218   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3219   PetscFunctionReturn(0);
3220 }
3221 
3222 /*@C
3223   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3224 
3225   Not Collective
3226 
3227   Input Parameters:
3228 + dm - The DMPlex object
3229 . numPoints - The number of input points for the meet
3230 - points - The input points
3231 
3232   Output Parameters:
3233 + numCoveredPoints - The number of points in the meet
3234 - coveredPoints - The points in the meet
3235 
3236   Level: intermediate
3237 
3238   Fortran Notes:
3239   Since it returns an array, this routine is only available in Fortran 90, and you must
3240   include petsc.h90 in your code.
3241 
3242   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3243 
3244 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3245 @*/
3246 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3247 {
3248   PetscErrorCode ierr;
3249 
3250   PetscFunctionBegin;
3251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3252   if (points) PetscValidIntPointer(points,3);
3253   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3254   PetscValidPointer(coveredPoints,5);
3255   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3256   if (numCoveredPoints) *numCoveredPoints = 0;
3257   PetscFunctionReturn(0);
3258 }
3259 
3260 /*@C
3261   DMPlexGetFullMeet - Get an array for the meet of the set of points
3262 
3263   Not Collective
3264 
3265   Input Parameters:
3266 + dm - The DMPlex object
3267 . numPoints - The number of input points for the meet
3268 - points - The input points
3269 
3270   Output Parameters:
3271 + numCoveredPoints - The number of points in the meet
3272 - coveredPoints - The points in the meet
3273 
3274   Level: intermediate
3275 
3276   Fortran Notes:
3277   Since it returns an array, this routine is only available in Fortran 90, and you must
3278   include petsc.h90 in your code.
3279 
3280   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3281 
3282 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3283 @*/
3284 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3285 {
3286   DM_Plex       *mesh = (DM_Plex*) dm->data;
3287   PetscInt      *offsets, **closures;
3288   PetscInt      *meet[2];
3289   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3290   PetscInt       p, h, c, m, mc;
3291   PetscErrorCode ierr;
3292 
3293   PetscFunctionBegin;
3294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3295   PetscValidPointer(points, 2);
3296   PetscValidPointer(numCoveredPoints, 3);
3297   PetscValidPointer(coveredPoints, 4);
3298 
3299   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3300   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3301   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3302   mc      = mesh->maxConeSize;
3303   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3304   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3305   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3306 
3307   for (p = 0; p < numPoints; ++p) {
3308     PetscInt closureSize;
3309 
3310     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3311 
3312     offsets[p*(height+2)+0] = 0;
3313     for (h = 0; h < height+1; ++h) {
3314       PetscInt pStart, pEnd, i;
3315 
3316       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3317       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3318         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3319           offsets[p*(height+2)+h+1] = i;
3320           break;
3321         }
3322       }
3323       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3324     }
3325     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3326   }
3327   for (h = 0; h < height+1; ++h) {
3328     PetscInt dof;
3329 
3330     /* Copy in cone of first point */
3331     dof = offsets[h+1] - offsets[h];
3332     for (meetSize = 0; meetSize < dof; ++meetSize) {
3333       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3334     }
3335     /* Check each successive cone */
3336     for (p = 1; p < numPoints && meetSize; ++p) {
3337       PetscInt newMeetSize = 0;
3338 
3339       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3340       for (c = 0; c < dof; ++c) {
3341         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3342 
3343         for (m = 0; m < meetSize; ++m) {
3344           if (point == meet[i][m]) {
3345             meet[1-i][newMeetSize++] = point;
3346             break;
3347           }
3348         }
3349       }
3350       meetSize = newMeetSize;
3351       i        = 1-i;
3352     }
3353     if (meetSize) break;
3354   }
3355   *numCoveredPoints = meetSize;
3356   *coveredPoints    = meet[i];
3357   for (p = 0; p < numPoints; ++p) {
3358     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3359   }
3360   ierr = PetscFree(closures);CHKERRQ(ierr);
3361   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3362   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3363   PetscFunctionReturn(0);
3364 }
3365 
3366 /*@C
3367   DMPlexEqual - Determine if two DMs have the same topology
3368 
3369   Not Collective
3370 
3371   Input Parameters:
3372 + dmA - A DMPlex object
3373 - dmB - A DMPlex object
3374 
3375   Output Parameters:
3376 . equal - PETSC_TRUE if the topologies are identical
3377 
3378   Level: intermediate
3379 
3380   Notes:
3381   We are not solving graph isomorphism, so we do not permutation.
3382 
3383 .seealso: DMPlexGetCone()
3384 @*/
3385 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3386 {
3387   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3388   PetscErrorCode ierr;
3389 
3390   PetscFunctionBegin;
3391   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3392   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3393   PetscValidPointer(equal, 3);
3394 
3395   *equal = PETSC_FALSE;
3396   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3397   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3398   if (depth != depthB) PetscFunctionReturn(0);
3399   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3400   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3401   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3402   for (p = pStart; p < pEnd; ++p) {
3403     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3404     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3405 
3406     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3407     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3408     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3409     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3410     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3411     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3412     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3413     for (c = 0; c < coneSize; ++c) {
3414       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3415       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3416     }
3417     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3418     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3419     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3420     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3421     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3422     for (s = 0; s < supportSize; ++s) {
3423       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3424     }
3425   }
3426   *equal = PETSC_TRUE;
3427   PetscFunctionReturn(0);
3428 }
3429 
3430 /*@C
3431   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3432 
3433   Not Collective
3434 
3435   Input Parameters:
3436 + dm         - The DMPlex
3437 . cellDim    - The cell dimension
3438 - numCorners - The number of vertices on a cell
3439 
3440   Output Parameters:
3441 . numFaceVertices - The number of vertices on a face
3442 
3443   Level: developer
3444 
3445   Notes:
3446   Of course this can only work for a restricted set of symmetric shapes
3447 
3448 .seealso: DMPlexGetCone()
3449 @*/
3450 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3451 {
3452   MPI_Comm       comm;
3453   PetscErrorCode ierr;
3454 
3455   PetscFunctionBegin;
3456   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3457   PetscValidPointer(numFaceVertices,3);
3458   switch (cellDim) {
3459   case 0:
3460     *numFaceVertices = 0;
3461     break;
3462   case 1:
3463     *numFaceVertices = 1;
3464     break;
3465   case 2:
3466     switch (numCorners) {
3467     case 3: /* triangle */
3468       *numFaceVertices = 2; /* Edge has 2 vertices */
3469       break;
3470     case 4: /* quadrilateral */
3471       *numFaceVertices = 2; /* Edge has 2 vertices */
3472       break;
3473     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3474       *numFaceVertices = 3; /* Edge has 3 vertices */
3475       break;
3476     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3477       *numFaceVertices = 3; /* Edge has 3 vertices */
3478       break;
3479     default:
3480       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3481     }
3482     break;
3483   case 3:
3484     switch (numCorners) {
3485     case 4: /* tetradehdron */
3486       *numFaceVertices = 3; /* Face has 3 vertices */
3487       break;
3488     case 6: /* tet cohesive cells */
3489       *numFaceVertices = 4; /* Face has 4 vertices */
3490       break;
3491     case 8: /* hexahedron */
3492       *numFaceVertices = 4; /* Face has 4 vertices */
3493       break;
3494     case 9: /* tet cohesive Lagrange cells */
3495       *numFaceVertices = 6; /* Face has 6 vertices */
3496       break;
3497     case 10: /* quadratic tetrahedron */
3498       *numFaceVertices = 6; /* Face has 6 vertices */
3499       break;
3500     case 12: /* hex cohesive Lagrange cells */
3501       *numFaceVertices = 6; /* Face has 6 vertices */
3502       break;
3503     case 18: /* quadratic tet cohesive Lagrange cells */
3504       *numFaceVertices = 6; /* Face has 6 vertices */
3505       break;
3506     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3507       *numFaceVertices = 9; /* Face has 9 vertices */
3508       break;
3509     default:
3510       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3511     }
3512     break;
3513   default:
3514     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3515   }
3516   PetscFunctionReturn(0);
3517 }
3518 
3519 /*@
3520   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3521 
3522   Not Collective
3523 
3524   Input Parameter:
3525 . dm    - The DMPlex object
3526 
3527   Output Parameter:
3528 . depthLabel - The DMLabel recording point depth
3529 
3530   Level: developer
3531 
3532 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3533 @*/
3534 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3535 {
3536   PetscErrorCode ierr;
3537 
3538   PetscFunctionBegin;
3539   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3540   PetscValidPointer(depthLabel, 2);
3541   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3542   *depthLabel = dm->depthLabel;
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 /*@
3547   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3548 
3549   Not Collective
3550 
3551   Input Parameter:
3552 . dm    - The DMPlex object
3553 
3554   Output Parameter:
3555 . depth - The number of strata (breadth first levels) in the DAG
3556 
3557   Level: developer
3558 
3559 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3560 @*/
3561 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3562 {
3563   DMLabel        label;
3564   PetscInt       d = 0;
3565   PetscErrorCode ierr;
3566 
3567   PetscFunctionBegin;
3568   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3569   PetscValidPointer(depth, 2);
3570   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3571   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3572   *depth = d-1;
3573   PetscFunctionReturn(0);
3574 }
3575 
3576 /*@
3577   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3578 
3579   Not Collective
3580 
3581   Input Parameters:
3582 + dm           - The DMPlex object
3583 - stratumValue - The requested depth
3584 
3585   Output Parameters:
3586 + start - The first point at this depth
3587 - end   - One beyond the last point at this depth
3588 
3589   Notes:
3590   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3591   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3592   higher dimension, e.g., "edges".
3593 
3594   Level: developer
3595 
3596 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3597 @*/
3598 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3599 {
3600   DMLabel        label;
3601   PetscInt       pStart, pEnd;
3602   PetscErrorCode ierr;
3603 
3604   PetscFunctionBegin;
3605   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3606   if (start) {PetscValidPointer(start, 3); *start = 0;}
3607   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3608   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3609   if (pStart == pEnd) PetscFunctionReturn(0);
3610   if (stratumValue < 0) {
3611     if (start) *start = pStart;
3612     if (end)   *end   = pEnd;
3613     PetscFunctionReturn(0);
3614   }
3615   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3616   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3617   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3618   PetscFunctionReturn(0);
3619 }
3620 
3621 /*@
3622   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3623 
3624   Not Collective
3625 
3626   Input Parameters:
3627 + dm           - The DMPlex object
3628 - stratumValue - The requested height
3629 
3630   Output Parameters:
3631 + start - The first point at this height
3632 - end   - One beyond the last point at this height
3633 
3634   Notes:
3635   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3636   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3637   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3638 
3639   Level: developer
3640 
3641 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3642 @*/
3643 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3644 {
3645   DMLabel        label;
3646   PetscInt       depth, pStart, pEnd;
3647   PetscErrorCode ierr;
3648 
3649   PetscFunctionBegin;
3650   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3651   if (start) {PetscValidPointer(start, 3); *start = 0;}
3652   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3653   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3654   if (pStart == pEnd) PetscFunctionReturn(0);
3655   if (stratumValue < 0) {
3656     if (start) *start = pStart;
3657     if (end)   *end   = pEnd;
3658     PetscFunctionReturn(0);
3659   }
3660   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3661   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3662   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3663   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3664   PetscFunctionReturn(0);
3665 }
3666 
3667 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3668 {
3669   PetscSection   section, s;
3670   Mat            m;
3671   PetscInt       maxHeight;
3672   PetscErrorCode ierr;
3673 
3674   PetscFunctionBegin;
3675   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3676   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3677   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3678   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3679   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3680   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3681   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3682   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3683   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3684   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3685   ierr = MatDestroy(&m);CHKERRQ(ierr);
3686 
3687   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3688   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3689   PetscFunctionReturn(0);
3690 }
3691 
3692 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3693 {
3694   Vec            coordsLocal;
3695   DM             coordsDM;
3696   PetscErrorCode ierr;
3697 
3698   PetscFunctionBegin;
3699   *field = NULL;
3700   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3701   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3702   if (coordsLocal && coordsDM) {
3703     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3704   }
3705   PetscFunctionReturn(0);
3706 }
3707 
3708 /*@C
3709   DMPlexGetConeSection - Return a section which describes the layout of cone data
3710 
3711   Not Collective
3712 
3713   Input Parameters:
3714 . dm        - The DMPlex object
3715 
3716   Output Parameter:
3717 . section - The PetscSection object
3718 
3719   Level: developer
3720 
3721 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3722 @*/
3723 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3724 {
3725   DM_Plex *mesh = (DM_Plex*) dm->data;
3726 
3727   PetscFunctionBegin;
3728   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3729   if (section) *section = mesh->coneSection;
3730   PetscFunctionReturn(0);
3731 }
3732 
3733 /*@C
3734   DMPlexGetSupportSection - Return a section which describes the layout of support data
3735 
3736   Not Collective
3737 
3738   Input Parameters:
3739 . dm        - The DMPlex object
3740 
3741   Output Parameter:
3742 . section - The PetscSection object
3743 
3744   Level: developer
3745 
3746 .seealso: DMPlexGetConeSection()
3747 @*/
3748 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3749 {
3750   DM_Plex *mesh = (DM_Plex*) dm->data;
3751 
3752   PetscFunctionBegin;
3753   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3754   if (section) *section = mesh->supportSection;
3755   PetscFunctionReturn(0);
3756 }
3757 
3758 /*@C
3759   DMPlexGetCones - Return cone data
3760 
3761   Not Collective
3762 
3763   Input Parameters:
3764 . dm        - The DMPlex object
3765 
3766   Output Parameter:
3767 . cones - The cone for each point
3768 
3769   Level: developer
3770 
3771 .seealso: DMPlexGetConeSection()
3772 @*/
3773 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3774 {
3775   DM_Plex *mesh = (DM_Plex*) dm->data;
3776 
3777   PetscFunctionBegin;
3778   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3779   if (cones) *cones = mesh->cones;
3780   PetscFunctionReturn(0);
3781 }
3782 
3783 /*@C
3784   DMPlexGetConeOrientations - Return cone orientation data
3785 
3786   Not Collective
3787 
3788   Input Parameters:
3789 . dm        - The DMPlex object
3790 
3791   Output Parameter:
3792 . coneOrientations - The cone orientation for each point
3793 
3794   Level: developer
3795 
3796 .seealso: DMPlexGetConeSection()
3797 @*/
3798 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3799 {
3800   DM_Plex *mesh = (DM_Plex*) dm->data;
3801 
3802   PetscFunctionBegin;
3803   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3804   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3805   PetscFunctionReturn(0);
3806 }
3807 
3808 /******************************** FEM Support **********************************/
3809 
3810 /*
3811  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3812  representing a line in the section.
3813 */
3814 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3815 {
3816   PetscErrorCode ierr;
3817 
3818   PetscFunctionBeginHot;
3819   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3820   if (line < 0) {
3821     *k = 0;
3822     *Nc = 0;
3823   } else if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3824     *k = 1;
3825   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3826     /* An order k SEM disc has k-1 dofs on an edge */
3827     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3828     *k = *k / *Nc + 1;
3829   }
3830   PetscFunctionReturn(0);
3831 }
3832 
3833 /*@
3834 
3835   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3836   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3837   section provided (or the section of the DM).
3838 
3839   Input Parameters:
3840 + dm      - The DM
3841 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3842 - section - The PetscSection to reorder, or NULL for the default section
3843 
3844   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3845   degree of the basis.
3846 
3847   Example:
3848   A typical interpolated single-quad mesh might order points as
3849 .vb
3850   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3851 
3852   v4 -- e6 -- v3
3853   |           |
3854   e7    c0    e8
3855   |           |
3856   v1 -- e5 -- v2
3857 .ve
3858 
3859   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3860   dofs in the order of points, e.g.,
3861 .vb
3862     c0 -> [0,1,2,3]
3863     v1 -> [4]
3864     ...
3865     e5 -> [8, 9]
3866 .ve
3867 
3868   which corresponds to the dofs
3869 .vb
3870     6   10  11  7
3871     13  2   3   15
3872     12  0   1   14
3873     4   8   9   5
3874 .ve
3875 
3876   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3877 .vb
3878   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3879 .ve
3880 
3881   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3882 .vb
3883    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3884 .ve
3885 
3886   Level: developer
3887 
3888 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlocalSection()
3889 @*/
3890 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3891 {
3892   DMLabel        label;
3893   PetscInt      *perm;
3894   PetscInt       dim, depth = -1, eStart = -1, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3895   PetscBool      vertexchart;
3896   PetscErrorCode ierr;
3897 
3898   PetscFunctionBegin;
3899   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3900   if (dim < 1) PetscFunctionReturn(0);
3901   if (point < 0) {
3902     PetscInt sStart,sEnd;
3903 
3904     ierr = DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd);CHKERRQ(ierr);
3905     point = sEnd-sStart ? sStart : point;
3906   }
3907   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3908   if (point >= 0) { ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr); }
3909   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
3910   if (depth == 1) {eStart = point;}
3911   else if  (depth == dim) {
3912     const PetscInt *cone;
3913 
3914     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3915     if (dim == 2) eStart = cone[0];
3916     else if (dim == 3) {
3917       const PetscInt *cone2;
3918       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3919       eStart = cone2[0];
3920     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3921   } else if (depth >= 0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3922   {                             /* Determine whether the chart covers all points or just vertices. */
3923     PetscInt pStart,pEnd,cStart,cEnd;
3924     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3925     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3926     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3927     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3928   }
3929   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3930   for (f = 0; f < Nf; ++f) {
3931     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3932     size += PetscPowInt(k+1, dim)*Nc;
3933   }
3934   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3935   for (f = 0; f < Nf; ++f) {
3936     switch (dim) {
3937     case 1:
3938       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3939       /*
3940         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3941         We want              [ vtx0; edge of length k-1; vtx1 ]
3942       */
3943       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3944       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3945       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3946       foffset = offset;
3947       break;
3948     case 2:
3949       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3950       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3951       /* The SEM order is
3952 
3953          v_lb, {e_b}, v_rb,
3954          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3955          v_lt, reverse {e_t}, v_rt
3956       */
3957       {
3958         const PetscInt of   = 0;
3959         const PetscInt oeb  = of   + PetscSqr(k-1);
3960         const PetscInt oer  = oeb  + (k-1);
3961         const PetscInt oet  = oer  + (k-1);
3962         const PetscInt oel  = oet  + (k-1);
3963         const PetscInt ovlb = oel  + (k-1);
3964         const PetscInt ovrb = ovlb + 1;
3965         const PetscInt ovrt = ovrb + 1;
3966         const PetscInt ovlt = ovrt + 1;
3967         PetscInt       o;
3968 
3969         /* bottom */
3970         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3971         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3972         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3973         /* middle */
3974         for (i = 0; i < k-1; ++i) {
3975           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3976           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3977           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3978         }
3979         /* top */
3980         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3981         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3982         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3983         foffset = offset;
3984       }
3985       break;
3986     case 3:
3987       /* The original hex closure is
3988 
3989          {c,
3990           f_b, f_t, f_f, f_b, f_r, f_l,
3991           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3992           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3993       */
3994       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3995       /* The SEM order is
3996          Bottom Slice
3997          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3998          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3999          v_blb, {e_bb}, v_brb,
4000 
4001          Middle Slice (j)
4002          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
4003          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
4004          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
4005 
4006          Top Slice
4007          v_tlf, {e_tf}, v_trf,
4008          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
4009          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
4010       */
4011       {
4012         const PetscInt oc    = 0;
4013         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4014         const PetscInt oft   = ofb   + PetscSqr(k-1);
4015         const PetscInt off   = oft   + PetscSqr(k-1);
4016         const PetscInt ofk   = off   + PetscSqr(k-1);
4017         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4018         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4019         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4020         const PetscInt oebb  = oebl  + (k-1);
4021         const PetscInt oebr  = oebb  + (k-1);
4022         const PetscInt oebf  = oebr  + (k-1);
4023         const PetscInt oetf  = oebf  + (k-1);
4024         const PetscInt oetr  = oetf  + (k-1);
4025         const PetscInt oetb  = oetr  + (k-1);
4026         const PetscInt oetl  = oetb  + (k-1);
4027         const PetscInt oerf  = oetl  + (k-1);
4028         const PetscInt oelf  = oerf  + (k-1);
4029         const PetscInt oelb  = oelf  + (k-1);
4030         const PetscInt oerb  = oelb  + (k-1);
4031         const PetscInt ovblf = oerb  + (k-1);
4032         const PetscInt ovblb = ovblf + 1;
4033         const PetscInt ovbrb = ovblb + 1;
4034         const PetscInt ovbrf = ovbrb + 1;
4035         const PetscInt ovtlf = ovbrf + 1;
4036         const PetscInt ovtrf = ovtlf + 1;
4037         const PetscInt ovtrb = ovtrf + 1;
4038         const PetscInt ovtlb = ovtrb + 1;
4039         PetscInt       o, n;
4040 
4041         /* Bottom Slice */
4042         /*   bottom */
4043         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4044         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4045         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4046         /*   middle */
4047         for (i = 0; i < k-1; ++i) {
4048           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4049           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4050           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4051         }
4052         /*   top */
4053         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4054         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4055         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4056 
4057         /* Middle Slice */
4058         for (j = 0; j < k-1; ++j) {
4059           /*   bottom */
4060           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4061           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4062           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4063           /*   middle */
4064           for (i = 0; i < k-1; ++i) {
4065             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4066             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4067             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4068           }
4069           /*   top */
4070           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4071           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4072           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4073         }
4074 
4075         /* Top Slice */
4076         /*   bottom */
4077         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4078         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4079         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4080         /*   middle */
4081         for (i = 0; i < k-1; ++i) {
4082           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4083           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4084           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4085         }
4086         /*   top */
4087         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4088         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4089         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4090 
4091         foffset = offset;
4092       }
4093       break;
4094     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4095     }
4096   }
4097   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4098   /* Check permutation */
4099   {
4100     PetscInt *check;
4101 
4102     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4103     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4104     for (i = 0; i < size; ++i) check[perm[i]] = i;
4105     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4106     ierr = PetscFree(check);CHKERRQ(ierr);
4107   }
4108   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4109   PetscFunctionReturn(0);
4110 }
4111 
4112 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4113 {
4114   PetscDS        prob;
4115   PetscInt       depth, Nf, h;
4116   DMLabel        label;
4117   PetscErrorCode ierr;
4118 
4119   PetscFunctionBeginHot;
4120   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4121   Nf      = prob->Nf;
4122   label   = dm->depthLabel;
4123   *dspace = NULL;
4124   if (field < Nf) {
4125     PetscObject disc = prob->disc[field];
4126 
4127     if (disc->classid == PETSCFE_CLASSID) {
4128       PetscDualSpace dsp;
4129 
4130       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4131       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4132       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4133       h    = depth - 1 - h;
4134       if (h) {
4135         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4136       } else {
4137         *dspace = dsp;
4138       }
4139     }
4140   }
4141   PetscFunctionReturn(0);
4142 }
4143 
4144 
4145 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4146 {
4147   PetscScalar    *array, *vArray;
4148   const PetscInt *cone, *coneO;
4149   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4150   PetscErrorCode  ierr;
4151 
4152   PetscFunctionBeginHot;
4153   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4154   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4155   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4156   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4157   if (!values || !*values) {
4158     if ((point >= pStart) && (point < pEnd)) {
4159       PetscInt dof;
4160 
4161       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4162       size += dof;
4163     }
4164     for (p = 0; p < numPoints; ++p) {
4165       const PetscInt cp = cone[p];
4166       PetscInt       dof;
4167 
4168       if ((cp < pStart) || (cp >= pEnd)) continue;
4169       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4170       size += dof;
4171     }
4172     if (!values) {
4173       if (csize) *csize = size;
4174       PetscFunctionReturn(0);
4175     }
4176     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4177   } else {
4178     array = *values;
4179   }
4180   size = 0;
4181   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4182   if ((point >= pStart) && (point < pEnd)) {
4183     PetscInt     dof, off, d;
4184     PetscScalar *varr;
4185 
4186     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4187     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4188     varr = &vArray[off];
4189     for (d = 0; d < dof; ++d, ++offset) {
4190       array[offset] = varr[d];
4191     }
4192     size += dof;
4193   }
4194   for (p = 0; p < numPoints; ++p) {
4195     const PetscInt cp = cone[p];
4196     PetscInt       o  = coneO[p];
4197     PetscInt       dof, off, d;
4198     PetscScalar   *varr;
4199 
4200     if ((cp < pStart) || (cp >= pEnd)) continue;
4201     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4202     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4203     varr = &vArray[off];
4204     if (o >= 0) {
4205       for (d = 0; d < dof; ++d, ++offset) {
4206         array[offset] = varr[d];
4207       }
4208     } else {
4209       for (d = dof-1; d >= 0; --d, ++offset) {
4210         array[offset] = varr[d];
4211       }
4212     }
4213     size += dof;
4214   }
4215   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4216   if (!*values) {
4217     if (csize) *csize = size;
4218     *values = array;
4219   } else {
4220     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4221     *csize = size;
4222   }
4223   PetscFunctionReturn(0);
4224 }
4225 
4226 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4227 {
4228   const PetscInt *cla;
4229   PetscInt       np, *pts = NULL;
4230   PetscErrorCode ierr;
4231 
4232   PetscFunctionBeginHot;
4233   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4234   if (!*clPoints) {
4235     PetscInt pStart, pEnd, p, q;
4236 
4237     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4238     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4239     /* Compress out points not in the section */
4240     for (p = 0, q = 0; p < np; p++) {
4241       PetscInt r = pts[2*p];
4242       if ((r >= pStart) && (r < pEnd)) {
4243         pts[q*2]   = r;
4244         pts[q*2+1] = pts[2*p+1];
4245         ++q;
4246       }
4247     }
4248     np = q;
4249     cla = NULL;
4250   } else {
4251     PetscInt dof, off;
4252 
4253     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4254     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4255     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4256     np   = dof/2;
4257     pts  = (PetscInt *) &cla[off];
4258   }
4259   *numPoints = np;
4260   *points    = pts;
4261   *clp       = cla;
4262 
4263   PetscFunctionReturn(0);
4264 }
4265 
4266 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4267 {
4268   PetscErrorCode ierr;
4269 
4270   PetscFunctionBeginHot;
4271   if (!*clPoints) {
4272     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4273   } else {
4274     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4275   }
4276   *numPoints = 0;
4277   *points    = NULL;
4278   *clSec     = NULL;
4279   *clPoints  = NULL;
4280   *clp       = NULL;
4281   PetscFunctionReturn(0);
4282 }
4283 
4284 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4285 {
4286   PetscInt          offset = 0, p;
4287   const PetscInt    **perms = NULL;
4288   const PetscScalar **flips = NULL;
4289   PetscErrorCode    ierr;
4290 
4291   PetscFunctionBeginHot;
4292   *size = 0;
4293   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4294   for (p = 0; p < numPoints; p++) {
4295     const PetscInt    point = points[2*p];
4296     const PetscInt    *perm = perms ? perms[p] : NULL;
4297     const PetscScalar *flip = flips ? flips[p] : NULL;
4298     PetscInt          dof, off, d;
4299     const PetscScalar *varr;
4300 
4301     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4302     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4303     varr = &vArray[off];
4304     if (clperm) {
4305       if (perm) {
4306         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4307       } else {
4308         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4309       }
4310       if (flip) {
4311         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4312       }
4313     } else {
4314       if (perm) {
4315         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4316       } else {
4317         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4318       }
4319       if (flip) {
4320         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4321       }
4322     }
4323     offset += dof;
4324   }
4325   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4326   *size = offset;
4327   PetscFunctionReturn(0);
4328 }
4329 
4330 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4331 {
4332   PetscInt          offset = 0, f;
4333   PetscErrorCode    ierr;
4334 
4335   PetscFunctionBeginHot;
4336   *size = 0;
4337   for (f = 0; f < numFields; ++f) {
4338     PetscInt          p;
4339     const PetscInt    **perms = NULL;
4340     const PetscScalar **flips = NULL;
4341 
4342     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4343     for (p = 0; p < numPoints; p++) {
4344       const PetscInt    point = points[2*p];
4345       PetscInt          fdof, foff, b;
4346       const PetscScalar *varr;
4347       const PetscInt    *perm = perms ? perms[p] : NULL;
4348       const PetscScalar *flip = flips ? flips[p] : NULL;
4349 
4350       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4351       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4352       varr = &vArray[foff];
4353       if (clperm) {
4354         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4355         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4356         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4357       } else {
4358         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4359         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4360         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4361       }
4362       offset += fdof;
4363     }
4364     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4365   }
4366   *size = offset;
4367   PetscFunctionReturn(0);
4368 }
4369 
4370 /*@C
4371   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4372 
4373   Not collective
4374 
4375   Input Parameters:
4376 + dm - The DM
4377 . section - The section describing the layout in v, or NULL to use the default section
4378 . v - The local vector
4379 . point - The point in the DM
4380 . csize - The size of the input values array, or NULL
4381 - values - An array to use for the values, or NULL to have it allocated automatically
4382 
4383   Output Parameters:
4384 + csize - The number of values in the closure
4385 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4386 
4387 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4388 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4389 $ assembly function, and a user may already have allocated storage for this operation.
4390 $
4391 $ A typical use could be
4392 $
4393 $  values = NULL;
4394 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4395 $  for (cl = 0; cl < clSize; ++cl) {
4396 $    <Compute on closure>
4397 $  }
4398 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4399 $
4400 $ or
4401 $
4402 $  PetscMalloc1(clMaxSize, &values);
4403 $  for (p = pStart; p < pEnd; ++p) {
4404 $    clSize = clMaxSize;
4405 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4406 $    for (cl = 0; cl < clSize; ++cl) {
4407 $      <Compute on closure>
4408 $    }
4409 $  }
4410 $  PetscFree(values);
4411 
4412   Fortran Notes:
4413   Since it returns an array, this routine is only available in Fortran 90, and you must
4414   include petsc.h90 in your code.
4415 
4416   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4417 
4418   Level: intermediate
4419 
4420 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4421 @*/
4422 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4423 {
4424   PetscSection       clSection;
4425   IS                 clPoints;
4426   PetscScalar       *array;
4427   const PetscScalar *vArray;
4428   PetscInt          *points = NULL;
4429   const PetscInt    *clp, *perm;
4430   PetscInt           depth, numFields, numPoints, size;
4431   PetscErrorCode     ierr;
4432 
4433   PetscFunctionBeginHot;
4434   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4435   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4436   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4437   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4438   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4439   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4440   if (depth == 1 && numFields < 2) {
4441     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4442     PetscFunctionReturn(0);
4443   }
4444   /* Get points */
4445   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4446   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4447   /* Get array */
4448   if (!values || !*values) {
4449     PetscInt asize = 0, dof, p;
4450 
4451     for (p = 0; p < numPoints*2; p += 2) {
4452       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4453       asize += dof;
4454     }
4455     if (!values) {
4456       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4457       if (csize) *csize = asize;
4458       PetscFunctionReturn(0);
4459     }
4460     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4461   } else {
4462     array = *values;
4463   }
4464   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4465   /* Get values */
4466   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4467   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4468   /* Cleanup points */
4469   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4470   /* Cleanup array */
4471   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4472   if (!*values) {
4473     if (csize) *csize = size;
4474     *values = array;
4475   } else {
4476     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4477     *csize = size;
4478   }
4479   PetscFunctionReturn(0);
4480 }
4481 
4482 /*@C
4483   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4484 
4485   Not collective
4486 
4487   Input Parameters:
4488 + dm - The DM
4489 . section - The section describing the layout in v, or NULL to use the default section
4490 . v - The local vector
4491 . point - The point in the DM
4492 . csize - The number of values in the closure, or NULL
4493 - values - The array of values, which is a borrowed array and should not be freed
4494 
4495   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4496 
4497   Fortran Notes:
4498   Since it returns an array, this routine is only available in Fortran 90, and you must
4499   include petsc.h90 in your code.
4500 
4501   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4502 
4503   Level: intermediate
4504 
4505 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4506 @*/
4507 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4508 {
4509   PetscInt       size = 0;
4510   PetscErrorCode ierr;
4511 
4512   PetscFunctionBegin;
4513   /* Should work without recalculating size */
4514   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4515   *values = NULL;
4516   PetscFunctionReturn(0);
4517 }
4518 
4519 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4520 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4521 
4522 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4523 {
4524   PetscInt        cdof;   /* The number of constraints on this point */
4525   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4526   PetscScalar    *a;
4527   PetscInt        off, cind = 0, k;
4528   PetscErrorCode  ierr;
4529 
4530   PetscFunctionBegin;
4531   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4532   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4533   a    = &array[off];
4534   if (!cdof || setBC) {
4535     if (clperm) {
4536       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4537       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4538     } else {
4539       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4540       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4541     }
4542   } else {
4543     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4544     if (clperm) {
4545       if (perm) {for (k = 0; k < dof; ++k) {
4546           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4547           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4548         }
4549       } else {
4550         for (k = 0; k < dof; ++k) {
4551           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4552           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4553         }
4554       }
4555     } else {
4556       if (perm) {
4557         for (k = 0; k < dof; ++k) {
4558           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4559           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4560         }
4561       } else {
4562         for (k = 0; k < dof; ++k) {
4563           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4564           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4565         }
4566       }
4567     }
4568   }
4569   PetscFunctionReturn(0);
4570 }
4571 
4572 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4573 {
4574   PetscInt        cdof;   /* The number of constraints on this point */
4575   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4576   PetscScalar    *a;
4577   PetscInt        off, cind = 0, k;
4578   PetscErrorCode  ierr;
4579 
4580   PetscFunctionBegin;
4581   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4582   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4583   a    = &array[off];
4584   if (cdof) {
4585     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4586     if (clperm) {
4587       if (perm) {
4588         for (k = 0; k < dof; ++k) {
4589           if ((cind < cdof) && (k == cdofs[cind])) {
4590             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4591             cind++;
4592           }
4593         }
4594       } else {
4595         for (k = 0; k < dof; ++k) {
4596           if ((cind < cdof) && (k == cdofs[cind])) {
4597             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4598             cind++;
4599           }
4600         }
4601       }
4602     } else {
4603       if (perm) {
4604         for (k = 0; k < dof; ++k) {
4605           if ((cind < cdof) && (k == cdofs[cind])) {
4606             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4607             cind++;
4608           }
4609         }
4610       } else {
4611         for (k = 0; k < dof; ++k) {
4612           if ((cind < cdof) && (k == cdofs[cind])) {
4613             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4614             cind++;
4615           }
4616         }
4617       }
4618     }
4619   }
4620   PetscFunctionReturn(0);
4621 }
4622 
4623 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4624 {
4625   PetscScalar    *a;
4626   PetscInt        fdof, foff, fcdof, foffset = *offset;
4627   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4628   PetscInt        cind = 0, b;
4629   PetscErrorCode  ierr;
4630 
4631   PetscFunctionBegin;
4632   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4633   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4634   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4635   a    = &array[foff];
4636   if (!fcdof || setBC) {
4637     if (clperm) {
4638       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4639       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4640     } else {
4641       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4642       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4643     }
4644   } else {
4645     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4646     if (clperm) {
4647       if (perm) {
4648         for (b = 0; b < fdof; b++) {
4649           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4650           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4651         }
4652       } else {
4653         for (b = 0; b < fdof; b++) {
4654           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4655           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4656         }
4657       }
4658     } else {
4659       if (perm) {
4660         for (b = 0; b < fdof; b++) {
4661           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4662           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4663         }
4664       } else {
4665         for (b = 0; b < fdof; b++) {
4666           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4667           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4668         }
4669       }
4670     }
4671   }
4672   *offset += fdof;
4673   PetscFunctionReturn(0);
4674 }
4675 
4676 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4677 {
4678   PetscScalar    *a;
4679   PetscInt        fdof, foff, fcdof, foffset = *offset;
4680   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4681   PetscInt        cind = 0, ncind = 0, b;
4682   PetscBool       ncSet, fcSet;
4683   PetscErrorCode  ierr;
4684 
4685   PetscFunctionBegin;
4686   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4687   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4688   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4689   a    = &array[foff];
4690   if (fcdof) {
4691     /* We just override fcdof and fcdofs with Ncc and comps */
4692     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4693     if (clperm) {
4694       if (perm) {
4695         if (comps) {
4696           for (b = 0; b < fdof; b++) {
4697             ncSet = fcSet = PETSC_FALSE;
4698             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4699             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4700             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4701           }
4702         } else {
4703           for (b = 0; b < fdof; b++) {
4704             if ((cind < fcdof) && (b == fcdofs[cind])) {
4705               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4706               ++cind;
4707             }
4708           }
4709         }
4710       } else {
4711         if (comps) {
4712           for (b = 0; b < fdof; b++) {
4713             ncSet = fcSet = PETSC_FALSE;
4714             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4715             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4716             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4717           }
4718         } else {
4719           for (b = 0; b < fdof; b++) {
4720             if ((cind < fcdof) && (b == fcdofs[cind])) {
4721               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4722               ++cind;
4723             }
4724           }
4725         }
4726       }
4727     } else {
4728       if (perm) {
4729         if (comps) {
4730           for (b = 0; b < fdof; b++) {
4731             ncSet = fcSet = PETSC_FALSE;
4732             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4733             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4734             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4735           }
4736         } else {
4737           for (b = 0; b < fdof; b++) {
4738             if ((cind < fcdof) && (b == fcdofs[cind])) {
4739               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4740               ++cind;
4741             }
4742           }
4743         }
4744       } else {
4745         if (comps) {
4746           for (b = 0; b < fdof; b++) {
4747             ncSet = fcSet = PETSC_FALSE;
4748             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4749             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4750             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4751           }
4752         } else {
4753           for (b = 0; b < fdof; b++) {
4754             if ((cind < fcdof) && (b == fcdofs[cind])) {
4755               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4756               ++cind;
4757             }
4758           }
4759         }
4760       }
4761     }
4762   }
4763   *offset += fdof;
4764   PetscFunctionReturn(0);
4765 }
4766 
4767 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4768 {
4769   PetscScalar    *array;
4770   const PetscInt *cone, *coneO;
4771   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4772   PetscErrorCode  ierr;
4773 
4774   PetscFunctionBeginHot;
4775   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4776   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4777   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4778   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4779   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4780   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4781     const PetscInt cp = !p ? point : cone[p-1];
4782     const PetscInt o  = !p ? 0     : coneO[p-1];
4783 
4784     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4785     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4786     /* ADD_VALUES */
4787     {
4788       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4789       PetscScalar    *a;
4790       PetscInt        cdof, coff, cind = 0, k;
4791 
4792       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4793       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4794       a    = &array[coff];
4795       if (!cdof) {
4796         if (o >= 0) {
4797           for (k = 0; k < dof; ++k) {
4798             a[k] += values[off+k];
4799           }
4800         } else {
4801           for (k = 0; k < dof; ++k) {
4802             a[k] += values[off+dof-k-1];
4803           }
4804         }
4805       } else {
4806         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4807         if (o >= 0) {
4808           for (k = 0; k < dof; ++k) {
4809             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4810             a[k] += values[off+k];
4811           }
4812         } else {
4813           for (k = 0; k < dof; ++k) {
4814             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4815             a[k] += values[off+dof-k-1];
4816           }
4817         }
4818       }
4819     }
4820   }
4821   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4822   PetscFunctionReturn(0);
4823 }
4824 
4825 /*@C
4826   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4827 
4828   Not collective
4829 
4830   Input Parameters:
4831 + dm - The DM
4832 . section - The section describing the layout in v, or NULL to use the default section
4833 . v - The local vector
4834 . point - The point in the DM
4835 . values - The array of values
4836 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4837          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4838 
4839   Fortran Notes:
4840   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4841 
4842   Level: intermediate
4843 
4844 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4845 @*/
4846 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4847 {
4848   PetscSection    clSection;
4849   IS              clPoints;
4850   PetscScalar    *array;
4851   PetscInt       *points = NULL;
4852   const PetscInt *clp, *clperm;
4853   PetscInt        depth, numFields, numPoints, p;
4854   PetscErrorCode  ierr;
4855 
4856   PetscFunctionBeginHot;
4857   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4858   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4859   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4860   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4861   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4862   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4863   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4864     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4865     PetscFunctionReturn(0);
4866   }
4867   /* Get points */
4868   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4869   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4870   /* Get array */
4871   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4872   /* Get values */
4873   if (numFields > 0) {
4874     PetscInt offset = 0, f;
4875     for (f = 0; f < numFields; ++f) {
4876       const PetscInt    **perms = NULL;
4877       const PetscScalar **flips = NULL;
4878 
4879       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4880       switch (mode) {
4881       case INSERT_VALUES:
4882         for (p = 0; p < numPoints; p++) {
4883           const PetscInt    point = points[2*p];
4884           const PetscInt    *perm = perms ? perms[p] : NULL;
4885           const PetscScalar *flip = flips ? flips[p] : NULL;
4886           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4887         } break;
4888       case INSERT_ALL_VALUES:
4889         for (p = 0; p < numPoints; p++) {
4890           const PetscInt    point = points[2*p];
4891           const PetscInt    *perm = perms ? perms[p] : NULL;
4892           const PetscScalar *flip = flips ? flips[p] : NULL;
4893           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4894         } break;
4895       case INSERT_BC_VALUES:
4896         for (p = 0; p < numPoints; p++) {
4897           const PetscInt    point = points[2*p];
4898           const PetscInt    *perm = perms ? perms[p] : NULL;
4899           const PetscScalar *flip = flips ? flips[p] : NULL;
4900           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4901         } break;
4902       case ADD_VALUES:
4903         for (p = 0; p < numPoints; p++) {
4904           const PetscInt    point = points[2*p];
4905           const PetscInt    *perm = perms ? perms[p] : NULL;
4906           const PetscScalar *flip = flips ? flips[p] : NULL;
4907           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4908         } break;
4909       case ADD_ALL_VALUES:
4910         for (p = 0; p < numPoints; p++) {
4911           const PetscInt    point = points[2*p];
4912           const PetscInt    *perm = perms ? perms[p] : NULL;
4913           const PetscScalar *flip = flips ? flips[p] : NULL;
4914           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4915         } break;
4916       case ADD_BC_VALUES:
4917         for (p = 0; p < numPoints; p++) {
4918           const PetscInt    point = points[2*p];
4919           const PetscInt    *perm = perms ? perms[p] : NULL;
4920           const PetscScalar *flip = flips ? flips[p] : NULL;
4921           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4922         } break;
4923       default:
4924         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4925       }
4926       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4927     }
4928   } else {
4929     PetscInt dof, off;
4930     const PetscInt    **perms = NULL;
4931     const PetscScalar **flips = NULL;
4932 
4933     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4934     switch (mode) {
4935     case INSERT_VALUES:
4936       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4937         const PetscInt    point = points[2*p];
4938         const PetscInt    *perm = perms ? perms[p] : NULL;
4939         const PetscScalar *flip = flips ? flips[p] : NULL;
4940         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4941         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4942       } break;
4943     case INSERT_ALL_VALUES:
4944       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4945         const PetscInt    point = points[2*p];
4946         const PetscInt    *perm = perms ? perms[p] : NULL;
4947         const PetscScalar *flip = flips ? flips[p] : NULL;
4948         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4949         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4950       } break;
4951     case INSERT_BC_VALUES:
4952       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4953         const PetscInt    point = points[2*p];
4954         const PetscInt    *perm = perms ? perms[p] : NULL;
4955         const PetscScalar *flip = flips ? flips[p] : NULL;
4956         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4957         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4958       } break;
4959     case ADD_VALUES:
4960       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4961         const PetscInt    point = points[2*p];
4962         const PetscInt    *perm = perms ? perms[p] : NULL;
4963         const PetscScalar *flip = flips ? flips[p] : NULL;
4964         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4965         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4966       } break;
4967     case ADD_ALL_VALUES:
4968       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4969         const PetscInt    point = points[2*p];
4970         const PetscInt    *perm = perms ? perms[p] : NULL;
4971         const PetscScalar *flip = flips ? flips[p] : NULL;
4972         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4973         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4974       } break;
4975     case ADD_BC_VALUES:
4976       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4977         const PetscInt    point = points[2*p];
4978         const PetscInt    *perm = perms ? perms[p] : NULL;
4979         const PetscScalar *flip = flips ? flips[p] : NULL;
4980         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4981         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4982       } break;
4983     default:
4984       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4985     }
4986     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4987   }
4988   /* Cleanup points */
4989   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4990   /* Cleanup array */
4991   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4992   PetscFunctionReturn(0);
4993 }
4994 
4995 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4996 {
4997   PetscSection      clSection;
4998   IS                clPoints;
4999   PetscScalar       *array;
5000   PetscInt          *points = NULL;
5001   const PetscInt    *clp, *clperm;
5002   PetscInt          numFields, numPoints, p;
5003   PetscInt          offset = 0, f;
5004   PetscErrorCode    ierr;
5005 
5006   PetscFunctionBeginHot;
5007   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5008   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5009   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5010   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5011   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5012   /* Get points */
5013   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5014   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5015   /* Get array */
5016   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5017   /* Get values */
5018   for (f = 0; f < numFields; ++f) {
5019     const PetscInt    **perms = NULL;
5020     const PetscScalar **flips = NULL;
5021 
5022     if (!fieldActive[f]) {
5023       for (p = 0; p < numPoints*2; p += 2) {
5024         PetscInt fdof;
5025         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5026         offset += fdof;
5027       }
5028       continue;
5029     }
5030     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5031     switch (mode) {
5032     case INSERT_VALUES:
5033       for (p = 0; p < numPoints; p++) {
5034         const PetscInt    point = points[2*p];
5035         const PetscInt    *perm = perms ? perms[p] : NULL;
5036         const PetscScalar *flip = flips ? flips[p] : NULL;
5037         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5038       } break;
5039     case INSERT_ALL_VALUES:
5040       for (p = 0; p < numPoints; p++) {
5041         const PetscInt    point = points[2*p];
5042         const PetscInt    *perm = perms ? perms[p] : NULL;
5043         const PetscScalar *flip = flips ? flips[p] : NULL;
5044         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5045         } break;
5046     case INSERT_BC_VALUES:
5047       for (p = 0; p < numPoints; p++) {
5048         const PetscInt    point = points[2*p];
5049         const PetscInt    *perm = perms ? perms[p] : NULL;
5050         const PetscScalar *flip = flips ? flips[p] : NULL;
5051         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
5052       } break;
5053     case ADD_VALUES:
5054       for (p = 0; p < numPoints; p++) {
5055         const PetscInt    point = points[2*p];
5056         const PetscInt    *perm = perms ? perms[p] : NULL;
5057         const PetscScalar *flip = flips ? flips[p] : NULL;
5058         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5059       } break;
5060     case ADD_ALL_VALUES:
5061       for (p = 0; p < numPoints; p++) {
5062         const PetscInt    point = points[2*p];
5063         const PetscInt    *perm = perms ? perms[p] : NULL;
5064         const PetscScalar *flip = flips ? flips[p] : NULL;
5065         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5066       } break;
5067     default:
5068       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5069     }
5070     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5071   }
5072   /* Cleanup points */
5073   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5074   /* Cleanup array */
5075   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5076   PetscFunctionReturn(0);
5077 }
5078 
5079 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5080 {
5081   PetscMPIInt    rank;
5082   PetscInt       i, j;
5083   PetscErrorCode ierr;
5084 
5085   PetscFunctionBegin;
5086   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5087   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5088   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5089   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5090   numCIndices = numCIndices ? numCIndices : numRIndices;
5091   for (i = 0; i < numRIndices; i++) {
5092     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5093     for (j = 0; j < numCIndices; j++) {
5094 #if defined(PETSC_USE_COMPLEX)
5095       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5096 #else
5097       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5098 #endif
5099     }
5100     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5101   }
5102   PetscFunctionReturn(0);
5103 }
5104 
5105 /*
5106   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5107 
5108   Input Parameters:
5109 + section - The section for this data layout
5110 . point   - The point contributing dofs with these indices
5111 . off     - The global offset of this point
5112 . loff    - The local offset of each field
5113 . setBC   - The flag determining whether to include indices of bounsary values
5114 . perm    - A permutation of the dofs on this point, or NULL
5115 - indperm - A permutation of the entire indices array, or NULL
5116 
5117   Output Parameter:
5118 . indices - Indices for dofs on this point
5119 
5120   Level: developer
5121 
5122   Note: The indices could be local or global, depending on the value of 'off'.
5123 */
5124 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5125 {
5126   PetscInt        dof;   /* The number of unknowns on this point */
5127   PetscInt        cdof;  /* The number of constraints on this point */
5128   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5129   PetscInt        cind = 0, k;
5130   PetscErrorCode  ierr;
5131 
5132   PetscFunctionBegin;
5133   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5134   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5135   if (!cdof || setBC) {
5136     for (k = 0; k < dof; ++k) {
5137       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5138       const PetscInt ind    = indperm ? indperm[preind] : preind;
5139 
5140       indices[ind] = off + k;
5141     }
5142   } else {
5143     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5144     for (k = 0; k < dof; ++k) {
5145       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5146       const PetscInt ind    = indperm ? indperm[preind] : preind;
5147 
5148       if ((cind < cdof) && (k == cdofs[cind])) {
5149         /* Insert check for returning constrained indices */
5150         indices[ind] = -(off+k+1);
5151         ++cind;
5152       } else {
5153         indices[ind] = off+k-cind;
5154       }
5155     }
5156   }
5157   *loff += dof;
5158   PetscFunctionReturn(0);
5159 }
5160 
5161 /*
5162   This version only believes the point offset from the globalSection
5163 
5164  . off - The global offset of this point
5165 */
5166 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5167 {
5168   PetscInt       numFields, foff, f;
5169   PetscErrorCode ierr;
5170 
5171   PetscFunctionBegin;
5172   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5173   for (f = 0, foff = 0; f < numFields; ++f) {
5174     PetscInt        fdof, cfdof;
5175     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5176     PetscInt        cind = 0, b;
5177     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5178 
5179     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5180     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5181     if (!cfdof || setBC) {
5182       for (b = 0; b < fdof; ++b) {
5183         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5184         const PetscInt ind    = indperm ? indperm[preind] : preind;
5185 
5186         indices[ind] = off+foff+b;
5187       }
5188     } else {
5189       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5190       for (b = 0; b < fdof; ++b) {
5191         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5192         const PetscInt ind    = indperm ? indperm[preind] : preind;
5193 
5194         if ((cind < cfdof) && (b == fcdofs[cind])) {
5195           indices[ind] = -(off+foff+b+1);
5196           ++cind;
5197         } else {
5198           indices[ind] = off+foff+b-cind;
5199         }
5200       }
5201     }
5202     foff     += (setBC ? fdof : (fdof - cfdof));
5203     foffs[f] += fdof;
5204   }
5205   PetscFunctionReturn(0);
5206 }
5207 
5208 /*
5209   This version believes the globalSection offsets for each field, rather than just the point offset
5210 
5211  . foffs - The offset into 'indices' for each field, since it is segregated by field
5212 */
5213 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5214 {
5215   PetscInt       numFields, foff, f;
5216   PetscErrorCode ierr;
5217 
5218   PetscFunctionBegin;
5219   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5220   for (f = 0; f < numFields; ++f) {
5221     PetscInt        fdof, cfdof;
5222     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5223     PetscInt        cind = 0, b;
5224     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5225 
5226     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5227     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5228     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5229     if (!cfdof || setBC) {
5230       for (b = 0; b < fdof; ++b) {
5231         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5232         const PetscInt ind    = indperm ? indperm[preind] : preind;
5233 
5234         indices[ind] = foff+b;
5235       }
5236     } else {
5237       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5238       for (b = 0; b < fdof; ++b) {
5239         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5240         const PetscInt ind    = indperm ? indperm[preind] : preind;
5241 
5242         if ((cind < cfdof) && (b == fcdofs[cind])) {
5243           indices[ind] = -(foff+b+1);
5244           ++cind;
5245         } else {
5246           indices[ind] = foff+b-cind;
5247         }
5248       }
5249     }
5250     foffs[f] += fdof;
5251   }
5252   PetscFunctionReturn(0);
5253 }
5254 
5255 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5256 {
5257   Mat             cMat;
5258   PetscSection    aSec, cSec;
5259   IS              aIS;
5260   PetscInt        aStart = -1, aEnd = -1;
5261   const PetscInt  *anchors;
5262   PetscInt        numFields, f, p, q, newP = 0;
5263   PetscInt        newNumPoints = 0, newNumIndices = 0;
5264   PetscInt        *newPoints, *indices, *newIndices;
5265   PetscInt        maxAnchor, maxDof;
5266   PetscInt        newOffsets[32];
5267   PetscInt        *pointMatOffsets[32];
5268   PetscInt        *newPointOffsets[32];
5269   PetscScalar     *pointMat[32];
5270   PetscScalar     *newValues=NULL,*tmpValues;
5271   PetscBool       anyConstrained = PETSC_FALSE;
5272   PetscErrorCode  ierr;
5273 
5274   PetscFunctionBegin;
5275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5276   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5277   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5278 
5279   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5280   /* if there are point-to-point constraints */
5281   if (aSec) {
5282     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5283     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5284     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5285     /* figure out how many points are going to be in the new element matrix
5286      * (we allow double counting, because it's all just going to be summed
5287      * into the global matrix anyway) */
5288     for (p = 0; p < 2*numPoints; p+=2) {
5289       PetscInt b    = points[p];
5290       PetscInt bDof = 0, bSecDof;
5291 
5292       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5293       if (!bSecDof) {
5294         continue;
5295       }
5296       if (b >= aStart && b < aEnd) {
5297         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5298       }
5299       if (bDof) {
5300         /* this point is constrained */
5301         /* it is going to be replaced by its anchors */
5302         PetscInt bOff, q;
5303 
5304         anyConstrained = PETSC_TRUE;
5305         newNumPoints  += bDof;
5306         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5307         for (q = 0; q < bDof; q++) {
5308           PetscInt a = anchors[bOff + q];
5309           PetscInt aDof;
5310 
5311           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5312           newNumIndices += aDof;
5313           for (f = 0; f < numFields; ++f) {
5314             PetscInt fDof;
5315 
5316             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5317             newOffsets[f+1] += fDof;
5318           }
5319         }
5320       }
5321       else {
5322         /* this point is not constrained */
5323         newNumPoints++;
5324         newNumIndices += bSecDof;
5325         for (f = 0; f < numFields; ++f) {
5326           PetscInt fDof;
5327 
5328           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5329           newOffsets[f+1] += fDof;
5330         }
5331       }
5332     }
5333   }
5334   if (!anyConstrained) {
5335     if (outNumPoints)  *outNumPoints  = 0;
5336     if (outNumIndices) *outNumIndices = 0;
5337     if (outPoints)     *outPoints     = NULL;
5338     if (outValues)     *outValues     = NULL;
5339     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5340     PetscFunctionReturn(0);
5341   }
5342 
5343   if (outNumPoints)  *outNumPoints  = newNumPoints;
5344   if (outNumIndices) *outNumIndices = newNumIndices;
5345 
5346   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5347 
5348   if (!outPoints && !outValues) {
5349     if (offsets) {
5350       for (f = 0; f <= numFields; f++) {
5351         offsets[f] = newOffsets[f];
5352       }
5353     }
5354     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5355     PetscFunctionReturn(0);
5356   }
5357 
5358   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5359 
5360   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5361 
5362   /* workspaces */
5363   if (numFields) {
5364     for (f = 0; f < numFields; f++) {
5365       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5366       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5367     }
5368   }
5369   else {
5370     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5371     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5372   }
5373 
5374   /* get workspaces for the point-to-point matrices */
5375   if (numFields) {
5376     PetscInt totalOffset, totalMatOffset;
5377 
5378     for (p = 0; p < numPoints; p++) {
5379       PetscInt b    = points[2*p];
5380       PetscInt bDof = 0, bSecDof;
5381 
5382       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5383       if (!bSecDof) {
5384         for (f = 0; f < numFields; f++) {
5385           newPointOffsets[f][p + 1] = 0;
5386           pointMatOffsets[f][p + 1] = 0;
5387         }
5388         continue;
5389       }
5390       if (b >= aStart && b < aEnd) {
5391         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5392       }
5393       if (bDof) {
5394         for (f = 0; f < numFields; f++) {
5395           PetscInt fDof, q, bOff, allFDof = 0;
5396 
5397           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5398           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5399           for (q = 0; q < bDof; q++) {
5400             PetscInt a = anchors[bOff + q];
5401             PetscInt aFDof;
5402 
5403             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5404             allFDof += aFDof;
5405           }
5406           newPointOffsets[f][p+1] = allFDof;
5407           pointMatOffsets[f][p+1] = fDof * allFDof;
5408         }
5409       }
5410       else {
5411         for (f = 0; f < numFields; f++) {
5412           PetscInt fDof;
5413 
5414           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5415           newPointOffsets[f][p+1] = fDof;
5416           pointMatOffsets[f][p+1] = 0;
5417         }
5418       }
5419     }
5420     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5421       newPointOffsets[f][0] = totalOffset;
5422       pointMatOffsets[f][0] = totalMatOffset;
5423       for (p = 0; p < numPoints; p++) {
5424         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5425         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5426       }
5427       totalOffset    = newPointOffsets[f][numPoints];
5428       totalMatOffset = pointMatOffsets[f][numPoints];
5429       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5430     }
5431   }
5432   else {
5433     for (p = 0; p < numPoints; p++) {
5434       PetscInt b    = points[2*p];
5435       PetscInt bDof = 0, bSecDof;
5436 
5437       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5438       if (!bSecDof) {
5439         newPointOffsets[0][p + 1] = 0;
5440         pointMatOffsets[0][p + 1] = 0;
5441         continue;
5442       }
5443       if (b >= aStart && b < aEnd) {
5444         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5445       }
5446       if (bDof) {
5447         PetscInt bOff, q, allDof = 0;
5448 
5449         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5450         for (q = 0; q < bDof; q++) {
5451           PetscInt a = anchors[bOff + q], aDof;
5452 
5453           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5454           allDof += aDof;
5455         }
5456         newPointOffsets[0][p+1] = allDof;
5457         pointMatOffsets[0][p+1] = bSecDof * allDof;
5458       }
5459       else {
5460         newPointOffsets[0][p+1] = bSecDof;
5461         pointMatOffsets[0][p+1] = 0;
5462       }
5463     }
5464     newPointOffsets[0][0] = 0;
5465     pointMatOffsets[0][0] = 0;
5466     for (p = 0; p < numPoints; p++) {
5467       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5468       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5469     }
5470     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5471   }
5472 
5473   /* output arrays */
5474   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5475 
5476   /* get the point-to-point matrices; construct newPoints */
5477   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5478   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5479   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5480   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5481   if (numFields) {
5482     for (p = 0, newP = 0; p < numPoints; p++) {
5483       PetscInt b    = points[2*p];
5484       PetscInt o    = points[2*p+1];
5485       PetscInt bDof = 0, bSecDof;
5486 
5487       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5488       if (!bSecDof) {
5489         continue;
5490       }
5491       if (b >= aStart && b < aEnd) {
5492         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5493       }
5494       if (bDof) {
5495         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5496 
5497         fStart[0] = 0;
5498         fEnd[0]   = 0;
5499         for (f = 0; f < numFields; f++) {
5500           PetscInt fDof;
5501 
5502           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5503           fStart[f+1] = fStart[f] + fDof;
5504           fEnd[f+1]   = fStart[f+1];
5505         }
5506         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5507         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5508 
5509         fAnchorStart[0] = 0;
5510         fAnchorEnd[0]   = 0;
5511         for (f = 0; f < numFields; f++) {
5512           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5513 
5514           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5515           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5516         }
5517         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5518         for (q = 0; q < bDof; q++) {
5519           PetscInt a = anchors[bOff + q], aOff;
5520 
5521           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5522           newPoints[2*(newP + q)]     = a;
5523           newPoints[2*(newP + q) + 1] = 0;
5524           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5525           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5526         }
5527         newP += bDof;
5528 
5529         if (outValues) {
5530           /* get the point-to-point submatrix */
5531           for (f = 0; f < numFields; f++) {
5532             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5533           }
5534         }
5535       }
5536       else {
5537         newPoints[2 * newP]     = b;
5538         newPoints[2 * newP + 1] = o;
5539         newP++;
5540       }
5541     }
5542   } else {
5543     for (p = 0; p < numPoints; p++) {
5544       PetscInt b    = points[2*p];
5545       PetscInt o    = points[2*p+1];
5546       PetscInt bDof = 0, bSecDof;
5547 
5548       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5549       if (!bSecDof) {
5550         continue;
5551       }
5552       if (b >= aStart && b < aEnd) {
5553         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5554       }
5555       if (bDof) {
5556         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5557 
5558         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5559         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5560 
5561         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5562         for (q = 0; q < bDof; q++) {
5563           PetscInt a = anchors[bOff + q], aOff;
5564 
5565           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5566 
5567           newPoints[2*(newP + q)]     = a;
5568           newPoints[2*(newP + q) + 1] = 0;
5569           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5570           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5571         }
5572         newP += bDof;
5573 
5574         /* get the point-to-point submatrix */
5575         if (outValues) {
5576           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5577         }
5578       }
5579       else {
5580         newPoints[2 * newP]     = b;
5581         newPoints[2 * newP + 1] = o;
5582         newP++;
5583       }
5584     }
5585   }
5586 
5587   if (outValues) {
5588     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5589     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5590     /* multiply constraints on the right */
5591     if (numFields) {
5592       for (f = 0; f < numFields; f++) {
5593         PetscInt oldOff = offsets[f];
5594 
5595         for (p = 0; p < numPoints; p++) {
5596           PetscInt cStart = newPointOffsets[f][p];
5597           PetscInt b      = points[2 * p];
5598           PetscInt c, r, k;
5599           PetscInt dof;
5600 
5601           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5602           if (!dof) {
5603             continue;
5604           }
5605           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5606             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5607             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5608 
5609             for (r = 0; r < numIndices; r++) {
5610               for (c = 0; c < nCols; c++) {
5611                 for (k = 0; k < dof; k++) {
5612                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5613                 }
5614               }
5615             }
5616           }
5617           else {
5618             /* copy this column as is */
5619             for (r = 0; r < numIndices; r++) {
5620               for (c = 0; c < dof; c++) {
5621                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5622               }
5623             }
5624           }
5625           oldOff += dof;
5626         }
5627       }
5628     }
5629     else {
5630       PetscInt oldOff = 0;
5631       for (p = 0; p < numPoints; p++) {
5632         PetscInt cStart = newPointOffsets[0][p];
5633         PetscInt b      = points[2 * p];
5634         PetscInt c, r, k;
5635         PetscInt dof;
5636 
5637         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5638         if (!dof) {
5639           continue;
5640         }
5641         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5642           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5643           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5644 
5645           for (r = 0; r < numIndices; r++) {
5646             for (c = 0; c < nCols; c++) {
5647               for (k = 0; k < dof; k++) {
5648                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5649               }
5650             }
5651           }
5652         }
5653         else {
5654           /* copy this column as is */
5655           for (r = 0; r < numIndices; r++) {
5656             for (c = 0; c < dof; c++) {
5657               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5658             }
5659           }
5660         }
5661         oldOff += dof;
5662       }
5663     }
5664 
5665     if (multiplyLeft) {
5666       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5667       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5668       /* multiply constraints transpose on the left */
5669       if (numFields) {
5670         for (f = 0; f < numFields; f++) {
5671           PetscInt oldOff = offsets[f];
5672 
5673           for (p = 0; p < numPoints; p++) {
5674             PetscInt rStart = newPointOffsets[f][p];
5675             PetscInt b      = points[2 * p];
5676             PetscInt c, r, k;
5677             PetscInt dof;
5678 
5679             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5680             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5681               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5682               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5683 
5684               for (r = 0; r < nRows; r++) {
5685                 for (c = 0; c < newNumIndices; c++) {
5686                   for (k = 0; k < dof; k++) {
5687                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5688                   }
5689                 }
5690               }
5691             }
5692             else {
5693               /* copy this row as is */
5694               for (r = 0; r < dof; r++) {
5695                 for (c = 0; c < newNumIndices; c++) {
5696                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5697                 }
5698               }
5699             }
5700             oldOff += dof;
5701           }
5702         }
5703       }
5704       else {
5705         PetscInt oldOff = 0;
5706 
5707         for (p = 0; p < numPoints; p++) {
5708           PetscInt rStart = newPointOffsets[0][p];
5709           PetscInt b      = points[2 * p];
5710           PetscInt c, r, k;
5711           PetscInt dof;
5712 
5713           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5714           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5715             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5716             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5717 
5718             for (r = 0; r < nRows; r++) {
5719               for (c = 0; c < newNumIndices; c++) {
5720                 for (k = 0; k < dof; k++) {
5721                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5722                 }
5723               }
5724             }
5725           }
5726           else {
5727             /* copy this row as is */
5728             for (r = 0; r < dof; r++) {
5729               for (c = 0; c < newNumIndices; c++) {
5730                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5731               }
5732             }
5733           }
5734           oldOff += dof;
5735         }
5736       }
5737 
5738       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5739     }
5740     else {
5741       newValues = tmpValues;
5742     }
5743   }
5744 
5745   /* clean up */
5746   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5747   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5748 
5749   if (numFields) {
5750     for (f = 0; f < numFields; f++) {
5751       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5752       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5753       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5754     }
5755   }
5756   else {
5757     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5758     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5759     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5760   }
5761   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5762 
5763   /* output */
5764   if (outPoints) {
5765     *outPoints = newPoints;
5766   }
5767   else {
5768     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5769   }
5770   if (outValues) {
5771     *outValues = newValues;
5772   }
5773   for (f = 0; f <= numFields; f++) {
5774     offsets[f] = newOffsets[f];
5775   }
5776   PetscFunctionReturn(0);
5777 }
5778 
5779 /*@C
5780   DMPlexGetClosureIndices - Get the global indices for all local points in the closure of the given point
5781 
5782   Not collective
5783 
5784   Input Parameters:
5785 + dm - The DM
5786 . section - The section describing the local layout
5787 . globalSection - The section describing the parallel layout
5788 - point - The mesh point
5789 
5790   Output parameters:
5791 + numIndices - The number of indices
5792 . indices - The indices
5793 - outOffsets - Field offset if not NULL
5794 
5795   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5796 
5797   Level: advanced
5798 
5799 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5800 @*/
5801 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5802 {
5803   PetscSection    clSection;
5804   IS              clPoints;
5805   const PetscInt *clp, *clperm;
5806   const PetscInt  **perms[32] = {NULL};
5807   PetscInt       *points = NULL, *pointsNew;
5808   PetscInt        numPoints, numPointsNew;
5809   PetscInt        offsets[32];
5810   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5811   PetscErrorCode  ierr;
5812 
5813   PetscFunctionBegin;
5814   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5815   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5816   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5817   if (numIndices) PetscValidPointer(numIndices, 4);
5818   PetscValidPointer(indices, 5);
5819   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5820   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5821   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5822   /* Get points in closure */
5823   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5824   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5825   /* Get number of indices and indices per field */
5826   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5827     PetscInt dof, fdof;
5828 
5829     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5830     for (f = 0; f < Nf; ++f) {
5831       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5832       offsets[f+1] += fdof;
5833     }
5834     Nind += dof;
5835   }
5836   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5837   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5838   if (!Nf) offsets[1] = Nind;
5839   /* Get dual space symmetries */
5840   for (f = 0; f < PetscMax(1,Nf); f++) {
5841     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5842     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5843   }
5844   /* Correct for hanging node constraints */
5845   {
5846     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5847     if (numPointsNew) {
5848       for (f = 0; f < PetscMax(1,Nf); f++) {
5849         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5850         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5851       }
5852       for (f = 0; f < PetscMax(1,Nf); f++) {
5853         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5854         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5855       }
5856       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5857       numPoints = numPointsNew;
5858       Nind      = NindNew;
5859       points    = pointsNew;
5860     }
5861   }
5862   /* Calculate indices */
5863   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5864   if (Nf) {
5865     if (outOffsets) {
5866       PetscInt f;
5867 
5868       for (f = 0; f <= Nf; f++) {
5869         outOffsets[f] = offsets[f];
5870       }
5871     }
5872     for (p = 0; p < numPoints; p++) {
5873       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5874       ierr = DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);CHKERRQ(ierr);
5875     }
5876   } else {
5877     for (p = 0, off = 0; p < numPoints; p++) {
5878       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5879 
5880       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5881       ierr = DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);CHKERRQ(ierr);
5882     }
5883   }
5884   /* Cleanup points */
5885   for (f = 0; f < PetscMax(1,Nf); f++) {
5886     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5887     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5888   }
5889   if (numPointsNew) {
5890     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5891   } else {
5892     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5893   }
5894   if (numIndices) *numIndices = Nind;
5895   PetscFunctionReturn(0);
5896 }
5897 
5898 /*@C
5899   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5900 
5901   Not collective
5902 
5903   Input Parameters:
5904 + dm - The DM
5905 . section - The section describing the layout in v, or NULL to use the default section
5906 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5907 . point - The mesh point
5908 . numIndices - The number of indices
5909 . indices - The indices
5910 - outOffsets - Field offset if not NULL
5911 
5912   Level: advanced
5913 
5914 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5915 @*/
5916 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5917 {
5918   PetscErrorCode ierr;
5919 
5920   PetscFunctionBegin;
5921   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5922   PetscValidPointer(indices, 5);
5923   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5924   PetscFunctionReturn(0);
5925 }
5926 
5927 /*@C
5928   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5929 
5930   Not collective
5931 
5932   Input Parameters:
5933 + dm - The DM
5934 . section - The section describing the layout in v, or NULL to use the default section
5935 . globalSection - The section describing the layout in v, or NULL to use the default global section
5936 . A - The matrix
5937 . point - The point in the DM
5938 . values - The array of values
5939 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5940 
5941   Fortran Notes:
5942   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5943 
5944   Level: intermediate
5945 
5946 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5947 @*/
5948 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5949 {
5950   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5951   PetscSection        clSection;
5952   IS                  clPoints;
5953   PetscInt           *points = NULL, *newPoints;
5954   const PetscInt     *clp, *clperm;
5955   PetscInt           *indices;
5956   PetscInt            offsets[32];
5957   const PetscInt    **perms[32] = {NULL};
5958   const PetscScalar **flips[32] = {NULL};
5959   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5960   PetscScalar        *valCopy = NULL;
5961   PetscScalar        *newValues;
5962   PetscErrorCode      ierr;
5963 
5964   PetscFunctionBegin;
5965   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5966   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5967   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5968   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5969   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5970   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5971   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5972   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5973   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5974   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5975   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5976   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5977     PetscInt fdof;
5978 
5979     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5980     for (f = 0; f < numFields; ++f) {
5981       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5982       offsets[f+1] += fdof;
5983     }
5984     numIndices += dof;
5985   }
5986   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5987 
5988   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5989   /* Get symmetries */
5990   for (f = 0; f < PetscMax(1,numFields); f++) {
5991     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5992     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5993     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5994       PetscInt foffset = offsets[f];
5995 
5996       for (p = 0; p < numPoints; p++) {
5997         PetscInt point          = points[2*p], fdof;
5998         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5999 
6000         if (!numFields) {
6001           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
6002         } else {
6003           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
6004         }
6005         if (flip) {
6006           PetscInt i, j, k;
6007 
6008           if (!valCopy) {
6009             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6010             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6011             values = valCopy;
6012           }
6013           for (i = 0; i < fdof; i++) {
6014             PetscScalar fval = flip[i];
6015 
6016             for (k = 0; k < numIndices; k++) {
6017               valCopy[numIndices * (foffset + i) + k] *= fval;
6018               valCopy[numIndices * k + (foffset + i)] *= fval;
6019             }
6020           }
6021         }
6022         foffset += fdof;
6023       }
6024     }
6025   }
6026   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6027   if (newNumPoints) {
6028     if (valCopy) {
6029       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6030     }
6031     for (f = 0; f < PetscMax(1,numFields); f++) {
6032       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6033       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6034     }
6035     for (f = 0; f < PetscMax(1,numFields); f++) {
6036       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6037       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6038     }
6039     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6040     numPoints  = newNumPoints;
6041     numIndices = newNumIndices;
6042     points     = newPoints;
6043     values     = newValues;
6044   }
6045   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6046   if (numFields) {
6047     PetscBool useFieldOffsets;
6048 
6049     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6050     if (useFieldOffsets) {
6051       for (p = 0; p < numPoints; p++) {
6052         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
6053       }
6054     } else {
6055       for (p = 0; p < numPoints; p++) {
6056         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6057         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
6058       }
6059     }
6060   } else {
6061     for (p = 0, off = 0; p < numPoints; p++) {
6062       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6063       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6064       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
6065     }
6066   }
6067   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6068   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
6069   if (mesh->printFEM > 1) {
6070     PetscInt i;
6071     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6072     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6073     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6074   }
6075   if (ierr) {
6076     PetscMPIInt    rank;
6077     PetscErrorCode ierr2;
6078 
6079     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6080     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6081     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6082     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6083     CHKERRQ(ierr);
6084   }
6085   for (f = 0; f < PetscMax(1,numFields); f++) {
6086     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6087     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6088   }
6089   if (newNumPoints) {
6090     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6091     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6092   }
6093   else {
6094     if (valCopy) {
6095       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6096     }
6097     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6098   }
6099   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6100   PetscFunctionReturn(0);
6101 }
6102 
6103 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6104 {
6105   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6106   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6107   PetscInt       *cpoints = NULL;
6108   PetscInt       *findices, *cindices;
6109   const PetscInt *fclperm, *cclperm;
6110   PetscInt        foffsets[32], coffsets[32];
6111   CellRefiner     cellRefiner;
6112   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6113   PetscErrorCode  ierr;
6114 
6115   PetscFunctionBegin;
6116   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6117   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6118   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6119   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6120   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6121   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6122   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6123   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6124   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6125   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6126   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6127   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6128   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6129   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6130   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6131   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6132   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6133   /* Column indices */
6134   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6135   maxFPoints = numCPoints;
6136   /* Compress out points not in the section */
6137   /*   TODO: Squeeze out points with 0 dof as well */
6138   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6139   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6140     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6141       cpoints[q*2]   = cpoints[p];
6142       cpoints[q*2+1] = cpoints[p+1];
6143       ++q;
6144     }
6145   }
6146   numCPoints = q;
6147   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6148     PetscInt fdof;
6149 
6150     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6151     if (!dof) continue;
6152     for (f = 0; f < numFields; ++f) {
6153       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6154       coffsets[f+1] += fdof;
6155     }
6156     numCIndices += dof;
6157   }
6158   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6159   /* Row indices */
6160   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6161   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6162   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6163   for (r = 0, q = 0; r < numSubcells; ++r) {
6164     /* TODO Map from coarse to fine cells */
6165     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6166     /* Compress out points not in the section */
6167     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6168     for (p = 0; p < numFPoints*2; p += 2) {
6169       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6170         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6171         if (!dof) continue;
6172         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6173         if (s < q) continue;
6174         ftotpoints[q*2]   = fpoints[p];
6175         ftotpoints[q*2+1] = fpoints[p+1];
6176         ++q;
6177       }
6178     }
6179     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6180   }
6181   numFPoints = q;
6182   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6183     PetscInt fdof;
6184 
6185     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6186     if (!dof) continue;
6187     for (f = 0; f < numFields; ++f) {
6188       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6189       foffsets[f+1] += fdof;
6190     }
6191     numFIndices += dof;
6192   }
6193   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6194 
6195   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6196   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6197   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6198   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6199   if (numFields) {
6200     const PetscInt **permsF[32] = {NULL};
6201     const PetscInt **permsC[32] = {NULL};
6202 
6203     for (f = 0; f < numFields; f++) {
6204       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6205       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6206     }
6207     for (p = 0; p < numFPoints; p++) {
6208       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6209       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6210     }
6211     for (p = 0; p < numCPoints; p++) {
6212       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6213       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6214     }
6215     for (f = 0; f < numFields; f++) {
6216       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6217       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6218     }
6219   } else {
6220     const PetscInt **permsF = NULL;
6221     const PetscInt **permsC = NULL;
6222 
6223     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6224     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6225     for (p = 0, off = 0; p < numFPoints; p++) {
6226       const PetscInt *perm = permsF ? permsF[p] : NULL;
6227 
6228       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6229       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6230     }
6231     for (p = 0, off = 0; p < numCPoints; p++) {
6232       const PetscInt *perm = permsC ? permsC[p] : NULL;
6233 
6234       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6235       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6236     }
6237     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6238     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6239   }
6240   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6241   /* TODO: flips */
6242   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6243   if (ierr) {
6244     PetscMPIInt    rank;
6245     PetscErrorCode ierr2;
6246 
6247     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6248     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6249     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6250     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6251     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6252     CHKERRQ(ierr);
6253   }
6254   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6255   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6256   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6257   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6258   PetscFunctionReturn(0);
6259 }
6260 
6261 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6262 {
6263   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6264   PetscInt      *cpoints = NULL;
6265   PetscInt       foffsets[32], coffsets[32];
6266   const PetscInt *fclperm, *cclperm;
6267   CellRefiner    cellRefiner;
6268   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6269   PetscErrorCode ierr;
6270 
6271   PetscFunctionBegin;
6272   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6273   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6274   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6275   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6276   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6277   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6278   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6279   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6280   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6281   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6282   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6283   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6284   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6285   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6286   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6287   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6288   /* Column indices */
6289   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6290   maxFPoints = numCPoints;
6291   /* Compress out points not in the section */
6292   /*   TODO: Squeeze out points with 0 dof as well */
6293   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6294   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6295     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6296       cpoints[q*2]   = cpoints[p];
6297       cpoints[q*2+1] = cpoints[p+1];
6298       ++q;
6299     }
6300   }
6301   numCPoints = q;
6302   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6303     PetscInt fdof;
6304 
6305     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6306     if (!dof) continue;
6307     for (f = 0; f < numFields; ++f) {
6308       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6309       coffsets[f+1] += fdof;
6310     }
6311     numCIndices += dof;
6312   }
6313   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6314   /* Row indices */
6315   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6316   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6317   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6318   for (r = 0, q = 0; r < numSubcells; ++r) {
6319     /* TODO Map from coarse to fine cells */
6320     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6321     /* Compress out points not in the section */
6322     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6323     for (p = 0; p < numFPoints*2; p += 2) {
6324       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6325         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6326         if (!dof) continue;
6327         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6328         if (s < q) continue;
6329         ftotpoints[q*2]   = fpoints[p];
6330         ftotpoints[q*2+1] = fpoints[p+1];
6331         ++q;
6332       }
6333     }
6334     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6335   }
6336   numFPoints = q;
6337   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6338     PetscInt fdof;
6339 
6340     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6341     if (!dof) continue;
6342     for (f = 0; f < numFields; ++f) {
6343       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6344       foffsets[f+1] += fdof;
6345     }
6346     numFIndices += dof;
6347   }
6348   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6349 
6350   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6351   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6352   if (numFields) {
6353     const PetscInt **permsF[32] = {NULL};
6354     const PetscInt **permsC[32] = {NULL};
6355 
6356     for (f = 0; f < numFields; f++) {
6357       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6358       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6359     }
6360     for (p = 0; p < numFPoints; p++) {
6361       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6362       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6363     }
6364     for (p = 0; p < numCPoints; p++) {
6365       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6366       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6367     }
6368     for (f = 0; f < numFields; f++) {
6369       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6370       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6371     }
6372   } else {
6373     const PetscInt **permsF = NULL;
6374     const PetscInt **permsC = NULL;
6375 
6376     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6377     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6378     for (p = 0, off = 0; p < numFPoints; p++) {
6379       const PetscInt *perm = permsF ? permsF[p] : NULL;
6380 
6381       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6382       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6383     }
6384     for (p = 0, off = 0; p < numCPoints; p++) {
6385       const PetscInt *perm = permsC ? permsC[p] : NULL;
6386 
6387       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6388       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6389     }
6390     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6391     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6392   }
6393   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6394   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6395   PetscFunctionReturn(0);
6396 }
6397 
6398 /*@
6399   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6400 
6401   Input Parameter:
6402 . dm - The DMPlex object
6403 
6404   Output Parameters:
6405 + cMax - The first hybrid cell
6406 . fMax - The first hybrid face
6407 . eMax - The first hybrid edge
6408 - vMax - The first hybrid vertex
6409 
6410   Level: developer
6411 
6412 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6413 @*/
6414 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6415 {
6416   DM_Plex       *mesh = (DM_Plex*) dm->data;
6417   PetscInt       dim;
6418   PetscErrorCode ierr;
6419 
6420   PetscFunctionBegin;
6421   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6422   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6423   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6424   if (cMax) *cMax = mesh->hybridPointMax[dim];
6425   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6426   if (eMax) *eMax = mesh->hybridPointMax[1];
6427   if (vMax) *vMax = mesh->hybridPointMax[0];
6428   PetscFunctionReturn(0);
6429 }
6430 
6431 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6432 {
6433   IS             is, his;
6434   PetscInt       first = 0, stride;
6435   PetscBool      isStride;
6436   PetscErrorCode ierr;
6437 
6438   PetscFunctionBegin;
6439   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6440   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6441   if (isStride) {
6442     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6443   }
6444   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6445   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6446   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6447   ierr = ISDestroy(&his);CHKERRQ(ierr);
6448   ierr = ISDestroy(&is);CHKERRQ(ierr);
6449   PetscFunctionReturn(0);
6450 }
6451 
6452 /*@
6453   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6454 
6455   Input Parameters:
6456 + dm   - The DMPlex object
6457 . cMax - The first hybrid cell
6458 . fMax - The first hybrid face
6459 . eMax - The first hybrid edge
6460 - vMax - The first hybrid vertex
6461 
6462   Level: developer
6463 
6464 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6465 @*/
6466 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6467 {
6468   DM_Plex       *mesh = (DM_Plex*) dm->data;
6469   PetscInt       dim;
6470   PetscErrorCode ierr;
6471 
6472   PetscFunctionBegin;
6473   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6474   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6475   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6476   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6477   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6478   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6479   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6480   PetscFunctionReturn(0);
6481 }
6482 
6483 /*@C
6484   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6485 
6486   Input Parameter:
6487 . dm   - The DMPlex object
6488 
6489   Output Parameter:
6490 . cellHeight - The height of a cell
6491 
6492   Level: developer
6493 
6494 .seealso DMPlexSetVTKCellHeight()
6495 @*/
6496 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6497 {
6498   DM_Plex *mesh = (DM_Plex*) dm->data;
6499 
6500   PetscFunctionBegin;
6501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6502   PetscValidPointer(cellHeight, 2);
6503   *cellHeight = mesh->vtkCellHeight;
6504   PetscFunctionReturn(0);
6505 }
6506 
6507 /*@C
6508   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6509 
6510   Input Parameters:
6511 + dm   - The DMPlex object
6512 - cellHeight - The height of a cell
6513 
6514   Level: developer
6515 
6516 .seealso DMPlexGetVTKCellHeight()
6517 @*/
6518 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6519 {
6520   DM_Plex *mesh = (DM_Plex*) dm->data;
6521 
6522   PetscFunctionBegin;
6523   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6524   mesh->vtkCellHeight = cellHeight;
6525   PetscFunctionReturn(0);
6526 }
6527 
6528 /* We can easily have a form that takes an IS instead */
6529 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6530 {
6531   PetscSection   section, globalSection;
6532   PetscInt      *numbers, p;
6533   PetscErrorCode ierr;
6534 
6535   PetscFunctionBegin;
6536   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6537   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6538   for (p = pStart; p < pEnd; ++p) {
6539     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6540   }
6541   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6542   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6543   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6544   for (p = pStart; p < pEnd; ++p) {
6545     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6546     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6547     else                       numbers[p-pStart] += shift;
6548   }
6549   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6550   if (globalSize) {
6551     PetscLayout layout;
6552     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6553     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6554     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6555   }
6556   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6557   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6558   PetscFunctionReturn(0);
6559 }
6560 
6561 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6562 {
6563   PetscInt       cellHeight, cStart, cEnd, cMax;
6564   PetscErrorCode ierr;
6565 
6566   PetscFunctionBegin;
6567   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6568   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6569   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6570   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6571   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6572   PetscFunctionReturn(0);
6573 }
6574 
6575 /*@
6576   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6577 
6578   Input Parameter:
6579 . dm   - The DMPlex object
6580 
6581   Output Parameter:
6582 . globalCellNumbers - Global cell numbers for all cells on this process
6583 
6584   Level: developer
6585 
6586 .seealso DMPlexGetVertexNumbering()
6587 @*/
6588 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6589 {
6590   DM_Plex       *mesh = (DM_Plex*) dm->data;
6591   PetscErrorCode ierr;
6592 
6593   PetscFunctionBegin;
6594   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6595   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6596   *globalCellNumbers = mesh->globalCellNumbers;
6597   PetscFunctionReturn(0);
6598 }
6599 
6600 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6601 {
6602   PetscInt       vStart, vEnd, vMax;
6603   PetscErrorCode ierr;
6604 
6605   PetscFunctionBegin;
6606   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6607   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6608   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6609   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6610   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6611   PetscFunctionReturn(0);
6612 }
6613 
6614 /*@
6615   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
6616 
6617   Input Parameter:
6618 . dm   - The DMPlex object
6619 
6620   Output Parameter:
6621 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6622 
6623   Level: developer
6624 
6625 .seealso DMPlexGetCellNumbering()
6626 @*/
6627 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6628 {
6629   DM_Plex       *mesh = (DM_Plex*) dm->data;
6630   PetscErrorCode ierr;
6631 
6632   PetscFunctionBegin;
6633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6634   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6635   *globalVertexNumbers = mesh->globalVertexNumbers;
6636   PetscFunctionReturn(0);
6637 }
6638 
6639 /*@
6640   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6641 
6642   Input Parameter:
6643 . dm   - The DMPlex object
6644 
6645   Output Parameter:
6646 . globalPointNumbers - Global numbers for all points on this process
6647 
6648   Level: developer
6649 
6650 .seealso DMPlexGetCellNumbering()
6651 @*/
6652 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6653 {
6654   IS             nums[4];
6655   PetscInt       depths[4], gdepths[4], starts[4];
6656   PetscInt       depth, d, shift = 0;
6657   PetscErrorCode ierr;
6658 
6659   PetscFunctionBegin;
6660   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6661   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6662   /* For unstratified meshes use dim instead of depth */
6663   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6664   for (d = 0; d <= depth; ++d) {
6665     PetscInt end;
6666 
6667     depths[d] = depth-d;
6668     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6669     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6670   }
6671   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6672   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6673   for (d = 0; d <= depth; ++d) {
6674     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6675   }
6676   for (d = 0; d <= depth; ++d) {
6677     PetscInt pStart, pEnd, gsize;
6678 
6679     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6680     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6681     shift += gsize;
6682   }
6683   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6684   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6685   PetscFunctionReturn(0);
6686 }
6687 
6688 
6689 /*@
6690   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6691 
6692   Input Parameter:
6693 . dm - The DMPlex object
6694 
6695   Output Parameter:
6696 . ranks - The rank field
6697 
6698   Options Database Keys:
6699 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6700 
6701   Level: intermediate
6702 
6703 .seealso: DMView()
6704 @*/
6705 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6706 {
6707   DM             rdm;
6708   PetscFE        fe;
6709   PetscScalar   *r;
6710   PetscMPIInt    rank;
6711   PetscInt       dim, cStart, cEnd, c;
6712   PetscErrorCode ierr;
6713 
6714   PetscFunctionBeginUser;
6715   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6716   PetscValidPointer(ranks, 2);
6717   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6718   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6719   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6720   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6721   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6722   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6723   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6724   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6725   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6726   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6727   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6728   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6729   for (c = cStart; c < cEnd; ++c) {
6730     PetscScalar *lr;
6731 
6732     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6733     *lr = rank;
6734   }
6735   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6736   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6737   PetscFunctionReturn(0);
6738 }
6739 
6740 /*@
6741   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6742 
6743   Input Parameters:
6744 + dm    - The DMPlex
6745 - label - The DMLabel
6746 
6747   Output Parameter:
6748 . val - The label value field
6749 
6750   Options Database Keys:
6751 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6752 
6753   Level: intermediate
6754 
6755 .seealso: DMView()
6756 @*/
6757 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6758 {
6759   DM             rdm;
6760   PetscFE        fe;
6761   PetscScalar   *v;
6762   PetscInt       dim, cStart, cEnd, c;
6763   PetscErrorCode ierr;
6764 
6765   PetscFunctionBeginUser;
6766   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6767   PetscValidPointer(label, 2);
6768   PetscValidPointer(val, 3);
6769   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6770   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6771   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6772   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6773   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6774   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6775   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6776   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6777   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6778   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6779   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6780   for (c = cStart; c < cEnd; ++c) {
6781     PetscScalar *lv;
6782     PetscInt     cval;
6783 
6784     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6785     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6786     *lv = cval;
6787   }
6788   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6789   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6790   PetscFunctionReturn(0);
6791 }
6792 
6793 /*@
6794   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6795 
6796   Input Parameter:
6797 . dm - The DMPlex object
6798 
6799   Note: This is a useful diagnostic when creating meshes programmatically.
6800 
6801   Level: developer
6802 
6803 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6804 @*/
6805 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6806 {
6807   PetscSection    coneSection, supportSection;
6808   const PetscInt *cone, *support;
6809   PetscInt        coneSize, c, supportSize, s;
6810   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6811   PetscBool       storagecheck = PETSC_TRUE;
6812   PetscErrorCode  ierr;
6813 
6814   PetscFunctionBegin;
6815   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6816   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6817   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6818   /* Check that point p is found in the support of its cone points, and vice versa */
6819   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6820   for (p = pStart; p < pEnd; ++p) {
6821     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6822     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6823     for (c = 0; c < coneSize; ++c) {
6824       PetscBool dup = PETSC_FALSE;
6825       PetscInt  d;
6826       for (d = c-1; d >= 0; --d) {
6827         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6828       }
6829       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6830       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6831       for (s = 0; s < supportSize; ++s) {
6832         if (support[s] == p) break;
6833       }
6834       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6835         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6836         for (s = 0; s < coneSize; ++s) {
6837           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6838         }
6839         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6840         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6841         for (s = 0; s < supportSize; ++s) {
6842           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6843         }
6844         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6845         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6846         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6847       }
6848     }
6849     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6850     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6851     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6852     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6853     for (s = 0; s < supportSize; ++s) {
6854       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6855       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6856       for (c = 0; c < coneSize; ++c) {
6857         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6858         if (cone[c] != pp) { c = 0; break; }
6859         if (cone[c] == p) break;
6860       }
6861       if (c >= coneSize) {
6862         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6863         for (c = 0; c < supportSize; ++c) {
6864           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6865         }
6866         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6867         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6868         for (c = 0; c < coneSize; ++c) {
6869           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6870         }
6871         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6872         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6873       }
6874     }
6875   }
6876   if (storagecheck) {
6877     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6878     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6879     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6880   }
6881   PetscFunctionReturn(0);
6882 }
6883 
6884 /*@
6885   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6886 
6887   Input Parameters:
6888 + dm - The DMPlex object
6889 - cellHeight - Normally 0
6890 
6891   Note: This is a useful diagnostic when creating meshes programmatically.
6892   Currently applicable only to homogeneous simplex or tensor meshes.
6893 
6894   Level: developer
6895 
6896 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6897 @*/
6898 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6899 {
6900   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6901   PetscBool      isSimplex = PETSC_FALSE;
6902   PetscErrorCode ierr;
6903 
6904   PetscFunctionBegin;
6905   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6906   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6907   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6908   if (cStart < cEnd) {
6909     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6910     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6911   }
6912   switch (dim) {
6913   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6914   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6915   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6916   default:
6917     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6918   }
6919   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6920   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6921   cMax = cMax >= 0 ? cMax : cEnd;
6922   for (c = cStart; c < cMax; ++c) {
6923     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6924 
6925     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6926     for (cl = 0; cl < closureSize*2; cl += 2) {
6927       const PetscInt p = closure[cl];
6928       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6929     }
6930     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6931     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6932   }
6933   for (c = cMax; c < cEnd; ++c) {
6934     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6935 
6936     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6937     for (cl = 0; cl < closureSize*2; cl += 2) {
6938       const PetscInt p = closure[cl];
6939       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6940     }
6941     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6942     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6943   }
6944   PetscFunctionReturn(0);
6945 }
6946 
6947 /*@
6948   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6949 
6950   Input Parameters:
6951 + dm - The DMPlex object
6952 - cellHeight - Normally 0
6953 
6954   Note: This is a useful diagnostic when creating meshes programmatically.
6955 
6956   Level: developer
6957 
6958 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6959 @*/
6960 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6961 {
6962   PetscInt       pMax[4];
6963   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6964   PetscErrorCode ierr;
6965 
6966   PetscFunctionBegin;
6967   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6968   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6969   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6970   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6971   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6972   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6973     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6974     for (c = cStart; c < cEnd; ++c) {
6975       const PetscInt *cone, *ornt, *faces;
6976       PetscInt        numFaces, faceSize, coneSize,f;
6977       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6978 
6979       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6980       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6981       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6982       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6983       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6984       for (cl = 0; cl < closureSize*2; cl += 2) {
6985         const PetscInt p = closure[cl];
6986         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6987       }
6988       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6989       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6990       for (f = 0; f < numFaces; ++f) {
6991         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6992 
6993         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6994         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6995           const PetscInt p = fclosure[cl];
6996           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6997         }
6998         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6999         for (v = 0; v < fnumCorners; ++v) {
7000           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
7001         }
7002         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
7003       }
7004       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
7005       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
7006     }
7007   }
7008   PetscFunctionReturn(0);
7009 }
7010 
7011 /*@
7012   DMPlexCheckGeometry - Check the geometry of mesh cells
7013 
7014   Input Parameter:
7015 . dm - The DMPlex object
7016 
7017   Note: This is a useful diagnostic when creating meshes programmatically.
7018 
7019   Level: developer
7020 
7021 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
7022 @*/
7023 PetscErrorCode DMPlexCheckGeometry(DM dm)
7024 {
7025   PetscReal      detJ, J[9], refVol = 1.0;
7026   PetscReal      vol;
7027   PetscInt       dim, depth, d, cStart, cEnd, c, cMax;
7028   PetscErrorCode ierr;
7029 
7030   PetscFunctionBegin;
7031   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7032   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7033   for (d = 0; d < dim; ++d) refVol *= 2.0;
7034   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7035   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7036   cMax = cMax < 0 ? cEnd : cMax;
7037   for (c = cStart; c < cMax; ++c) {
7038     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7039     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7040     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7041     if (depth > 1) {
7042       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7043       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7044       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7045     }
7046   }
7047   PetscFunctionReturn(0);
7048 }
7049 
7050 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
7051 {
7052   PetscInt i,l,n;
7053   const PetscInt *cone;
7054   PetscErrorCode ierr;
7055 
7056   PetscFunctionBegin;
7057   *missingPoint = -1;
7058   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
7059   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
7060   for (i=0; i<n; i++) {
7061     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
7062     if (l < 0) {
7063       *missingPoint = cone[i];
7064       break;
7065     }
7066   }
7067   PetscFunctionReturn(0);
7068 }
7069 
7070 /*@
7071   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
7072 
7073   Input Parameters:
7074 . dm - The DMPlex object
7075 
7076   Notes:
7077   This is mainly intended for debugging/testing purposes.
7078   It currently checks only meshes with no partition overlapping.
7079 
7080   Level: developer
7081 
7082 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7083 @*/
7084 PetscErrorCode DMPlexCheckPointSF(DM dm)
7085 {
7086   PetscSF         sf;
7087   PetscInt        d,depth,i,nleaves,p,plo,phi,missingPoint;
7088   const PetscInt *locals;
7089   PetscErrorCode  ierr;
7090 
7091   PetscFunctionBegin;
7092   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7093   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7094   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
7095   ierr = DMPlexGetOverlap(dm, &d);CHKERRQ(ierr);
7096   if (d) {
7097     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");
7098     PetscFunctionReturn(0);
7099   }
7100   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
7101 
7102   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7103   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
7104   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
7105   for (i=0; i<nleaves; i++) {
7106     p = locals[i];
7107     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
7108   }
7109 
7110   /* 2) if some point is in interface, then all its cone points must be also in interface  */
7111   for (i=0; i<nleaves; i++) {
7112     p = locals[i];
7113     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
7114     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
7115   }
7116   PetscFunctionReturn(0);
7117 }
7118 
7119 typedef struct cell_stats
7120 {
7121   PetscReal min, max, sum, squaresum;
7122   PetscInt  count;
7123 } cell_stats_t;
7124 
7125 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7126 {
7127   PetscInt i, N = *len;
7128 
7129   for (i = 0; i < N; i++) {
7130     cell_stats_t *A = (cell_stats_t *) a;
7131     cell_stats_t *B = (cell_stats_t *) b;
7132 
7133     B->min = PetscMin(A->min,B->min);
7134     B->max = PetscMax(A->max,B->max);
7135     B->sum += A->sum;
7136     B->squaresum += A->squaresum;
7137     B->count += A->count;
7138   }
7139 }
7140 
7141 /*@
7142   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
7143 
7144   Collective on dm
7145 
7146   Input Parameters:
7147 + dm        - The DMPlex object
7148 . output    - If true, statistics will be displayed on stdout
7149 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
7150 
7151   Note: This is mainly intended for debugging/testing purposes.
7152 
7153   Level: developer
7154 
7155 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7156 @*/
7157 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
7158 {
7159   DM             dmCoarse;
7160   cell_stats_t   stats, globalStats;
7161   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7162   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7163   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
7164   PetscInt       cdim, cStart, cEnd, cMax, c, eStart, eEnd, count = 0;
7165   PetscMPIInt    rank,size;
7166   PetscErrorCode ierr;
7167 
7168   PetscFunctionBegin;
7169   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7170   stats.min   = PETSC_MAX_REAL;
7171   stats.max   = PETSC_MIN_REAL;
7172   stats.sum   = stats.squaresum = 0.;
7173   stats.count = 0;
7174 
7175   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
7176   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7177   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
7178   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
7179   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7180   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
7181   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7182   cMax = cMax < 0 ? cEnd : cMax;
7183   for (c = cStart; c < cMax; c++) {
7184     PetscInt  i;
7185     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7186 
7187     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7188     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7189     for (i = 0; i < PetscSqr(cdim); ++i) {
7190       frobJ    += J[i] * J[i];
7191       frobInvJ += invJ[i] * invJ[i];
7192     }
7193     cond2 = frobJ * frobInvJ;
7194     cond  = PetscSqrtReal(cond2);
7195 
7196     stats.min        = PetscMin(stats.min,cond);
7197     stats.max        = PetscMax(stats.max,cond);
7198     stats.sum       += cond;
7199     stats.squaresum += cond2;
7200     stats.count++;
7201     if (output && cond > limit) {
7202       PetscSection coordSection;
7203       Vec          coordsLocal;
7204       PetscScalar *coords = NULL;
7205       PetscInt     Nv, d, clSize, cl, *closure = NULL;
7206 
7207       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
7208       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
7209       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7210       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
7211       for (i = 0; i < Nv/cdim; ++i) {
7212         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
7213         for (d = 0; d < cdim; ++d) {
7214           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
7215           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
7216         }
7217         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
7218       }
7219       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7220       for (cl = 0; cl < clSize*2; cl += 2) {
7221         const PetscInt edge = closure[cl];
7222 
7223         if ((edge >= eStart) && (edge < eEnd)) {
7224           PetscReal len;
7225 
7226           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
7227           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
7228         }
7229       }
7230       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7231       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7232     }
7233   }
7234   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
7235 
7236   if (size > 1) {
7237     PetscMPIInt   blockLengths[2] = {4,1};
7238     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7239     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7240     MPI_Op        statReduce;
7241 
7242     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7243     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7244     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7245     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7246     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7247     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7248   } else {
7249     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7250   }
7251   if (!rank) {
7252     count = globalStats.count;
7253     min   = globalStats.min;
7254     max   = globalStats.max;
7255     mean  = globalStats.sum / globalStats.count;
7256     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7257   }
7258 
7259   if (output) {
7260     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7261   }
7262   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7263 
7264   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7265   if (dmCoarse) {
7266     PetscBool isplex;
7267 
7268     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7269     if (isplex) {
7270       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
7271     }
7272   }
7273   PetscFunctionReturn(0);
7274 }
7275 
7276 /* Pointwise interpolation
7277      Just code FEM for now
7278      u^f = I u^c
7279      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7280      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7281      I_{ij} = psi^f_i phi^c_j
7282 */
7283 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7284 {
7285   PetscSection   gsc, gsf;
7286   PetscInt       m, n;
7287   void          *ctx;
7288   DM             cdm;
7289   PetscBool      regular, ismatis;
7290   PetscErrorCode ierr;
7291 
7292   PetscFunctionBegin;
7293   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7294   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7295   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7296   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7297 
7298   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7299   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7300   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7301   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7302   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7303 
7304   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7305   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7306   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7307   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7308   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7309   if (scaling) {
7310     /* Use naive scaling */
7311     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7312   }
7313   PetscFunctionReturn(0);
7314 }
7315 
7316 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7317 {
7318   PetscErrorCode ierr;
7319   VecScatter     ctx;
7320 
7321   PetscFunctionBegin;
7322   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7323   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7324   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7325   PetscFunctionReturn(0);
7326 }
7327 
7328 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7329 {
7330   PetscSection   gsc, gsf;
7331   PetscInt       m, n;
7332   void          *ctx;
7333   DM             cdm;
7334   PetscBool      regular;
7335   PetscErrorCode ierr;
7336 
7337   PetscFunctionBegin;
7338   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7339   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7340   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7341   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7342 
7343   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7344   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7345   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7346   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7347 
7348   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7349   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7350   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7351   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7352   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7353   PetscFunctionReturn(0);
7354 }
7355 
7356 /*@
7357   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7358 
7359   Input Parameter:
7360 . dm - The DMPlex object
7361 
7362   Output Parameter:
7363 . regular - The flag
7364 
7365   Level: intermediate
7366 
7367 .seealso: DMPlexSetRegularRefinement()
7368 @*/
7369 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7370 {
7371   PetscFunctionBegin;
7372   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7373   PetscValidPointer(regular, 2);
7374   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7375   PetscFunctionReturn(0);
7376 }
7377 
7378 /*@
7379   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7380 
7381   Input Parameters:
7382 + dm - The DMPlex object
7383 - regular - The flag
7384 
7385   Level: intermediate
7386 
7387 .seealso: DMPlexGetRegularRefinement()
7388 @*/
7389 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7390 {
7391   PetscFunctionBegin;
7392   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7393   ((DM_Plex *) dm->data)->regularRefinement = regular;
7394   PetscFunctionReturn(0);
7395 }
7396 
7397 /* anchors */
7398 /*@
7399   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7400   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7401 
7402   not collective
7403 
7404   Input Parameters:
7405 . dm - The DMPlex object
7406 
7407   Output Parameters:
7408 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7409 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7410 
7411 
7412   Level: intermediate
7413 
7414 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7415 @*/
7416 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7417 {
7418   DM_Plex *plex = (DM_Plex *)dm->data;
7419   PetscErrorCode ierr;
7420 
7421   PetscFunctionBegin;
7422   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7423   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7424   if (anchorSection) *anchorSection = plex->anchorSection;
7425   if (anchorIS) *anchorIS = plex->anchorIS;
7426   PetscFunctionReturn(0);
7427 }
7428 
7429 /*@
7430   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7431   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7432   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7433 
7434   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7435   DMGetConstraints() and filling in the entries in the constraint matrix.
7436 
7437   collective on dm
7438 
7439   Input Parameters:
7440 + dm - The DMPlex object
7441 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7442 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7443 
7444   The reference counts of anchorSection and anchorIS are incremented.
7445 
7446   Level: intermediate
7447 
7448 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7449 @*/
7450 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7451 {
7452   DM_Plex        *plex = (DM_Plex *)dm->data;
7453   PetscMPIInt    result;
7454   PetscErrorCode ierr;
7455 
7456   PetscFunctionBegin;
7457   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7458   if (anchorSection) {
7459     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7460     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7461     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7462   }
7463   if (anchorIS) {
7464     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7465     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7466     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7467   }
7468 
7469   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7470   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7471   plex->anchorSection = anchorSection;
7472 
7473   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7474   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7475   plex->anchorIS = anchorIS;
7476 
7477 #if defined(PETSC_USE_DEBUG)
7478   if (anchorIS && anchorSection) {
7479     PetscInt size, a, pStart, pEnd;
7480     const PetscInt *anchors;
7481 
7482     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7483     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7484     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7485     for (a = 0; a < size; a++) {
7486       PetscInt p;
7487 
7488       p = anchors[a];
7489       if (p >= pStart && p < pEnd) {
7490         PetscInt dof;
7491 
7492         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7493         if (dof) {
7494           PetscErrorCode ierr2;
7495 
7496           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7497           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7498         }
7499       }
7500     }
7501     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7502   }
7503 #endif
7504   /* reset the generic constraints */
7505   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7506   PetscFunctionReturn(0);
7507 }
7508 
7509 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7510 {
7511   PetscSection anchorSection;
7512   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7513   PetscErrorCode ierr;
7514 
7515   PetscFunctionBegin;
7516   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7517   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7518   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7519   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7520   if (numFields) {
7521     PetscInt f;
7522     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7523 
7524     for (f = 0; f < numFields; f++) {
7525       PetscInt numComp;
7526 
7527       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7528       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7529     }
7530   }
7531   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7532   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7533   pStart = PetscMax(pStart,sStart);
7534   pEnd   = PetscMin(pEnd,sEnd);
7535   pEnd   = PetscMax(pStart,pEnd);
7536   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7537   for (p = pStart; p < pEnd; p++) {
7538     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7539     if (dof) {
7540       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7541       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7542       for (f = 0; f < numFields; f++) {
7543         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7544         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7545       }
7546     }
7547   }
7548   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7549   PetscFunctionReturn(0);
7550 }
7551 
7552 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7553 {
7554   PetscSection aSec;
7555   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7556   const PetscInt *anchors;
7557   PetscInt numFields, f;
7558   IS aIS;
7559   PetscErrorCode ierr;
7560 
7561   PetscFunctionBegin;
7562   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7563   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7564   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7565   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7566   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7567   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7568   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7569   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7570   /* cSec will be a subset of aSec and section */
7571   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7572   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7573   i[0] = 0;
7574   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7575   for (p = pStart; p < pEnd; p++) {
7576     PetscInt rDof, rOff, r;
7577 
7578     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7579     if (!rDof) continue;
7580     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7581     if (numFields) {
7582       for (f = 0; f < numFields; f++) {
7583         annz = 0;
7584         for (r = 0; r < rDof; r++) {
7585           a = anchors[rOff + r];
7586           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7587           annz += aDof;
7588         }
7589         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7590         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7591         for (q = 0; q < dof; q++) {
7592           i[off + q + 1] = i[off + q] + annz;
7593         }
7594       }
7595     }
7596     else {
7597       annz = 0;
7598       for (q = 0; q < dof; q++) {
7599         a = anchors[off + q];
7600         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7601         annz += aDof;
7602       }
7603       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7604       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7605       for (q = 0; q < dof; q++) {
7606         i[off + q + 1] = i[off + q] + annz;
7607       }
7608     }
7609   }
7610   nnz = i[m];
7611   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7612   offset = 0;
7613   for (p = pStart; p < pEnd; p++) {
7614     if (numFields) {
7615       for (f = 0; f < numFields; f++) {
7616         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7617         for (q = 0; q < dof; q++) {
7618           PetscInt rDof, rOff, r;
7619           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7620           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7621           for (r = 0; r < rDof; r++) {
7622             PetscInt s;
7623 
7624             a = anchors[rOff + r];
7625             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7626             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7627             for (s = 0; s < aDof; s++) {
7628               j[offset++] = aOff + s;
7629             }
7630           }
7631         }
7632       }
7633     }
7634     else {
7635       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7636       for (q = 0; q < dof; q++) {
7637         PetscInt rDof, rOff, r;
7638         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7639         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7640         for (r = 0; r < rDof; r++) {
7641           PetscInt s;
7642 
7643           a = anchors[rOff + r];
7644           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7645           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7646           for (s = 0; s < aDof; s++) {
7647             j[offset++] = aOff + s;
7648           }
7649         }
7650       }
7651     }
7652   }
7653   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7654   ierr = PetscFree(i);CHKERRQ(ierr);
7655   ierr = PetscFree(j);CHKERRQ(ierr);
7656   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7657   PetscFunctionReturn(0);
7658 }
7659 
7660 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7661 {
7662   DM_Plex        *plex = (DM_Plex *)dm->data;
7663   PetscSection   anchorSection, section, cSec;
7664   Mat            cMat;
7665   PetscErrorCode ierr;
7666 
7667   PetscFunctionBegin;
7668   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7669   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7670   if (anchorSection) {
7671     PetscInt Nf;
7672 
7673     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
7674     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7675     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7676     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7677     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7678     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7679     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7680     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7681   }
7682   PetscFunctionReturn(0);
7683 }
7684 
7685 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7686 {
7687   IS             subis;
7688   PetscSection   section, subsection;
7689   PetscErrorCode ierr;
7690 
7691   PetscFunctionBegin;
7692   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
7693   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7694   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7695   /* Create subdomain */
7696   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7697   /* Create submodel */
7698   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7699   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7700   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7701   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
7702   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7703   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7704   /* Create map from submodel to global model */
7705   if (is) {
7706     PetscSection    sectionGlobal, subsectionGlobal;
7707     IS              spIS;
7708     const PetscInt *spmap;
7709     PetscInt       *subIndices;
7710     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7711     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7712 
7713     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7714     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7715     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7716     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7717     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7718     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7719     for (p = pStart; p < pEnd; ++p) {
7720       PetscInt gdof, pSubSize  = 0;
7721 
7722       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7723       if (gdof > 0) {
7724         for (f = 0; f < Nf; ++f) {
7725           PetscInt fdof, fcdof;
7726 
7727           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7728           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7729           pSubSize += fdof-fcdof;
7730         }
7731         subSize += pSubSize;
7732         if (pSubSize) {
7733           if (bs < 0) {
7734             bs = pSubSize;
7735           } else if (bs != pSubSize) {
7736             /* Layout does not admit a pointwise block size */
7737             bs = 1;
7738           }
7739         }
7740       }
7741     }
7742     /* Must have same blocksize on all procs (some might have no points) */
7743     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7744     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7745     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7746     else                            {bs = bsMinMax[0];}
7747     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7748     for (p = pStart; p < pEnd; ++p) {
7749       PetscInt gdof, goff;
7750 
7751       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7752       if (gdof > 0) {
7753         const PetscInt point = spmap[p];
7754 
7755         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7756         for (f = 0; f < Nf; ++f) {
7757           PetscInt fdof, fcdof, fc, f2, poff = 0;
7758 
7759           /* Can get rid of this loop by storing field information in the global section */
7760           for (f2 = 0; f2 < f; ++f2) {
7761             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7762             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7763             poff += fdof-fcdof;
7764           }
7765           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7766           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7767           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7768             subIndices[subOff] = goff+poff+fc;
7769           }
7770         }
7771       }
7772     }
7773     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7774     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7775     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7776     if (bs > 1) {
7777       /* We need to check that the block size does not come from non-contiguous fields */
7778       PetscInt i, j, set = 1;
7779       for (i = 0; i < subSize; i += bs) {
7780         for (j = 0; j < bs; ++j) {
7781           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7782         }
7783       }
7784       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7785     }
7786     /* Attach nullspace */
7787     for (f = 0; f < Nf; ++f) {
7788       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7789       if ((*subdm)->nullspaceConstructors[f]) break;
7790     }
7791     if (f < Nf) {
7792       MatNullSpace nullSpace;
7793 
7794       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7795       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7796       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7797     }
7798   }
7799   PetscFunctionReturn(0);
7800 }
7801