xref: /petsc/src/dm/impls/plex/plex.c (revision 5d80c0bf20dfdf3e47e6174603b3725e9d56818d)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetLocalSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", (double) PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     char         lname[PETSC_MAX_PATH_LEN];
675     PetscReal    scale         = 2.0;
676     PetscReal    tikzscale     = 1.0;
677     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
678     double       tcoords[3];
679     PetscScalar *coords;
680     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
681     PetscMPIInt  rank, size;
682     char         **names, **colors, **lcolors;
683     PetscBool    plotEdges, flg, lflg;
684     PetscBT      wp = NULL;
685     PetscInt     pEnd, pStart;
686 
687     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
688     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
689     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
690     numLabels  = PetscMax(numLabels, 10);
691     numColors  = 10;
692     numLColors = 10;
693     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
694     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
695     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
696     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
697     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
698     if (!useLabels) numLabels = 0;
699     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
700     if (!useColors) {
701       numColors = 3;
702       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
703     }
704     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
705     if (!useColors) {
706       numLColors = 4;
707       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
708     }
709     ierr = PetscOptionsGetString(((PetscObject) viewer)->options, ((PetscObject) viewer)->prefix, "-dm_plex_view_label_filter", lname, PETSC_MAX_PATH_LEN, &lflg);CHKERRQ(ierr);
710     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
711     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
712     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
713     if (depth < dim) plotEdges = PETSC_FALSE;
714 
715     /* filter points with labelvalue != labeldefaultvalue */
716     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
717     if (lflg) {
718       DMLabel lbl;
719 
720       ierr = DMGetLabel(dm, lname, &lbl);CHKERRQ(ierr);
721       if (lbl) {
722         PetscInt val, defval;
723 
724         ierr = DMLabelGetDefaultValue(lbl, &defval);CHKERRQ(ierr);
725         ierr = PetscBTCreate(pEnd-pStart, &wp);CHKERRQ(ierr);
726         for (c = pStart;  c < pEnd; c++) {
727           PetscInt *closure = NULL;
728           PetscInt  closureSize;
729 
730           ierr = DMLabelGetValue(lbl, c, &val);CHKERRQ(ierr);
731           if (val == defval) continue;
732 
733           ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
734           for (p = 0; p < closureSize*2; p += 2) {
735             ierr = PetscBTSet(wp, closure[p] - pStart);CHKERRQ(ierr);
736           }
737           ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
738         }
739       }
740     }
741 
742     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
743     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
744     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
745     ierr = PetscViewerASCIIPrintf(viewer, "\
746 \\documentclass[tikz]{standalone}\n\n\
747 \\usepackage{pgflibraryshapes}\n\
748 \\usetikzlibrary{backgrounds}\n\
749 \\usetikzlibrary{arrows}\n\
750 \\begin{document}\n");CHKERRQ(ierr);
751     if (size > 1) {
752       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
753       for (p = 0; p < size; ++p) {
754         if (p > 0 && p == size-1) {
755           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
756         } else if (p > 0) {
757           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
758         }
759         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
760       }
761       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
762     }
763     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double) tikzscale);CHKERRQ(ierr);
764 
765     /* Plot vertices */
766     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
767     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
768     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
769     for (v = vStart; v < vEnd; ++v) {
770       PetscInt  off, dof, d;
771       PetscBool isLabeled = PETSC_FALSE;
772 
773       if (wp && !PetscBTLookup(wp,v - pStart)) continue;
774       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
775       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
776       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
777       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
778       for (d = 0; d < dof; ++d) {
779         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
780         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
781       }
782       /* Rotate coordinates since PGF makes z point out of the page instead of up */
783       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
784       for (d = 0; d < dof; ++d) {
785         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
786         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) tcoords[d]);CHKERRQ(ierr);
787       }
788       color = colors[rank%numColors];
789       for (l = 0; l < numLabels; ++l) {
790         PetscInt val;
791         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
792         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
793       }
794       if (useNumbers) {
795         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
796       } else {
797         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
798       }
799     }
800     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
801     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
802     /* Plot cells */
803     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
804     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
805     if (dim == 3 || !useNumbers) {
806       for (e = eStart; e < eEnd; ++e) {
807         const PetscInt *cone;
808 
809         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
810         color = colors[rank%numColors];
811         for (l = 0; l < numLabels; ++l) {
812           PetscInt val;
813           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
814           if (val >= 0) {color = lcolors[l%numLColors]; break;}
815         }
816         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
817         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
818       }
819     } else {
820       for (c = cStart; c < cEnd; ++c) {
821         PetscInt *closure = NULL;
822         PetscInt  closureSize, firstPoint = -1;
823 
824         if (wp && !PetscBTLookup(wp,c - pStart)) continue;
825         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
826         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
827         for (p = 0; p < closureSize*2; p += 2) {
828           const PetscInt point = closure[p];
829 
830           if ((point < vStart) || (point >= vEnd)) continue;
831           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
832           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
833           if (firstPoint < 0) firstPoint = point;
834         }
835         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
836         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
837         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
838       }
839     }
840     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
841     for (c = cStart; c < cEnd; ++c) {
842       double    ccoords[3] = {0.0, 0.0, 0.0};
843       PetscBool isLabeled  = PETSC_FALSE;
844       PetscInt *closure    = NULL;
845       PetscInt  closureSize, dof, d, n = 0;
846 
847       if (wp && !PetscBTLookup(wp,c - pStart)) continue;
848       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
849       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
850       for (p = 0; p < closureSize*2; p += 2) {
851         const PetscInt point = closure[p];
852         PetscInt       off;
853 
854         if ((point < vStart) || (point >= vEnd)) continue;
855         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
856         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
857         for (d = 0; d < dof; ++d) {
858           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
859           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
860         }
861         /* Rotate coordinates since PGF makes z point out of the page instead of up */
862         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
863         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
864         ++n;
865       }
866       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
867       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
868       for (d = 0; d < dof; ++d) {
869         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
870         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double) ccoords[d]);CHKERRQ(ierr);
871       }
872       color = colors[rank%numColors];
873       for (l = 0; l < numLabels; ++l) {
874         PetscInt val;
875         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
876         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
877       }
878       if (useNumbers) {
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
880       } else {
881         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
882       }
883     }
884     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
885     /* Plot edges */
886     if (plotEdges) {
887       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
888       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
889       for (e = eStart; e < eEnd; ++e) {
890         const PetscInt *cone;
891         PetscInt        coneSize, offA, offB, dof, d;
892 
893         if (wp && !PetscBTLookup(wp,e - pStart)) continue;
894         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
895         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
896         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
897         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
898         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
899         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
900         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
901         for (d = 0; d < dof; ++d) {
902           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
903           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
904         }
905         /* Rotate coordinates since PGF makes z point out of the page instead of up */
906         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
907         for (d = 0; d < dof; ++d) {
908           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
909           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
910         }
911         color = colors[rank%numColors];
912         for (l = 0; l < numLabels; ++l) {
913           PetscInt val;
914           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
915           if (val >= 0) {color = lcolors[l%numLColors]; break;}
916         }
917         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
918       }
919       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
920       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
921       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
922     }
923     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
924     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
925     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
926     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
927     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
928     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
929     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
930     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
931     ierr = PetscBTDestroy(&wp);CHKERRQ(ierr);
932   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
933     Vec                    cown,acown;
934     VecScatter             sct;
935     ISLocalToGlobalMapping g2l;
936     IS                     gid,acis;
937     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
938     MPI_Group              ggroup,ngroup;
939     PetscScalar            *array,nid;
940     const PetscInt         *idxs;
941     PetscInt               *idxs2,*start,*adjacency,*work;
942     PetscInt64             lm[3],gm[3];
943     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
944     PetscMPIInt            d1,d2,rank;
945 
946     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
947     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
948 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
949     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
950 #endif
951     if (ncomm != MPI_COMM_NULL) {
952       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
953       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
954       d1   = 0;
955       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
956       nid  = d2;
957       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
958       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
959       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
960     } else nid = 0.0;
961 
962     /* Get connectivity */
963     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
964     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
965 
966     /* filter overlapped local cells */
967     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
968     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
969     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
970     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
971     for (c = cStart, cum = 0; c < cEnd; c++) {
972       if (idxs[c-cStart] < 0) continue;
973       idxs2[cum++] = idxs[c-cStart];
974     }
975     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
976     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
977     ierr = ISDestroy(&gid);CHKERRQ(ierr);
978     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
979 
980     /* support for node-aware cell locality */
981     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
982     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
983     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
984     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
985     for (c = 0; c < numVertices; c++) array[c] = nid;
986     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
987     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
988     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
989     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
990     ierr = ISDestroy(&acis);CHKERRQ(ierr);
991     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
992     ierr = VecDestroy(&cown);CHKERRQ(ierr);
993 
994     /* compute edgeCut */
995     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
996     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
997     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
998     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
999     ierr = ISDestroy(&gid);CHKERRQ(ierr);
1000     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
1001     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1002       PetscInt totl;
1003 
1004       totl = start[c+1]-start[c];
1005       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
1006       for (i = 0; i < totl; i++) {
1007         if (work[i] < 0) {
1008           ect  += 1;
1009           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1010         }
1011       }
1012     }
1013     ierr  = PetscFree(work);CHKERRQ(ierr);
1014     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
1015     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
1016     lm[1] = -numVertices;
1017     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
1018     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
1019     lm[0] = ect; /* edgeCut */
1020     lm[1] = ectn; /* node-aware edgeCut */
1021     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1022     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
1023     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
1024 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1025     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
1026 #else
1027     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
1028 #endif
1029     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
1030     ierr  = PetscFree(start);CHKERRQ(ierr);
1031     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
1032     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
1033   } else {
1034     MPI_Comm    comm;
1035     PetscInt   *sizes, *hybsizes;
1036     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
1037     PetscInt    pStart, pEnd, p;
1038     PetscInt    numLabels, l;
1039     const char *name;
1040     PetscMPIInt size;
1041 
1042     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1043     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1044     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1045     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1046     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1047     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1048     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1049     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1050     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1051     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1052     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1053     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1054     if (depth == 1) {
1055       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1056       pEnd = pEnd - pStart;
1057       pMax[0] -= pStart;
1058       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1059       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1060       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1061       for (p = 0; p < size; ++p) {
1062         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1063         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1064       }
1065       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1066       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1067       pEnd = pEnd - pStart;
1068       pMax[depth] -= pStart;
1069       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1070       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1071       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1072       for (p = 0; p < size; ++p) {
1073         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1074         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1075       }
1076       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1077     } else {
1078       PetscMPIInt rank;
1079       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1080       for (d = 0; d <= dim; d++) {
1081         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1082         pEnd    -= pStart;
1083         pMax[d] -= pStart;
1084         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1085         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1086         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1087         for (p = 0; p < size; ++p) {
1088           if (!rank) {
1089             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1090             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1091           }
1092         }
1093         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1094       }
1095     }
1096     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1097     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1098     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1099     for (l = 0; l < numLabels; ++l) {
1100       DMLabel         label;
1101       const char     *name;
1102       IS              valueIS;
1103       const PetscInt *values;
1104       PetscInt        numValues, v;
1105 
1106       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1107       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1108       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1109       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1110       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1111       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1112       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1113       for (v = 0; v < numValues; ++v) {
1114         PetscInt size;
1115 
1116         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1117         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1118         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1119       }
1120       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1121       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1122       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1123       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1124     }
1125     /* If no fields are specified, people do not want to see adjacency */
1126     if (dm->Nf) {
1127       PetscInt f;
1128 
1129       for (f = 0; f < dm->Nf; ++f) {
1130         const char *name;
1131 
1132         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1133         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1134         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1135         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1136         if (dm->fields[f].adjacency[0]) {
1137           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1138           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1139         } else {
1140           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1141           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1142         }
1143         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1144       }
1145     }
1146     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1147     if (cdm) {
1148       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1149       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1150       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1151     }
1152   }
1153   PetscFunctionReturn(0);
1154 }
1155 
1156 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1157 {
1158   PetscDraw          draw;
1159   DM                 cdm;
1160   PetscSection       coordSection;
1161   Vec                coordinates;
1162   const PetscScalar *coords;
1163   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1164   PetscBool          isnull;
1165   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1166   PetscMPIInt        rank;
1167   PetscErrorCode     ierr;
1168 
1169   PetscFunctionBegin;
1170   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1171   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1172   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1173   ierr = DMGetLocalSection(cdm, &coordSection);CHKERRQ(ierr);
1174   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1175   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1176   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1177 
1178   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1179   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1180   if (isnull) PetscFunctionReturn(0);
1181   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1182 
1183   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1184   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1185   for (c = 0; c < N; c += dim) {
1186     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1187     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1188   }
1189   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1190   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1191   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1192   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1193   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1194 
1195   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1196   for (c = cStart; c < cEnd; ++c) {
1197     PetscScalar *coords = NULL;
1198     PetscInt     numCoords,coneSize;
1199 
1200     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1201     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202     switch (coneSize) {
1203     case 3:
1204       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1205                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1206                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1207                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1208       break;
1209     case 4:
1210       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1211                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1212                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1213                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1214                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1215       break;
1216     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1217     }
1218     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1219   }
1220   for (c = cStart; c < cEnd; ++c) {
1221     PetscScalar *coords = NULL;
1222     PetscInt     numCoords,coneSize;
1223 
1224     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1225     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1226     switch (coneSize) {
1227     case 3:
1228       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1229       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1230       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1231       break;
1232     case 4:
1233       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1234       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1235       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1236       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1237       break;
1238     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1239     }
1240     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1241   }
1242   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1243   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1244   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1249 {
1250   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1251   char           name[PETSC_MAX_PATH_LEN];
1252   PetscErrorCode ierr;
1253 
1254   PetscFunctionBegin;
1255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1256   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1257   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1258   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1259   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1260   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1261   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1262   if (iascii) {
1263     PetscViewerFormat format;
1264     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1265     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1266       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1267     } else {
1268       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1269     }
1270   } else if (ishdf5) {
1271 #if defined(PETSC_HAVE_HDF5)
1272     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1273 #else
1274     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1275 #endif
1276   } else if (isvtk) {
1277     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1278   } else if (isdraw) {
1279     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1280   } else if (isglvis) {
1281     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1282   } else {
1283     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1284   }
1285   /* Optionally view the partition */
1286   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1287   if (flg) {
1288     Vec ranks;
1289     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1290     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1291     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1292   }
1293   /* Optionally view a label */
1294   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1295   if (flg) {
1296     DMLabel label;
1297     Vec     val;
1298 
1299     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1300     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1301     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1302     ierr = VecView(val, viewer);CHKERRQ(ierr);
1303     ierr = VecDestroy(&val);CHKERRQ(ierr);
1304   }
1305   PetscFunctionReturn(0);
1306 }
1307 
1308 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1309 {
1310   PetscBool      ishdf5;
1311   PetscErrorCode ierr;
1312 
1313   PetscFunctionBegin;
1314   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1315   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1316   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1317   if (ishdf5) {
1318 #if defined(PETSC_HAVE_HDF5)
1319     PetscViewerFormat format;
1320     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1321     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1322       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1323     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1324       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1325     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1326 #else
1327     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1328 #endif
1329   } else {
1330     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1331   }
1332   PetscFunctionReturn(0);
1333 }
1334 
1335 PetscErrorCode DMDestroy_Plex(DM dm)
1336 {
1337   DM_Plex       *mesh = (DM_Plex*) dm->data;
1338   PetscErrorCode ierr;
1339 
1340   PetscFunctionBegin;
1341   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1342   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1343   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C", NULL);CHKERRQ(ierr);
1344   if (--mesh->refct > 0) PetscFunctionReturn(0);
1345   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1346   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1347   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1348   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1349   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1350   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1351   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1352   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1353   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1354   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1355   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1356   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1357   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1358   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1359   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1360   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1361   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1362   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1363   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1364   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1365   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1366   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1367   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1368   ierr = PetscFree(mesh);CHKERRQ(ierr);
1369   PetscFunctionReturn(0);
1370 }
1371 
1372 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1373 {
1374   PetscSection           sectionGlobal;
1375   PetscInt               bs = -1, mbs;
1376   PetscInt               localSize;
1377   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1378   PetscErrorCode         ierr;
1379   MatType                mtype;
1380   ISLocalToGlobalMapping ltog;
1381 
1382   PetscFunctionBegin;
1383   ierr = MatInitializePackage();CHKERRQ(ierr);
1384   mtype = dm->mattype;
1385   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1386   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1387   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1388   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1389   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1390   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1391   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1392   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1393   if (mbs > 1) bs = mbs;
1394   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1395   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1396   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1397   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1398   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1399   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1400   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1401   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1402   if (!isShell) {
1403     PetscSection subSection;
1404     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1405     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1406     PetscInt     pStart, pEnd, p, dof, cdof;
1407 
1408     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1409     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1410       PetscSection section;
1411       PetscInt     size;
1412 
1413       ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
1414       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1415       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1416       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1417     } else {
1418       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1419     }
1420     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1421     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1422       PetscInt bdof;
1423 
1424       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1425       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1426       dof  = dof < 0 ? -(dof+1) : dof;
1427       bdof = cdof && (dof-cdof) ? 1 : dof;
1428       if (dof) {
1429         if (bs < 0)          {bs = bdof;}
1430         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1431       }
1432       if (isMatIS) {
1433         PetscInt loff,c,off;
1434         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1435         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1436         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1437       }
1438     }
1439     /* Must have same blocksize on all procs (some might have no points) */
1440     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1441     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1442     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1443     else                            {bs = bsMinMax[0];}
1444     bs = PetscMax(1,bs);
1445     if (isMatIS) { /* Must reduce indices by blocksize */
1446       PetscInt l;
1447 
1448       lsize = lsize/bs;
1449       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] = ltogidx[l*bs]/bs;
1450       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1451     }
1452     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1453     if (isMatIS) {
1454       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1455     }
1456     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1457     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1458     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1459   }
1460   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1461   PetscFunctionReturn(0);
1462 }
1463 
1464 /*@
1465   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1466 
1467   Not collective
1468 
1469   Input Parameter:
1470 . mesh - The DMPlex
1471 
1472   Output Parameters:
1473 . subsection - The subdomain section
1474 
1475   Level: developer
1476 
1477 .seealso:
1478 @*/
1479 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1480 {
1481   DM_Plex       *mesh = (DM_Plex*) dm->data;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   if (!mesh->subdomainSection) {
1487     PetscSection section;
1488     PetscSF      sf;
1489 
1490     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1491     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
1492     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1493     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1494   }
1495   *subsection = mesh->subdomainSection;
1496   PetscFunctionReturn(0);
1497 }
1498 
1499 /*@
1500   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1501 
1502   Not collective
1503 
1504   Input Parameter:
1505 . mesh - The DMPlex
1506 
1507   Output Parameters:
1508 + pStart - The first mesh point
1509 - pEnd   - The upper bound for mesh points
1510 
1511   Level: beginner
1512 
1513 .seealso: DMPlexCreate(), DMPlexSetChart()
1514 @*/
1515 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1516 {
1517   DM_Plex       *mesh = (DM_Plex*) dm->data;
1518   PetscErrorCode ierr;
1519 
1520   PetscFunctionBegin;
1521   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1522   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1523   PetscFunctionReturn(0);
1524 }
1525 
1526 /*@
1527   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1528 
1529   Not collective
1530 
1531   Input Parameters:
1532 + mesh - The DMPlex
1533 . pStart - The first mesh point
1534 - pEnd   - The upper bound for mesh points
1535 
1536   Output Parameters:
1537 
1538   Level: beginner
1539 
1540 .seealso: DMPlexCreate(), DMPlexGetChart()
1541 @*/
1542 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1543 {
1544   DM_Plex       *mesh = (DM_Plex*) dm->data;
1545   PetscErrorCode ierr;
1546 
1547   PetscFunctionBegin;
1548   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1549   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1550   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1551   PetscFunctionReturn(0);
1552 }
1553 
1554 /*@
1555   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1556 
1557   Not collective
1558 
1559   Input Parameters:
1560 + mesh - The DMPlex
1561 - p - The point, which must lie in the chart set with DMPlexSetChart()
1562 
1563   Output Parameter:
1564 . size - The cone size for point p
1565 
1566   Level: beginner
1567 
1568 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1569 @*/
1570 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1571 {
1572   DM_Plex       *mesh = (DM_Plex*) dm->data;
1573   PetscErrorCode ierr;
1574 
1575   PetscFunctionBegin;
1576   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1577   PetscValidPointer(size, 3);
1578   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1579   PetscFunctionReturn(0);
1580 }
1581 
1582 /*@
1583   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1584 
1585   Not collective
1586 
1587   Input Parameters:
1588 + mesh - The DMPlex
1589 . p - The point, which must lie in the chart set with DMPlexSetChart()
1590 - size - The cone size for point p
1591 
1592   Output Parameter:
1593 
1594   Note:
1595   This should be called after DMPlexSetChart().
1596 
1597   Level: beginner
1598 
1599 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1600 @*/
1601 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1602 {
1603   DM_Plex       *mesh = (DM_Plex*) dm->data;
1604   PetscErrorCode ierr;
1605 
1606   PetscFunctionBegin;
1607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1608   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1609 
1610   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1611   PetscFunctionReturn(0);
1612 }
1613 
1614 /*@
1615   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1616 
1617   Not collective
1618 
1619   Input Parameters:
1620 + mesh - The DMPlex
1621 . p - The point, which must lie in the chart set with DMPlexSetChart()
1622 - size - The additional cone size for point p
1623 
1624   Output Parameter:
1625 
1626   Note:
1627   This should be called after DMPlexSetChart().
1628 
1629   Level: beginner
1630 
1631 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1632 @*/
1633 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1634 {
1635   DM_Plex       *mesh = (DM_Plex*) dm->data;
1636   PetscInt       csize;
1637   PetscErrorCode ierr;
1638 
1639   PetscFunctionBegin;
1640   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1641   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1642   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1643 
1644   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1645   PetscFunctionReturn(0);
1646 }
1647 
1648 /*@C
1649   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1650 
1651   Not collective
1652 
1653   Input Parameters:
1654 + dm - The DMPlex
1655 - p - The point, which must lie in the chart set with DMPlexSetChart()
1656 
1657   Output Parameter:
1658 . cone - An array of points which are on the in-edges for point p
1659 
1660   Level: beginner
1661 
1662   Fortran Notes:
1663   Since it returns an array, this routine is only available in Fortran 90, and you must
1664   include petsc.h90 in your code.
1665   You must also call DMPlexRestoreCone() after you finish using the returned array.
1666   DMPlexRestoreCone() is not needed/available in C.
1667 
1668 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1669 @*/
1670 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1671 {
1672   DM_Plex       *mesh = (DM_Plex*) dm->data;
1673   PetscInt       off;
1674   PetscErrorCode ierr;
1675 
1676   PetscFunctionBegin;
1677   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1678   PetscValidPointer(cone, 3);
1679   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1680   *cone = &mesh->cones[off];
1681   PetscFunctionReturn(0);
1682 }
1683 
1684 /*@C
1685   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1686 
1687   Not collective
1688 
1689   Input Parameters:
1690 + dm - The DMPlex
1691 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1692 
1693   Output Parameter:
1694 + pConesSection - PetscSection describing the layout of pCones
1695 - pCones - An array of points which are on the in-edges for the point set p
1696 
1697   Level: intermediate
1698 
1699 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1700 @*/
1701 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1702 {
1703   PetscSection        cs, newcs;
1704   PetscInt            *cones;
1705   PetscInt            *newarr=NULL;
1706   PetscInt            n;
1707   PetscErrorCode      ierr;
1708 
1709   PetscFunctionBegin;
1710   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1711   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1712   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1713   if (pConesSection) *pConesSection = newcs;
1714   if (pCones) {
1715     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1716     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1717   }
1718   PetscFunctionReturn(0);
1719 }
1720 
1721 /*@
1722   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
1723 
1724   Not collective
1725 
1726   Input Parameters:
1727 + dm - The DMPlex
1728 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1729 
1730   Output Parameter:
1731 . expandedPoints - An array of vertices recursively expanded from input points
1732 
1733   Level: advanced
1734 
1735   Notes:
1736   Like DMPlexGetConeRecursive but returns only the 0-depth IS (i.e. vertices only) and no sections.
1737   There is no corresponding Restore function, just call ISDestroy() on the returned IS to deallocate.
1738 
1739 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexRestoreConeRecursive(), DMPlexGetDepth()
1740 @*/
1741 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
1742 {
1743   IS                  *expandedPointsAll;
1744   PetscInt            depth;
1745   PetscErrorCode      ierr;
1746 
1747   PetscFunctionBegin;
1748   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1749   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1750   PetscValidPointer(expandedPoints, 3);
1751   ierr = DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1752   *expandedPoints = expandedPointsAll[0];
1753   ierr = PetscObjectReference((PetscObject)expandedPointsAll[0]);
1754   ierr = DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL);CHKERRQ(ierr);
1755   PetscFunctionReturn(0);
1756 }
1757 
1758 /*@
1759   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices (DAG points of depth 0, i.e. without cones).
1760 
1761   Not collective
1762 
1763   Input Parameters:
1764 + dm - The DMPlex
1765 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1766 
1767   Output Parameter:
1768 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1769 . expandedPoints - (optional) An array of index sets with recursively expanded cones
1770 - sections - (optional) An array of sections which describe mappings from points to their cone points
1771 
1772   Level: advanced
1773 
1774   Notes:
1775   Like DMPlexGetConeTuple() but recursive.
1776 
1777   Array expandedPoints has size equal to depth. Each expandedPoints[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
1778   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
1779 
1780   Array section has size equal to depth.  Each PetscSection sections[d] realizes mapping from expandedPoints[d+1] (section points) to expandedPoints[d] (section dofs) as follows:
1781   (1) DAG points in expandedPoints[d+1] with depth d+1 to their cone points in expandedPoints[d];
1782   (2) DAG points in expandedPoints[d+1] with depth in [0,d] to the same points in expandedPoints[d].
1783 
1784 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexRestoreConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1785 @*/
1786 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1787 {
1788   const PetscInt      *arr0=NULL, *cone=NULL;
1789   PetscInt            *arr=NULL, *newarr=NULL;
1790   PetscInt            d, depth_, i, n, newn, cn, co, start, end;
1791   IS                  *expandedPoints_;
1792   PetscSection        *sections_;
1793   PetscErrorCode      ierr;
1794 
1795   PetscFunctionBegin;
1796   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1797   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
1798   if (depth) PetscValidIntPointer(depth, 3);
1799   if (expandedPoints) PetscValidPointer(expandedPoints, 4);
1800   if (sections) PetscValidPointer(sections, 5);
1801   ierr = ISGetLocalSize(points, &n);CHKERRQ(ierr);
1802   ierr = ISGetIndices(points, &arr0);CHKERRQ(ierr);
1803   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1804   ierr = PetscCalloc1(depth_, &expandedPoints_);CHKERRQ(ierr);
1805   ierr = PetscCalloc1(depth_, &sections_);CHKERRQ(ierr);
1806   arr = (PetscInt*) arr0; /* this is ok because first generation of arr is not modified */
1807   for (d=depth_-1; d>=0; d--) {
1808     ierr = PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]);CHKERRQ(ierr);
1809     ierr = PetscSectionSetChart(sections_[d], 0, n);CHKERRQ(ierr);
1810     for (i=0; i<n; i++) {
1811       ierr = DMPlexGetDepthStratum(dm, d+1, &start, &end);CHKERRQ(ierr);
1812       if (arr[i] >= start && arr[i] < end) {
1813         ierr = DMPlexGetConeSize(dm, arr[i], &cn);CHKERRQ(ierr);
1814         ierr = PetscSectionSetDof(sections_[d], i, cn);CHKERRQ(ierr);
1815       } else {
1816         ierr = PetscSectionSetDof(sections_[d], i, 1);CHKERRQ(ierr);
1817       }
1818     }
1819     ierr = PetscSectionSetUp(sections_[d]);CHKERRQ(ierr);
1820     ierr = PetscSectionGetStorageSize(sections_[d], &newn);CHKERRQ(ierr);
1821     ierr = PetscMalloc1(newn, &newarr);CHKERRQ(ierr);
1822     for (i=0; i<n; i++) {
1823       ierr = PetscSectionGetDof(sections_[d], i, &cn);CHKERRQ(ierr);
1824       ierr = PetscSectionGetOffset(sections_[d], i, &co);CHKERRQ(ierr);
1825       if (cn > 1) {
1826         ierr = DMPlexGetCone(dm, arr[i], &cone);CHKERRQ(ierr);
1827         ierr = PetscMemcpy(&newarr[co], cone, cn*sizeof(PetscInt));CHKERRQ(ierr);
1828       } else {
1829         newarr[co] = arr[i];
1830       }
1831     }
1832     ierr = ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]);CHKERRQ(ierr);
1833     arr = newarr;
1834     n = newn;
1835   }
1836   *depth = depth_;
1837   if (expandedPoints) *expandedPoints = expandedPoints_;
1838   else {
1839     for (d=0; d<depth_; d++) {ierr = ISDestroy(&expandedPoints_[d]);CHKERRQ(ierr);}
1840     ierr = PetscFree(expandedPoints_);CHKERRQ(ierr);
1841   }
1842   if (sections) *sections = sections_;
1843   else {
1844     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&sections_[d]);CHKERRQ(ierr);}
1845     ierr = PetscFree(sections_);CHKERRQ(ierr);
1846   }
1847   PetscFunctionReturn(0);
1848 }
1849 
1850 /*@
1851   DMPlexRestoreConeRecursive - Deallocates arrays created by DMPlexGetConeRecursive
1852 
1853   Not collective
1854 
1855   Input Parameters:
1856 + dm - The DMPlex
1857 - points - The IS of points, which must lie in the chart set with DMPlexSetChart()
1858 
1859   Output Parameter:
1860 + depth - (optional) Size of the output arrays, equal to DMPlex depth, returned by DMPlexGetDepth()
1861 . expandedPoints - (optional) An array of recursively expanded cones
1862 - sections - (optional) An array of sections which describe mappings from points to their cone points
1863 
1864   Level: advanced
1865 
1866   Notes:
1867   See DMPlexGetConeRecursive() for details.
1868 
1869 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple(), DMPlexGetConeRecursive(), DMPlexGetConeRecursiveVertices(), DMPlexGetDepth()
1870 @*/
1871 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
1872 {
1873   PetscInt            d, depth_;
1874   PetscErrorCode      ierr;
1875 
1876   PetscFunctionBegin;
1877   ierr = DMPlexGetDepth(dm, &depth_);CHKERRQ(ierr);
1878   if (depth && *depth != depth_) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
1879   if (depth) *depth = 0;
1880   if (expandedPoints) {
1881     for (d=0; d<depth_; d++) {ierr = ISDestroy(&((*expandedPoints)[d]));CHKERRQ(ierr);}
1882     ierr = PetscFree(*expandedPoints);CHKERRQ(ierr);
1883   }
1884   if (sections)  {
1885     for (d=0; d<depth_; d++) {ierr = PetscSectionDestroy(&((*sections)[d]));CHKERRQ(ierr);}
1886     ierr = PetscFree(*sections);CHKERRQ(ierr);
1887   }
1888   PetscFunctionReturn(0);
1889 }
1890 
1891 /*@
1892   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1893 
1894   Not collective
1895 
1896   Input Parameters:
1897 + mesh - The DMPlex
1898 . p - The point, which must lie in the chart set with DMPlexSetChart()
1899 - cone - An array of points which are on the in-edges for point p
1900 
1901   Output Parameter:
1902 
1903   Note:
1904   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1905 
1906   Developer Note: Why not call this DMPlexSetCover()
1907 
1908   Level: beginner
1909 
1910 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1911 @*/
1912 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1913 {
1914   DM_Plex       *mesh = (DM_Plex*) dm->data;
1915   PetscInt       pStart, pEnd;
1916   PetscInt       dof, off, c;
1917   PetscErrorCode ierr;
1918 
1919   PetscFunctionBegin;
1920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1921   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1922   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1923   if (dof) PetscValidPointer(cone, 3);
1924   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1925   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1926   for (c = 0; c < dof; ++c) {
1927     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1928     mesh->cones[off+c] = cone[c];
1929   }
1930   PetscFunctionReturn(0);
1931 }
1932 
1933 /*@C
1934   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1935 
1936   Not collective
1937 
1938   Input Parameters:
1939 + mesh - The DMPlex
1940 - p - The point, which must lie in the chart set with DMPlexSetChart()
1941 
1942   Output Parameter:
1943 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1944                     integer giving the prescription for cone traversal. If it is negative, the cone is
1945                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1946                     the index of the cone point on which to start.
1947 
1948   Level: beginner
1949 
1950   Fortran Notes:
1951   Since it returns an array, this routine is only available in Fortran 90, and you must
1952   include petsc.h90 in your code.
1953   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1954   DMPlexRestoreConeOrientation() is not needed/available in C.
1955 
1956 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1957 @*/
1958 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1959 {
1960   DM_Plex       *mesh = (DM_Plex*) dm->data;
1961   PetscInt       off;
1962   PetscErrorCode ierr;
1963 
1964   PetscFunctionBegin;
1965   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1966 #if defined(PETSC_USE_DEBUG)
1967   {
1968     PetscInt dof;
1969     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1970     if (dof) PetscValidPointer(coneOrientation, 3);
1971   }
1972 #endif
1973   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1974 
1975   *coneOrientation = &mesh->coneOrientations[off];
1976   PetscFunctionReturn(0);
1977 }
1978 
1979 /*@
1980   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1981 
1982   Not collective
1983 
1984   Input Parameters:
1985 + mesh - The DMPlex
1986 . p - The point, which must lie in the chart set with DMPlexSetChart()
1987 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1988                     integer giving the prescription for cone traversal. If it is negative, the cone is
1989                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1990                     the index of the cone point on which to start.
1991 
1992   Output Parameter:
1993 
1994   Note:
1995   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1996 
1997   Level: beginner
1998 
1999 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2000 @*/
2001 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
2002 {
2003   DM_Plex       *mesh = (DM_Plex*) dm->data;
2004   PetscInt       pStart, pEnd;
2005   PetscInt       dof, off, c;
2006   PetscErrorCode ierr;
2007 
2008   PetscFunctionBegin;
2009   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2010   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2011   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2012   if (dof) PetscValidPointer(coneOrientation, 3);
2013   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2014   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2015   for (c = 0; c < dof; ++c) {
2016     PetscInt cdof, o = coneOrientation[c];
2017 
2018     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
2019     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
2020     mesh->coneOrientations[off+c] = o;
2021   }
2022   PetscFunctionReturn(0);
2023 }
2024 
2025 /*@
2026   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
2027 
2028   Not collective
2029 
2030   Input Parameters:
2031 + mesh - The DMPlex
2032 . p - The point, which must lie in the chart set with DMPlexSetChart()
2033 . conePos - The local index in the cone where the point should be put
2034 - conePoint - The mesh point to insert
2035 
2036   Level: beginner
2037 
2038 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2039 @*/
2040 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
2041 {
2042   DM_Plex       *mesh = (DM_Plex*) dm->data;
2043   PetscInt       pStart, pEnd;
2044   PetscInt       dof, off;
2045   PetscErrorCode ierr;
2046 
2047   PetscFunctionBegin;
2048   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2049   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2050   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2051   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
2052   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2053   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2054   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2055   mesh->cones[off+conePos] = conePoint;
2056   PetscFunctionReturn(0);
2057 }
2058 
2059 /*@
2060   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
2061 
2062   Not collective
2063 
2064   Input Parameters:
2065 + mesh - The DMPlex
2066 . p - The point, which must lie in the chart set with DMPlexSetChart()
2067 . conePos - The local index in the cone where the point should be put
2068 - coneOrientation - The point orientation to insert
2069 
2070   Level: beginner
2071 
2072 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2073 @*/
2074 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
2075 {
2076   DM_Plex       *mesh = (DM_Plex*) dm->data;
2077   PetscInt       pStart, pEnd;
2078   PetscInt       dof, off;
2079   PetscErrorCode ierr;
2080 
2081   PetscFunctionBegin;
2082   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2083   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2084   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2085   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2086   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2087   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
2088   mesh->coneOrientations[off+conePos] = coneOrientation;
2089   PetscFunctionReturn(0);
2090 }
2091 
2092 /*@
2093   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
2094 
2095   Not collective
2096 
2097   Input Parameters:
2098 + mesh - The DMPlex
2099 - p - The point, which must lie in the chart set with DMPlexSetChart()
2100 
2101   Output Parameter:
2102 . size - The support size for point p
2103 
2104   Level: beginner
2105 
2106 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
2107 @*/
2108 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
2109 {
2110   DM_Plex       *mesh = (DM_Plex*) dm->data;
2111   PetscErrorCode ierr;
2112 
2113   PetscFunctionBegin;
2114   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2115   PetscValidPointer(size, 3);
2116   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2117   PetscFunctionReturn(0);
2118 }
2119 
2120 /*@
2121   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
2122 
2123   Not collective
2124 
2125   Input Parameters:
2126 + mesh - The DMPlex
2127 . p - The point, which must lie in the chart set with DMPlexSetChart()
2128 - size - The support size for point p
2129 
2130   Output Parameter:
2131 
2132   Note:
2133   This should be called after DMPlexSetChart().
2134 
2135   Level: beginner
2136 
2137 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
2138 @*/
2139 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
2140 {
2141   DM_Plex       *mesh = (DM_Plex*) dm->data;
2142   PetscErrorCode ierr;
2143 
2144   PetscFunctionBegin;
2145   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2146   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2147 
2148   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2149   PetscFunctionReturn(0);
2150 }
2151 
2152 /*@C
2153   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2154 
2155   Not collective
2156 
2157   Input Parameters:
2158 + mesh - The DMPlex
2159 - p - The point, which must lie in the chart set with DMPlexSetChart()
2160 
2161   Output Parameter:
2162 . support - An array of points which are on the out-edges for point p
2163 
2164   Level: beginner
2165 
2166   Fortran Notes:
2167   Since it returns an array, this routine is only available in Fortran 90, and you must
2168   include petsc.h90 in your code.
2169   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2170   DMPlexRestoreSupport() is not needed/available in C.
2171 
2172 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2173 @*/
2174 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2175 {
2176   DM_Plex       *mesh = (DM_Plex*) dm->data;
2177   PetscInt       off;
2178   PetscErrorCode ierr;
2179 
2180   PetscFunctionBegin;
2181   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2182   PetscValidPointer(support, 3);
2183   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2184   *support = &mesh->supports[off];
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 /*@
2189   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2190 
2191   Not collective
2192 
2193   Input Parameters:
2194 + mesh - The DMPlex
2195 . p - The point, which must lie in the chart set with DMPlexSetChart()
2196 - support - An array of points which are on the out-edges for point p
2197 
2198   Output Parameter:
2199 
2200   Note:
2201   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2202 
2203   Level: beginner
2204 
2205 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2206 @*/
2207 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2208 {
2209   DM_Plex       *mesh = (DM_Plex*) dm->data;
2210   PetscInt       pStart, pEnd;
2211   PetscInt       dof, off, c;
2212   PetscErrorCode ierr;
2213 
2214   PetscFunctionBegin;
2215   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2216   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2217   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2218   if (dof) PetscValidPointer(support, 3);
2219   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2220   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2221   for (c = 0; c < dof; ++c) {
2222     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2223     mesh->supports[off+c] = support[c];
2224   }
2225   PetscFunctionReturn(0);
2226 }
2227 
2228 /*@
2229   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2230 
2231   Not collective
2232 
2233   Input Parameters:
2234 + mesh - The DMPlex
2235 . p - The point, which must lie in the chart set with DMPlexSetChart()
2236 . supportPos - The local index in the cone where the point should be put
2237 - supportPoint - The mesh point to insert
2238 
2239   Level: beginner
2240 
2241 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2242 @*/
2243 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2244 {
2245   DM_Plex       *mesh = (DM_Plex*) dm->data;
2246   PetscInt       pStart, pEnd;
2247   PetscInt       dof, off;
2248   PetscErrorCode ierr;
2249 
2250   PetscFunctionBegin;
2251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2252   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2253   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2254   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2255   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2256   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2257   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2258   mesh->supports[off+supportPos] = supportPoint;
2259   PetscFunctionReturn(0);
2260 }
2261 
2262 /*@C
2263   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2264 
2265   Not collective
2266 
2267   Input Parameters:
2268 + mesh - The DMPlex
2269 . p - The point, which must lie in the chart set with DMPlexSetChart()
2270 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2271 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2272 
2273   Output Parameters:
2274 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2275 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2276 
2277   Note:
2278   If using internal storage (points is NULL on input), each call overwrites the last output.
2279 
2280   Fortran Notes:
2281   Since it returns an array, this routine is only available in Fortran 90, and you must
2282   include petsc.h90 in your code.
2283 
2284   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2285 
2286   Level: beginner
2287 
2288 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2289 @*/
2290 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2291 {
2292   DM_Plex        *mesh = (DM_Plex*) dm->data;
2293   PetscInt       *closure, *fifo;
2294   const PetscInt *tmp = NULL, *tmpO = NULL;
2295   PetscInt        tmpSize, t;
2296   PetscInt        depth       = 0, maxSize;
2297   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2298   PetscErrorCode  ierr;
2299 
2300   PetscFunctionBegin;
2301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2302   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2303   /* This is only 1-level */
2304   if (useCone) {
2305     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2306     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2307     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2308   } else {
2309     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2310     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2311   }
2312   if (depth == 1) {
2313     if (*points) {
2314       closure = *points;
2315     } else {
2316       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2317       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2318     }
2319     closure[0] = p; closure[1] = 0;
2320     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2321       closure[closureSize]   = tmp[t];
2322       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2323     }
2324     if (numPoints) *numPoints = closureSize/2;
2325     if (points)    *points    = closure;
2326     PetscFunctionReturn(0);
2327   }
2328   {
2329     PetscInt c, coneSeries, s,supportSeries;
2330 
2331     c = mesh->maxConeSize;
2332     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2333     s = mesh->maxSupportSize;
2334     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2335     maxSize = 2*PetscMax(coneSeries,supportSeries);
2336   }
2337   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2338   if (*points) {
2339     closure = *points;
2340   } else {
2341     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2342   }
2343   closure[0] = p; closure[1] = 0;
2344   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2345     const PetscInt cp = tmp[t];
2346     const PetscInt co = tmpO ? tmpO[t] : 0;
2347 
2348     closure[closureSize]   = cp;
2349     closure[closureSize+1] = co;
2350     fifo[fifoSize]         = cp;
2351     fifo[fifoSize+1]       = co;
2352   }
2353   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2354   while (fifoSize - fifoStart) {
2355     const PetscInt q   = fifo[fifoStart];
2356     const PetscInt o   = fifo[fifoStart+1];
2357     const PetscInt rev = o >= 0 ? 0 : 1;
2358     const PetscInt off = rev ? -(o+1) : o;
2359 
2360     if (useCone) {
2361       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2362       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2363       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2364     } else {
2365       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2366       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2367       tmpO = NULL;
2368     }
2369     for (t = 0; t < tmpSize; ++t) {
2370       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2371       const PetscInt cp = tmp[i];
2372       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2373       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2374        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2375       PetscInt       co = tmpO ? tmpO[i] : 0;
2376       PetscInt       c;
2377 
2378       if (rev) {
2379         PetscInt childSize, coff;
2380         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2381         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2382         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2383       }
2384       /* Check for duplicate */
2385       for (c = 0; c < closureSize; c += 2) {
2386         if (closure[c] == cp) break;
2387       }
2388       if (c == closureSize) {
2389         closure[closureSize]   = cp;
2390         closure[closureSize+1] = co;
2391         fifo[fifoSize]         = cp;
2392         fifo[fifoSize+1]       = co;
2393         closureSize           += 2;
2394         fifoSize              += 2;
2395       }
2396     }
2397     fifoStart += 2;
2398   }
2399   if (numPoints) *numPoints = closureSize/2;
2400   if (points)    *points    = closure;
2401   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2402   PetscFunctionReturn(0);
2403 }
2404 
2405 /*@C
2406   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2407 
2408   Not collective
2409 
2410   Input Parameters:
2411 + mesh - The DMPlex
2412 . p - The point, which must lie in the chart set with DMPlexSetChart()
2413 . orientation - The orientation of the point
2414 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2415 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2416 
2417   Output Parameters:
2418 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2419 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2420 
2421   Note:
2422   If using internal storage (points is NULL on input), each call overwrites the last output.
2423 
2424   Fortran Notes:
2425   Since it returns an array, this routine is only available in Fortran 90, and you must
2426   include petsc.h90 in your code.
2427 
2428   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2429 
2430   Level: beginner
2431 
2432 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2433 @*/
2434 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2435 {
2436   DM_Plex        *mesh = (DM_Plex*) dm->data;
2437   PetscInt       *closure, *fifo;
2438   const PetscInt *tmp = NULL, *tmpO = NULL;
2439   PetscInt        tmpSize, t;
2440   PetscInt        depth       = 0, maxSize;
2441   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2442   PetscErrorCode  ierr;
2443 
2444   PetscFunctionBegin;
2445   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2446   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2447   /* This is only 1-level */
2448   if (useCone) {
2449     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2450     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2451     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2452   } else {
2453     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2454     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2455   }
2456   if (depth == 1) {
2457     if (*points) {
2458       closure = *points;
2459     } else {
2460       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2461       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2462     }
2463     closure[0] = p; closure[1] = ornt;
2464     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2465       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2466       closure[closureSize]   = tmp[i];
2467       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2468     }
2469     if (numPoints) *numPoints = closureSize/2;
2470     if (points)    *points    = closure;
2471     PetscFunctionReturn(0);
2472   }
2473   {
2474     PetscInt c, coneSeries, s,supportSeries;
2475 
2476     c = mesh->maxConeSize;
2477     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2478     s = mesh->maxSupportSize;
2479     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2480     maxSize = 2*PetscMax(coneSeries,supportSeries);
2481   }
2482   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2483   if (*points) {
2484     closure = *points;
2485   } else {
2486     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2487   }
2488   closure[0] = p; closure[1] = ornt;
2489   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2490     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2491     const PetscInt cp = tmp[i];
2492     PetscInt       co = tmpO ? tmpO[i] : 0;
2493 
2494     if (ornt < 0) {
2495       PetscInt childSize, coff;
2496       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2497       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2498       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2499     }
2500     closure[closureSize]   = cp;
2501     closure[closureSize+1] = co;
2502     fifo[fifoSize]         = cp;
2503     fifo[fifoSize+1]       = co;
2504   }
2505   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2506   while (fifoSize - fifoStart) {
2507     const PetscInt q   = fifo[fifoStart];
2508     const PetscInt o   = fifo[fifoStart+1];
2509     const PetscInt rev = o >= 0 ? 0 : 1;
2510     const PetscInt off = rev ? -(o+1) : o;
2511 
2512     if (useCone) {
2513       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2514       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2515       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2516     } else {
2517       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2518       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2519       tmpO = NULL;
2520     }
2521     for (t = 0; t < tmpSize; ++t) {
2522       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2523       const PetscInt cp = tmp[i];
2524       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2525       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2526        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2527       PetscInt       co = tmpO ? tmpO[i] : 0;
2528       PetscInt       c;
2529 
2530       if (rev) {
2531         PetscInt childSize, coff;
2532         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2533         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2534         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2535       }
2536       /* Check for duplicate */
2537       for (c = 0; c < closureSize; c += 2) {
2538         if (closure[c] == cp) break;
2539       }
2540       if (c == closureSize) {
2541         closure[closureSize]   = cp;
2542         closure[closureSize+1] = co;
2543         fifo[fifoSize]         = cp;
2544         fifo[fifoSize+1]       = co;
2545         closureSize           += 2;
2546         fifoSize              += 2;
2547       }
2548     }
2549     fifoStart += 2;
2550   }
2551   if (numPoints) *numPoints = closureSize/2;
2552   if (points)    *points    = closure;
2553   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2554   PetscFunctionReturn(0);
2555 }
2556 
2557 /*@C
2558   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2559 
2560   Not collective
2561 
2562   Input Parameters:
2563 + mesh - The DMPlex
2564 . p - The point, which must lie in the chart set with DMPlexSetChart()
2565 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2566 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2567 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2568 
2569   Note:
2570   If not using internal storage (points is not NULL on input), this call is unnecessary
2571 
2572   Fortran Notes:
2573   Since it returns an array, this routine is only available in Fortran 90, and you must
2574   include petsc.h90 in your code.
2575 
2576   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2577 
2578   Level: beginner
2579 
2580 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2581 @*/
2582 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2583 {
2584   PetscErrorCode ierr;
2585 
2586   PetscFunctionBegin;
2587   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2588   if (numPoints) PetscValidIntPointer(numPoints,4);
2589   if (points) PetscValidPointer(points,5);
2590   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2591   if (numPoints) *numPoints = 0;
2592   PetscFunctionReturn(0);
2593 }
2594 
2595 /*@
2596   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2597 
2598   Not collective
2599 
2600   Input Parameter:
2601 . mesh - The DMPlex
2602 
2603   Output Parameters:
2604 + maxConeSize - The maximum number of in-edges
2605 - maxSupportSize - The maximum number of out-edges
2606 
2607   Level: beginner
2608 
2609 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2610 @*/
2611 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2612 {
2613   DM_Plex *mesh = (DM_Plex*) dm->data;
2614 
2615   PetscFunctionBegin;
2616   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2617   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2618   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2619   PetscFunctionReturn(0);
2620 }
2621 
2622 PetscErrorCode DMSetUp_Plex(DM dm)
2623 {
2624   DM_Plex       *mesh = (DM_Plex*) dm->data;
2625   PetscInt       size;
2626   PetscErrorCode ierr;
2627 
2628   PetscFunctionBegin;
2629   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2630   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2631   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2632   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2633   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2634   if (mesh->maxSupportSize) {
2635     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2636     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2637     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2638   }
2639   PetscFunctionReturn(0);
2640 }
2641 
2642 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2643 {
2644   PetscErrorCode ierr;
2645 
2646   PetscFunctionBegin;
2647   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2648   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2649   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2650   if (dm->useNatural && dm->sfMigration) {
2651     PetscSF        sfMigrationInv,sfNatural;
2652     PetscSection   section, sectionSeq;
2653 
2654     (*subdm)->sfMigration = dm->sfMigration;
2655     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2656     ierr = DMGetLocalSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2657     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2658     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2659     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2660 
2661     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2662     (*subdm)->sfNatural = sfNatural;
2663     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2664     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2665   }
2666   PetscFunctionReturn(0);
2667 }
2668 
2669 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2670 {
2671   PetscErrorCode ierr;
2672   PetscInt       i = 0;
2673 
2674   PetscFunctionBegin;
2675   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2676   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2677   (*superdm)->useNatural = PETSC_FALSE;
2678   for (i = 0; i < len; i++){
2679     if (dms[i]->useNatural && dms[i]->sfMigration) {
2680       PetscSF        sfMigrationInv,sfNatural;
2681       PetscSection   section, sectionSeq;
2682 
2683       (*superdm)->sfMigration = dms[i]->sfMigration;
2684       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2685       (*superdm)->useNatural = PETSC_TRUE;
2686       ierr = DMGetLocalSection((*superdm), &section);CHKERRQ(ierr);
2687       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2688       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2689       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2690 
2691       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2692       (*superdm)->sfNatural = sfNatural;
2693       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2694       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2695       break;
2696     }
2697   }
2698   PetscFunctionReturn(0);
2699 }
2700 
2701 /*@
2702   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2703 
2704   Not collective
2705 
2706   Input Parameter:
2707 . mesh - The DMPlex
2708 
2709   Output Parameter:
2710 
2711   Note:
2712   This should be called after all calls to DMPlexSetCone()
2713 
2714   Level: beginner
2715 
2716 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2717 @*/
2718 PetscErrorCode DMPlexSymmetrize(DM dm)
2719 {
2720   DM_Plex       *mesh = (DM_Plex*) dm->data;
2721   PetscInt      *offsets;
2722   PetscInt       supportSize;
2723   PetscInt       pStart, pEnd, p;
2724   PetscErrorCode ierr;
2725 
2726   PetscFunctionBegin;
2727   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2728   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2729   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2730   /* Calculate support sizes */
2731   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2732   for (p = pStart; p < pEnd; ++p) {
2733     PetscInt dof, off, c;
2734 
2735     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2736     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2737     for (c = off; c < off+dof; ++c) {
2738       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2739     }
2740   }
2741   for (p = pStart; p < pEnd; ++p) {
2742     PetscInt dof;
2743 
2744     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2745 
2746     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2747   }
2748   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2749   /* Calculate supports */
2750   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2751   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2752   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2753   for (p = pStart; p < pEnd; ++p) {
2754     PetscInt dof, off, c;
2755 
2756     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2757     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2758     for (c = off; c < off+dof; ++c) {
2759       const PetscInt q = mesh->cones[c];
2760       PetscInt       offS;
2761 
2762       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2763 
2764       mesh->supports[offS+offsets[q]] = p;
2765       ++offsets[q];
2766     }
2767   }
2768   ierr = PetscFree(offsets);CHKERRQ(ierr);
2769   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2770   PetscFunctionReturn(0);
2771 }
2772 
2773 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2774 {
2775   IS             stratumIS;
2776   PetscErrorCode ierr;
2777 
2778   PetscFunctionBegin;
2779   if (pStart >= pEnd) PetscFunctionReturn(0);
2780 #if defined(PETSC_USE_DEBUG)
2781   {
2782     PetscInt  qStart, qEnd, numLevels, level;
2783     PetscBool overlap = PETSC_FALSE;
2784     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2785     for (level = 0; level < numLevels; level++) {
2786       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2787       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2788     }
2789     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2790   }
2791 #endif
2792   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2793   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2794   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2795   PetscFunctionReturn(0);
2796 }
2797 
2798 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2799 
2800 /*@
2801   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2802   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2803   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2804   the DAG.
2805 
2806   Collective on dm
2807 
2808   Input Parameter:
2809 . mesh - The DMPlex
2810 
2811   Output Parameter:
2812 
2813   Notes:
2814   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2815   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2816   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2817   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2818 
2819   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2820 
2821   Level: beginner
2822 
2823 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2824 @*/
2825 PetscErrorCode DMPlexStratify(DM dm)
2826 {
2827   DM_Plex       *mesh = (DM_Plex*) dm->data;
2828   DMLabel        label;
2829   PetscInt       pStart, pEnd, p;
2830   PetscInt       numRoots = 0, numLeaves = 0;
2831   PetscInt       cMax, fMax, eMax, vMax;
2832   PetscErrorCode ierr;
2833 
2834   PetscFunctionBegin;
2835   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2836   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2837 
2838   /* Create depth label */
2839   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2840   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2841   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2842 
2843   {
2844     /* Initialize roots and count leaves */
2845     PetscInt sMin = PETSC_MAX_INT;
2846     PetscInt sMax = PETSC_MIN_INT;
2847     PetscInt coneSize, supportSize;
2848 
2849     for (p = pStart; p < pEnd; ++p) {
2850       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2851       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2852       if (!coneSize && supportSize) {
2853         sMin = PetscMin(p, sMin);
2854         sMax = PetscMax(p, sMax);
2855         ++numRoots;
2856       } else if (!supportSize && coneSize) {
2857         ++numLeaves;
2858       } else if (!supportSize && !coneSize) {
2859         /* Isolated points */
2860         sMin = PetscMin(p, sMin);
2861         sMax = PetscMax(p, sMax);
2862       }
2863     }
2864     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2865   }
2866 
2867   if (numRoots + numLeaves == (pEnd - pStart)) {
2868     PetscInt sMin = PETSC_MAX_INT;
2869     PetscInt sMax = PETSC_MIN_INT;
2870     PetscInt coneSize, supportSize;
2871 
2872     for (p = pStart; p < pEnd; ++p) {
2873       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2874       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2875       if (!supportSize && coneSize) {
2876         sMin = PetscMin(p, sMin);
2877         sMax = PetscMax(p, sMax);
2878       }
2879     }
2880     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2881   } else {
2882     PetscInt level = 0;
2883     PetscInt qStart, qEnd, q;
2884 
2885     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2886     while (qEnd > qStart) {
2887       PetscInt sMin = PETSC_MAX_INT;
2888       PetscInt sMax = PETSC_MIN_INT;
2889 
2890       for (q = qStart; q < qEnd; ++q) {
2891         const PetscInt *support;
2892         PetscInt        supportSize, s;
2893 
2894         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2895         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2896         for (s = 0; s < supportSize; ++s) {
2897           sMin = PetscMin(support[s], sMin);
2898           sMax = PetscMax(support[s], sMax);
2899         }
2900       }
2901       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2902       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2903       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2904     }
2905   }
2906   { /* just in case there is an empty process */
2907     PetscInt numValues, maxValues = 0, v;
2908 
2909     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2910     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2911     for (v = numValues; v < maxValues; v++) {
2912       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2913     }
2914   }
2915   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2916 
2917   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2918   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2919     PetscInt dim;
2920     DMLabel  dimLabel;
2921 
2922     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2923     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2924     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2925     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2926     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2927     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2928     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2929   }
2930   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2931   PetscFunctionReturn(0);
2932 }
2933 
2934 /*@C
2935   DMPlexGetJoin - Get an array for the join of the set of points
2936 
2937   Not Collective
2938 
2939   Input Parameters:
2940 + dm - The DMPlex object
2941 . numPoints - The number of input points for the join
2942 - points - The input points
2943 
2944   Output Parameters:
2945 + numCoveredPoints - The number of points in the join
2946 - coveredPoints - The points in the join
2947 
2948   Level: intermediate
2949 
2950   Note: Currently, this is restricted to a single level join
2951 
2952   Fortran Notes:
2953   Since it returns an array, this routine is only available in Fortran 90, and you must
2954   include petsc.h90 in your code.
2955 
2956   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2957 
2958 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2959 @*/
2960 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2961 {
2962   DM_Plex       *mesh = (DM_Plex*) dm->data;
2963   PetscInt      *join[2];
2964   PetscInt       joinSize, i = 0;
2965   PetscInt       dof, off, p, c, m;
2966   PetscErrorCode ierr;
2967 
2968   PetscFunctionBegin;
2969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2970   PetscValidIntPointer(points, 3);
2971   PetscValidIntPointer(numCoveredPoints, 4);
2972   PetscValidPointer(coveredPoints, 5);
2973   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2974   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2975   /* Copy in support of first point */
2976   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2977   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2978   for (joinSize = 0; joinSize < dof; ++joinSize) {
2979     join[i][joinSize] = mesh->supports[off+joinSize];
2980   }
2981   /* Check each successive support */
2982   for (p = 1; p < numPoints; ++p) {
2983     PetscInt newJoinSize = 0;
2984 
2985     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2986     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2987     for (c = 0; c < dof; ++c) {
2988       const PetscInt point = mesh->supports[off+c];
2989 
2990       for (m = 0; m < joinSize; ++m) {
2991         if (point == join[i][m]) {
2992           join[1-i][newJoinSize++] = point;
2993           break;
2994         }
2995       }
2996     }
2997     joinSize = newJoinSize;
2998     i        = 1-i;
2999   }
3000   *numCoveredPoints = joinSize;
3001   *coveredPoints    = join[i];
3002   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3003   PetscFunctionReturn(0);
3004 }
3005 
3006 /*@C
3007   DMPlexRestoreJoin - Restore an array for the join of the set of points
3008 
3009   Not Collective
3010 
3011   Input Parameters:
3012 + dm - The DMPlex object
3013 . numPoints - The number of input points for the join
3014 - points - The input points
3015 
3016   Output Parameters:
3017 + numCoveredPoints - The number of points in the join
3018 - coveredPoints - The points in the join
3019 
3020   Fortran Notes:
3021   Since it returns an array, this routine is only available in Fortran 90, and you must
3022   include petsc.h90 in your code.
3023 
3024   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3025 
3026   Level: intermediate
3027 
3028 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
3029 @*/
3030 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3031 {
3032   PetscErrorCode ierr;
3033 
3034   PetscFunctionBegin;
3035   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3036   if (points) PetscValidIntPointer(points,3);
3037   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3038   PetscValidPointer(coveredPoints, 5);
3039   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3040   if (numCoveredPoints) *numCoveredPoints = 0;
3041   PetscFunctionReturn(0);
3042 }
3043 
3044 /*@C
3045   DMPlexGetFullJoin - Get an array for the join of the set of points
3046 
3047   Not Collective
3048 
3049   Input Parameters:
3050 + dm - The DMPlex object
3051 . numPoints - The number of input points for the join
3052 - points - The input points
3053 
3054   Output Parameters:
3055 + numCoveredPoints - The number of points in the join
3056 - coveredPoints - The points in the join
3057 
3058   Fortran Notes:
3059   Since it returns an array, this routine is only available in Fortran 90, and you must
3060   include petsc.h90 in your code.
3061 
3062   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3063 
3064   Level: intermediate
3065 
3066 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
3067 @*/
3068 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3069 {
3070   DM_Plex       *mesh = (DM_Plex*) dm->data;
3071   PetscInt      *offsets, **closures;
3072   PetscInt      *join[2];
3073   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
3074   PetscInt       p, d, c, m, ms;
3075   PetscErrorCode ierr;
3076 
3077   PetscFunctionBegin;
3078   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3079   PetscValidIntPointer(points, 3);
3080   PetscValidIntPointer(numCoveredPoints, 4);
3081   PetscValidPointer(coveredPoints, 5);
3082 
3083   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3084   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
3085   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3086   ms      = mesh->maxSupportSize;
3087   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
3088   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
3089   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
3090 
3091   for (p = 0; p < numPoints; ++p) {
3092     PetscInt closureSize;
3093 
3094     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
3095 
3096     offsets[p*(depth+2)+0] = 0;
3097     for (d = 0; d < depth+1; ++d) {
3098       PetscInt pStart, pEnd, i;
3099 
3100       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
3101       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
3102         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3103           offsets[p*(depth+2)+d+1] = i;
3104           break;
3105         }
3106       }
3107       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
3108     }
3109     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
3110   }
3111   for (d = 0; d < depth+1; ++d) {
3112     PetscInt dof;
3113 
3114     /* Copy in support of first point */
3115     dof = offsets[d+1] - offsets[d];
3116     for (joinSize = 0; joinSize < dof; ++joinSize) {
3117       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
3118     }
3119     /* Check each successive cone */
3120     for (p = 1; p < numPoints && joinSize; ++p) {
3121       PetscInt newJoinSize = 0;
3122 
3123       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
3124       for (c = 0; c < dof; ++c) {
3125         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
3126 
3127         for (m = 0; m < joinSize; ++m) {
3128           if (point == join[i][m]) {
3129             join[1-i][newJoinSize++] = point;
3130             break;
3131           }
3132         }
3133       }
3134       joinSize = newJoinSize;
3135       i        = 1-i;
3136     }
3137     if (joinSize) break;
3138   }
3139   *numCoveredPoints = joinSize;
3140   *coveredPoints    = join[i];
3141   for (p = 0; p < numPoints; ++p) {
3142     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
3143   }
3144   ierr = PetscFree(closures);CHKERRQ(ierr);
3145   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3146   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3147   PetscFunctionReturn(0);
3148 }
3149 
3150 /*@C
3151   DMPlexGetMeet - Get an array for the meet of the set of points
3152 
3153   Not Collective
3154 
3155   Input Parameters:
3156 + dm - The DMPlex object
3157 . numPoints - The number of input points for the meet
3158 - points - The input points
3159 
3160   Output Parameters:
3161 + numCoveredPoints - The number of points in the meet
3162 - coveredPoints - The points in the meet
3163 
3164   Level: intermediate
3165 
3166   Note: Currently, this is restricted to a single level meet
3167 
3168   Fortran Notes:
3169   Since it returns an array, this routine is only available in Fortran 90, and you must
3170   include petsc.h90 in your code.
3171 
3172   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3173 
3174 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3175 @*/
3176 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3177 {
3178   DM_Plex       *mesh = (DM_Plex*) dm->data;
3179   PetscInt      *meet[2];
3180   PetscInt       meetSize, i = 0;
3181   PetscInt       dof, off, p, c, m;
3182   PetscErrorCode ierr;
3183 
3184   PetscFunctionBegin;
3185   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3186   PetscValidPointer(points, 2);
3187   PetscValidPointer(numCoveringPoints, 3);
3188   PetscValidPointer(coveringPoints, 4);
3189   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3190   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3191   /* Copy in cone of first point */
3192   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3193   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3194   for (meetSize = 0; meetSize < dof; ++meetSize) {
3195     meet[i][meetSize] = mesh->cones[off+meetSize];
3196   }
3197   /* Check each successive cone */
3198   for (p = 1; p < numPoints; ++p) {
3199     PetscInt newMeetSize = 0;
3200 
3201     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3202     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3203     for (c = 0; c < dof; ++c) {
3204       const PetscInt point = mesh->cones[off+c];
3205 
3206       for (m = 0; m < meetSize; ++m) {
3207         if (point == meet[i][m]) {
3208           meet[1-i][newMeetSize++] = point;
3209           break;
3210         }
3211       }
3212     }
3213     meetSize = newMeetSize;
3214     i        = 1-i;
3215   }
3216   *numCoveringPoints = meetSize;
3217   *coveringPoints    = meet[i];
3218   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3219   PetscFunctionReturn(0);
3220 }
3221 
3222 /*@C
3223   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3224 
3225   Not Collective
3226 
3227   Input Parameters:
3228 + dm - The DMPlex object
3229 . numPoints - The number of input points for the meet
3230 - points - The input points
3231 
3232   Output Parameters:
3233 + numCoveredPoints - The number of points in the meet
3234 - coveredPoints - The points in the meet
3235 
3236   Level: intermediate
3237 
3238   Fortran Notes:
3239   Since it returns an array, this routine is only available in Fortran 90, and you must
3240   include petsc.h90 in your code.
3241 
3242   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3243 
3244 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3245 @*/
3246 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3247 {
3248   PetscErrorCode ierr;
3249 
3250   PetscFunctionBegin;
3251   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3252   if (points) PetscValidIntPointer(points,3);
3253   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3254   PetscValidPointer(coveredPoints,5);
3255   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3256   if (numCoveredPoints) *numCoveredPoints = 0;
3257   PetscFunctionReturn(0);
3258 }
3259 
3260 /*@C
3261   DMPlexGetFullMeet - Get an array for the meet of the set of points
3262 
3263   Not Collective
3264 
3265   Input Parameters:
3266 + dm - The DMPlex object
3267 . numPoints - The number of input points for the meet
3268 - points - The input points
3269 
3270   Output Parameters:
3271 + numCoveredPoints - The number of points in the meet
3272 - coveredPoints - The points in the meet
3273 
3274   Level: intermediate
3275 
3276   Fortran Notes:
3277   Since it returns an array, this routine is only available in Fortran 90, and you must
3278   include petsc.h90 in your code.
3279 
3280   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3281 
3282 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3283 @*/
3284 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3285 {
3286   DM_Plex       *mesh = (DM_Plex*) dm->data;
3287   PetscInt      *offsets, **closures;
3288   PetscInt      *meet[2];
3289   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3290   PetscInt       p, h, c, m, mc;
3291   PetscErrorCode ierr;
3292 
3293   PetscFunctionBegin;
3294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3295   PetscValidPointer(points, 2);
3296   PetscValidPointer(numCoveredPoints, 3);
3297   PetscValidPointer(coveredPoints, 4);
3298 
3299   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3300   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3301   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3302   mc      = mesh->maxConeSize;
3303   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3304   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3305   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3306 
3307   for (p = 0; p < numPoints; ++p) {
3308     PetscInt closureSize;
3309 
3310     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3311 
3312     offsets[p*(height+2)+0] = 0;
3313     for (h = 0; h < height+1; ++h) {
3314       PetscInt pStart, pEnd, i;
3315 
3316       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3317       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3318         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3319           offsets[p*(height+2)+h+1] = i;
3320           break;
3321         }
3322       }
3323       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3324     }
3325     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3326   }
3327   for (h = 0; h < height+1; ++h) {
3328     PetscInt dof;
3329 
3330     /* Copy in cone of first point */
3331     dof = offsets[h+1] - offsets[h];
3332     for (meetSize = 0; meetSize < dof; ++meetSize) {
3333       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3334     }
3335     /* Check each successive cone */
3336     for (p = 1; p < numPoints && meetSize; ++p) {
3337       PetscInt newMeetSize = 0;
3338 
3339       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3340       for (c = 0; c < dof; ++c) {
3341         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3342 
3343         for (m = 0; m < meetSize; ++m) {
3344           if (point == meet[i][m]) {
3345             meet[1-i][newMeetSize++] = point;
3346             break;
3347           }
3348         }
3349       }
3350       meetSize = newMeetSize;
3351       i        = 1-i;
3352     }
3353     if (meetSize) break;
3354   }
3355   *numCoveredPoints = meetSize;
3356   *coveredPoints    = meet[i];
3357   for (p = 0; p < numPoints; ++p) {
3358     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3359   }
3360   ierr = PetscFree(closures);CHKERRQ(ierr);
3361   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3362   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3363   PetscFunctionReturn(0);
3364 }
3365 
3366 /*@C
3367   DMPlexEqual - Determine if two DMs have the same topology
3368 
3369   Not Collective
3370 
3371   Input Parameters:
3372 + dmA - A DMPlex object
3373 - dmB - A DMPlex object
3374 
3375   Output Parameters:
3376 . equal - PETSC_TRUE if the topologies are identical
3377 
3378   Level: intermediate
3379 
3380   Notes:
3381   We are not solving graph isomorphism, so we do not permutation.
3382 
3383 .seealso: DMPlexGetCone()
3384 @*/
3385 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3386 {
3387   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3388   PetscErrorCode ierr;
3389 
3390   PetscFunctionBegin;
3391   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3392   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3393   PetscValidPointer(equal, 3);
3394 
3395   *equal = PETSC_FALSE;
3396   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3397   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3398   if (depth != depthB) PetscFunctionReturn(0);
3399   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3400   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3401   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3402   for (p = pStart; p < pEnd; ++p) {
3403     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3404     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3405 
3406     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3407     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3408     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3409     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3410     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3411     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3412     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3413     for (c = 0; c < coneSize; ++c) {
3414       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3415       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3416     }
3417     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3418     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3419     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3420     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3421     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3422     for (s = 0; s < supportSize; ++s) {
3423       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3424     }
3425   }
3426   *equal = PETSC_TRUE;
3427   PetscFunctionReturn(0);
3428 }
3429 
3430 /*@C
3431   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3432 
3433   Not Collective
3434 
3435   Input Parameters:
3436 + dm         - The DMPlex
3437 . cellDim    - The cell dimension
3438 - numCorners - The number of vertices on a cell
3439 
3440   Output Parameters:
3441 . numFaceVertices - The number of vertices on a face
3442 
3443   Level: developer
3444 
3445   Notes:
3446   Of course this can only work for a restricted set of symmetric shapes
3447 
3448 .seealso: DMPlexGetCone()
3449 @*/
3450 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3451 {
3452   MPI_Comm       comm;
3453   PetscErrorCode ierr;
3454 
3455   PetscFunctionBegin;
3456   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3457   PetscValidPointer(numFaceVertices,3);
3458   switch (cellDim) {
3459   case 0:
3460     *numFaceVertices = 0;
3461     break;
3462   case 1:
3463     *numFaceVertices = 1;
3464     break;
3465   case 2:
3466     switch (numCorners) {
3467     case 3: /* triangle */
3468       *numFaceVertices = 2; /* Edge has 2 vertices */
3469       break;
3470     case 4: /* quadrilateral */
3471       *numFaceVertices = 2; /* Edge has 2 vertices */
3472       break;
3473     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3474       *numFaceVertices = 3; /* Edge has 3 vertices */
3475       break;
3476     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3477       *numFaceVertices = 3; /* Edge has 3 vertices */
3478       break;
3479     default:
3480       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3481     }
3482     break;
3483   case 3:
3484     switch (numCorners) {
3485     case 4: /* tetradehdron */
3486       *numFaceVertices = 3; /* Face has 3 vertices */
3487       break;
3488     case 6: /* tet cohesive cells */
3489       *numFaceVertices = 4; /* Face has 4 vertices */
3490       break;
3491     case 8: /* hexahedron */
3492       *numFaceVertices = 4; /* Face has 4 vertices */
3493       break;
3494     case 9: /* tet cohesive Lagrange cells */
3495       *numFaceVertices = 6; /* Face has 6 vertices */
3496       break;
3497     case 10: /* quadratic tetrahedron */
3498       *numFaceVertices = 6; /* Face has 6 vertices */
3499       break;
3500     case 12: /* hex cohesive Lagrange cells */
3501       *numFaceVertices = 6; /* Face has 6 vertices */
3502       break;
3503     case 18: /* quadratic tet cohesive Lagrange cells */
3504       *numFaceVertices = 6; /* Face has 6 vertices */
3505       break;
3506     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3507       *numFaceVertices = 9; /* Face has 9 vertices */
3508       break;
3509     default:
3510       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3511     }
3512     break;
3513   default:
3514     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3515   }
3516   PetscFunctionReturn(0);
3517 }
3518 
3519 /*@
3520   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3521 
3522   Not Collective
3523 
3524   Input Parameter:
3525 . dm    - The DMPlex object
3526 
3527   Output Parameter:
3528 . depthLabel - The DMLabel recording point depth
3529 
3530   Level: developer
3531 
3532 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3533 @*/
3534 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3535 {
3536   PetscFunctionBegin;
3537   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3538   PetscValidPointer(depthLabel, 2);
3539   *depthLabel = dm->depthLabel;
3540   PetscFunctionReturn(0);
3541 }
3542 
3543 /*@
3544   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3545 
3546   Not Collective
3547 
3548   Input Parameter:
3549 . dm    - The DMPlex object
3550 
3551   Output Parameter:
3552 . depth - The number of strata (breadth first levels) in the DAG
3553 
3554   Level: developer
3555 
3556 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3557 @*/
3558 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3559 {
3560   DMLabel        label;
3561   PetscInt       d = 0;
3562   PetscErrorCode ierr;
3563 
3564   PetscFunctionBegin;
3565   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3566   PetscValidPointer(depth, 2);
3567   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3568   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3569   *depth = d-1;
3570   PetscFunctionReturn(0);
3571 }
3572 
3573 /*@
3574   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3575 
3576   Not Collective
3577 
3578   Input Parameters:
3579 + dm           - The DMPlex object
3580 - stratumValue - The requested depth
3581 
3582   Output Parameters:
3583 + start - The first point at this depth
3584 - end   - One beyond the last point at this depth
3585 
3586   Notes:
3587   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3588   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3589   higher dimension, e.g., "edges".
3590 
3591   Level: developer
3592 
3593 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3594 @*/
3595 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3596 {
3597   DMLabel        label;
3598   PetscInt       pStart, pEnd;
3599   PetscErrorCode ierr;
3600 
3601   PetscFunctionBegin;
3602   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3603   if (start) {PetscValidPointer(start, 3); *start = 0;}
3604   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3605   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3606   if (pStart == pEnd) PetscFunctionReturn(0);
3607   if (stratumValue < 0) {
3608     if (start) *start = pStart;
3609     if (end)   *end   = pEnd;
3610     PetscFunctionReturn(0);
3611   }
3612   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3613   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3614   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3615   PetscFunctionReturn(0);
3616 }
3617 
3618 /*@
3619   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3620 
3621   Not Collective
3622 
3623   Input Parameters:
3624 + dm           - The DMPlex object
3625 - stratumValue - The requested height
3626 
3627   Output Parameters:
3628 + start - The first point at this height
3629 - end   - One beyond the last point at this height
3630 
3631   Notes:
3632   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3633   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3634   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3635 
3636   Level: developer
3637 
3638 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3639 @*/
3640 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3641 {
3642   DMLabel        label;
3643   PetscInt       depth, pStart, pEnd;
3644   PetscErrorCode ierr;
3645 
3646   PetscFunctionBegin;
3647   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3648   if (start) {PetscValidPointer(start, 3); *start = 0;}
3649   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3650   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3651   if (pStart == pEnd) PetscFunctionReturn(0);
3652   if (stratumValue < 0) {
3653     if (start) *start = pStart;
3654     if (end)   *end   = pEnd;
3655     PetscFunctionReturn(0);
3656   }
3657   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3658   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3659   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3660   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3661   PetscFunctionReturn(0);
3662 }
3663 
3664 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3665 {
3666   PetscSection   section, s;
3667   Mat            m;
3668   PetscInt       maxHeight;
3669   PetscErrorCode ierr;
3670 
3671   PetscFunctionBegin;
3672   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3673   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3674   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3675   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3676   ierr = DMSetLocalSection(*cdm, section);CHKERRQ(ierr);
3677   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3678   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3679   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3680   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3681   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3682   ierr = MatDestroy(&m);CHKERRQ(ierr);
3683 
3684   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3685   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3686   PetscFunctionReturn(0);
3687 }
3688 
3689 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3690 {
3691   Vec            coordsLocal;
3692   DM             coordsDM;
3693   PetscErrorCode ierr;
3694 
3695   PetscFunctionBegin;
3696   *field = NULL;
3697   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3698   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3699   if (coordsLocal && coordsDM) {
3700     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3701   }
3702   PetscFunctionReturn(0);
3703 }
3704 
3705 /*@C
3706   DMPlexGetConeSection - Return a section which describes the layout of cone data
3707 
3708   Not Collective
3709 
3710   Input Parameters:
3711 . dm        - The DMPlex object
3712 
3713   Output Parameter:
3714 . section - The PetscSection object
3715 
3716   Level: developer
3717 
3718 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3719 @*/
3720 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3721 {
3722   DM_Plex *mesh = (DM_Plex*) dm->data;
3723 
3724   PetscFunctionBegin;
3725   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3726   if (section) *section = mesh->coneSection;
3727   PetscFunctionReturn(0);
3728 }
3729 
3730 /*@C
3731   DMPlexGetSupportSection - Return a section which describes the layout of support data
3732 
3733   Not Collective
3734 
3735   Input Parameters:
3736 . dm        - The DMPlex object
3737 
3738   Output Parameter:
3739 . section - The PetscSection object
3740 
3741   Level: developer
3742 
3743 .seealso: DMPlexGetConeSection()
3744 @*/
3745 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3746 {
3747   DM_Plex *mesh = (DM_Plex*) dm->data;
3748 
3749   PetscFunctionBegin;
3750   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3751   if (section) *section = mesh->supportSection;
3752   PetscFunctionReturn(0);
3753 }
3754 
3755 /*@C
3756   DMPlexGetCones - Return cone data
3757 
3758   Not Collective
3759 
3760   Input Parameters:
3761 . dm        - The DMPlex object
3762 
3763   Output Parameter:
3764 . cones - The cone for each point
3765 
3766   Level: developer
3767 
3768 .seealso: DMPlexGetConeSection()
3769 @*/
3770 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3771 {
3772   DM_Plex *mesh = (DM_Plex*) dm->data;
3773 
3774   PetscFunctionBegin;
3775   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3776   if (cones) *cones = mesh->cones;
3777   PetscFunctionReturn(0);
3778 }
3779 
3780 /*@C
3781   DMPlexGetConeOrientations - Return cone orientation data
3782 
3783   Not Collective
3784 
3785   Input Parameters:
3786 . dm        - The DMPlex object
3787 
3788   Output Parameter:
3789 . coneOrientations - The cone orientation for each point
3790 
3791   Level: developer
3792 
3793 .seealso: DMPlexGetConeSection()
3794 @*/
3795 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3796 {
3797   DM_Plex *mesh = (DM_Plex*) dm->data;
3798 
3799   PetscFunctionBegin;
3800   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3801   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3802   PetscFunctionReturn(0);
3803 }
3804 
3805 /******************************** FEM Support **********************************/
3806 
3807 /*
3808  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3809  representing a line in the section.
3810 */
3811 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3812 {
3813   PetscErrorCode ierr;
3814 
3815   PetscFunctionBeginHot;
3816   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3817   if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3818     *k = 1;
3819   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3820     /* An order k SEM disc has k-1 dofs on an edge */
3821     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3822     *k = *k / *Nc + 1;
3823   }
3824   PetscFunctionReturn(0);
3825 }
3826 
3827 /*@
3828 
3829   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3830   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3831   section provided (or the section of the DM).
3832 
3833   Input Parameters:
3834 + dm      - The DM
3835 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3836 - section - The PetscSection to reorder, or NULL for the default section
3837 
3838   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3839   degree of the basis.
3840 
3841   Example:
3842   A typical interpolated single-quad mesh might order points as
3843 .vb
3844   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3845 
3846   v4 -- e6 -- v3
3847   |           |
3848   e7    c0    e8
3849   |           |
3850   v1 -- e5 -- v2
3851 .ve
3852 
3853   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3854   dofs in the order of points, e.g.,
3855 .vb
3856     c0 -> [0,1,2,3]
3857     v1 -> [4]
3858     ...
3859     e5 -> [8, 9]
3860 .ve
3861 
3862   which corresponds to the dofs
3863 .vb
3864     6   10  11  7
3865     13  2   3   15
3866     12  0   1   14
3867     4   8   9   5
3868 .ve
3869 
3870   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3871 .vb
3872   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3873 .ve
3874 
3875   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3876 .vb
3877    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3878 .ve
3879 
3880   Level: developer
3881 
3882 .seealso: DMGetLocalSection(), PetscSectionSetClosurePermutation(), DMSetGlocalSection()
3883 @*/
3884 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3885 {
3886   DMLabel        label;
3887   PetscInt      *perm;
3888   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3889   PetscBool      vertexchart;
3890   PetscErrorCode ierr;
3891 
3892   PetscFunctionBegin;
3893   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3894   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3895   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3896   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3897   if (depth == 1) {eStart = point;}
3898   else if  (depth == dim) {
3899     const PetscInt *cone;
3900 
3901     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3902     if (dim == 2) eStart = cone[0];
3903     else if (dim == 3) {
3904       const PetscInt *cone2;
3905       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3906       eStart = cone2[0];
3907     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3908   } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3909   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
3910   {                             /* Determine whether the chart covers all points or just vertices. */
3911     PetscInt pStart,pEnd,cStart,cEnd;
3912     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3913     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3914     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3915     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3916   }
3917   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3918   if (dim < 1) PetscFunctionReturn(0);
3919   for (f = 0; f < Nf; ++f) {
3920     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3921     size += PetscPowInt(k+1, dim)*Nc;
3922   }
3923   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3924   for (f = 0; f < Nf; ++f) {
3925     switch (dim) {
3926     case 1:
3927       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3928       /*
3929         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3930         We want              [ vtx0; edge of length k-1; vtx1 ]
3931       */
3932       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3933       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3934       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3935       foffset = offset;
3936       break;
3937     case 2:
3938       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3939       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3940       /* The SEM order is
3941 
3942          v_lb, {e_b}, v_rb,
3943          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3944          v_lt, reverse {e_t}, v_rt
3945       */
3946       {
3947         const PetscInt of   = 0;
3948         const PetscInt oeb  = of   + PetscSqr(k-1);
3949         const PetscInt oer  = oeb  + (k-1);
3950         const PetscInt oet  = oer  + (k-1);
3951         const PetscInt oel  = oet  + (k-1);
3952         const PetscInt ovlb = oel  + (k-1);
3953         const PetscInt ovrb = ovlb + 1;
3954         const PetscInt ovrt = ovrb + 1;
3955         const PetscInt ovlt = ovrt + 1;
3956         PetscInt       o;
3957 
3958         /* bottom */
3959         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3960         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3961         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3962         /* middle */
3963         for (i = 0; i < k-1; ++i) {
3964           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3965           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3966           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3967         }
3968         /* top */
3969         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3970         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3971         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3972         foffset = offset;
3973       }
3974       break;
3975     case 3:
3976       /* The original hex closure is
3977 
3978          {c,
3979           f_b, f_t, f_f, f_b, f_r, f_l,
3980           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3981           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3982       */
3983       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3984       /* The SEM order is
3985          Bottom Slice
3986          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3987          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3988          v_blb, {e_bb}, v_brb,
3989 
3990          Middle Slice (j)
3991          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3992          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3993          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3994 
3995          Top Slice
3996          v_tlf, {e_tf}, v_trf,
3997          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
3998          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
3999       */
4000       {
4001         const PetscInt oc    = 0;
4002         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
4003         const PetscInt oft   = ofb   + PetscSqr(k-1);
4004         const PetscInt off   = oft   + PetscSqr(k-1);
4005         const PetscInt ofk   = off   + PetscSqr(k-1);
4006         const PetscInt ofr   = ofk   + PetscSqr(k-1);
4007         const PetscInt ofl   = ofr   + PetscSqr(k-1);
4008         const PetscInt oebl  = ofl   + PetscSqr(k-1);
4009         const PetscInt oebb  = oebl  + (k-1);
4010         const PetscInt oebr  = oebb  + (k-1);
4011         const PetscInt oebf  = oebr  + (k-1);
4012         const PetscInt oetf  = oebf  + (k-1);
4013         const PetscInt oetr  = oetf  + (k-1);
4014         const PetscInt oetb  = oetr  + (k-1);
4015         const PetscInt oetl  = oetb  + (k-1);
4016         const PetscInt oerf  = oetl  + (k-1);
4017         const PetscInt oelf  = oerf  + (k-1);
4018         const PetscInt oelb  = oelf  + (k-1);
4019         const PetscInt oerb  = oelb  + (k-1);
4020         const PetscInt ovblf = oerb  + (k-1);
4021         const PetscInt ovblb = ovblf + 1;
4022         const PetscInt ovbrb = ovblb + 1;
4023         const PetscInt ovbrf = ovbrb + 1;
4024         const PetscInt ovtlf = ovbrf + 1;
4025         const PetscInt ovtrf = ovtlf + 1;
4026         const PetscInt ovtrb = ovtrf + 1;
4027         const PetscInt ovtlb = ovtrb + 1;
4028         PetscInt       o, n;
4029 
4030         /* Bottom Slice */
4031         /*   bottom */
4032         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
4033         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4034         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
4035         /*   middle */
4036         for (i = 0; i < k-1; ++i) {
4037           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
4038           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
4039           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
4040         }
4041         /*   top */
4042         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
4043         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4044         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
4045 
4046         /* Middle Slice */
4047         for (j = 0; j < k-1; ++j) {
4048           /*   bottom */
4049           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
4050           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4051           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
4052           /*   middle */
4053           for (i = 0; i < k-1; ++i) {
4054             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
4055             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
4056             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
4057           }
4058           /*   top */
4059           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
4060           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4061           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
4062         }
4063 
4064         /* Top Slice */
4065         /*   bottom */
4066         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
4067         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4068         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
4069         /*   middle */
4070         for (i = 0; i < k-1; ++i) {
4071           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
4072           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
4073           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
4074         }
4075         /*   top */
4076         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
4077         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
4078         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
4079 
4080         foffset = offset;
4081       }
4082       break;
4083     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
4084     }
4085   }
4086   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
4087   /* Check permutation */
4088   {
4089     PetscInt *check;
4090 
4091     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
4092     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
4093     for (i = 0; i < size; ++i) check[perm[i]] = i;
4094     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
4095     ierr = PetscFree(check);CHKERRQ(ierr);
4096   }
4097   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
4098   PetscFunctionReturn(0);
4099 }
4100 
4101 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
4102 {
4103   PetscDS        prob;
4104   PetscInt       depth, Nf, h;
4105   DMLabel        label;
4106   PetscErrorCode ierr;
4107 
4108   PetscFunctionBeginHot;
4109   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
4110   Nf      = prob->Nf;
4111   label   = dm->depthLabel;
4112   *dspace = NULL;
4113   if (field < Nf) {
4114     PetscObject disc = prob->disc[field];
4115 
4116     if (disc->classid == PETSCFE_CLASSID) {
4117       PetscDualSpace dsp;
4118 
4119       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
4120       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
4121       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
4122       h    = depth - 1 - h;
4123       if (h) {
4124         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
4125       } else {
4126         *dspace = dsp;
4127       }
4128     }
4129   }
4130   PetscFunctionReturn(0);
4131 }
4132 
4133 
4134 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4135 {
4136   PetscScalar    *array, *vArray;
4137   const PetscInt *cone, *coneO;
4138   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
4139   PetscErrorCode  ierr;
4140 
4141   PetscFunctionBeginHot;
4142   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4143   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4144   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4145   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4146   if (!values || !*values) {
4147     if ((point >= pStart) && (point < pEnd)) {
4148       PetscInt dof;
4149 
4150       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4151       size += dof;
4152     }
4153     for (p = 0; p < numPoints; ++p) {
4154       const PetscInt cp = cone[p];
4155       PetscInt       dof;
4156 
4157       if ((cp < pStart) || (cp >= pEnd)) continue;
4158       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4159       size += dof;
4160     }
4161     if (!values) {
4162       if (csize) *csize = size;
4163       PetscFunctionReturn(0);
4164     }
4165     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4166   } else {
4167     array = *values;
4168   }
4169   size = 0;
4170   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4171   if ((point >= pStart) && (point < pEnd)) {
4172     PetscInt     dof, off, d;
4173     PetscScalar *varr;
4174 
4175     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4176     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4177     varr = &vArray[off];
4178     for (d = 0; d < dof; ++d, ++offset) {
4179       array[offset] = varr[d];
4180     }
4181     size += dof;
4182   }
4183   for (p = 0; p < numPoints; ++p) {
4184     const PetscInt cp = cone[p];
4185     PetscInt       o  = coneO[p];
4186     PetscInt       dof, off, d;
4187     PetscScalar   *varr;
4188 
4189     if ((cp < pStart) || (cp >= pEnd)) continue;
4190     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4191     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4192     varr = &vArray[off];
4193     if (o >= 0) {
4194       for (d = 0; d < dof; ++d, ++offset) {
4195         array[offset] = varr[d];
4196       }
4197     } else {
4198       for (d = dof-1; d >= 0; --d, ++offset) {
4199         array[offset] = varr[d];
4200       }
4201     }
4202     size += dof;
4203   }
4204   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4205   if (!*values) {
4206     if (csize) *csize = size;
4207     *values = array;
4208   } else {
4209     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4210     *csize = size;
4211   }
4212   PetscFunctionReturn(0);
4213 }
4214 
4215 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4216 {
4217   const PetscInt *cla;
4218   PetscInt       np, *pts = NULL;
4219   PetscErrorCode ierr;
4220 
4221   PetscFunctionBeginHot;
4222   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4223   if (!*clPoints) {
4224     PetscInt pStart, pEnd, p, q;
4225 
4226     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4227     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4228     /* Compress out points not in the section */
4229     for (p = 0, q = 0; p < np; p++) {
4230       PetscInt r = pts[2*p];
4231       if ((r >= pStart) && (r < pEnd)) {
4232         pts[q*2]   = r;
4233         pts[q*2+1] = pts[2*p+1];
4234         ++q;
4235       }
4236     }
4237     np = q;
4238     cla = NULL;
4239   } else {
4240     PetscInt dof, off;
4241 
4242     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4243     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4244     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4245     np   = dof/2;
4246     pts  = (PetscInt *) &cla[off];
4247   }
4248   *numPoints = np;
4249   *points    = pts;
4250   *clp       = cla;
4251 
4252   PetscFunctionReturn(0);
4253 }
4254 
4255 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4256 {
4257   PetscErrorCode ierr;
4258 
4259   PetscFunctionBeginHot;
4260   if (!*clPoints) {
4261     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4262   } else {
4263     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4264   }
4265   *numPoints = 0;
4266   *points    = NULL;
4267   *clSec     = NULL;
4268   *clPoints  = NULL;
4269   *clp       = NULL;
4270   PetscFunctionReturn(0);
4271 }
4272 
4273 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4274 {
4275   PetscInt          offset = 0, p;
4276   const PetscInt    **perms = NULL;
4277   const PetscScalar **flips = NULL;
4278   PetscErrorCode    ierr;
4279 
4280   PetscFunctionBeginHot;
4281   *size = 0;
4282   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4283   for (p = 0; p < numPoints; p++) {
4284     const PetscInt    point = points[2*p];
4285     const PetscInt    *perm = perms ? perms[p] : NULL;
4286     const PetscScalar *flip = flips ? flips[p] : NULL;
4287     PetscInt          dof, off, d;
4288     const PetscScalar *varr;
4289 
4290     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4291     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4292     varr = &vArray[off];
4293     if (clperm) {
4294       if (perm) {
4295         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4296       } else {
4297         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4298       }
4299       if (flip) {
4300         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4301       }
4302     } else {
4303       if (perm) {
4304         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4305       } else {
4306         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4307       }
4308       if (flip) {
4309         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4310       }
4311     }
4312     offset += dof;
4313   }
4314   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4315   *size = offset;
4316   PetscFunctionReturn(0);
4317 }
4318 
4319 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4320 {
4321   PetscInt          offset = 0, f;
4322   PetscErrorCode    ierr;
4323 
4324   PetscFunctionBeginHot;
4325   *size = 0;
4326   for (f = 0; f < numFields; ++f) {
4327     PetscInt          p;
4328     const PetscInt    **perms = NULL;
4329     const PetscScalar **flips = NULL;
4330 
4331     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4332     for (p = 0; p < numPoints; p++) {
4333       const PetscInt    point = points[2*p];
4334       PetscInt          fdof, foff, b;
4335       const PetscScalar *varr;
4336       const PetscInt    *perm = perms ? perms[p] : NULL;
4337       const PetscScalar *flip = flips ? flips[p] : NULL;
4338 
4339       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4340       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4341       varr = &vArray[foff];
4342       if (clperm) {
4343         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4344         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4345         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4346       } else {
4347         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4348         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4349         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4350       }
4351       offset += fdof;
4352     }
4353     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4354   }
4355   *size = offset;
4356   PetscFunctionReturn(0);
4357 }
4358 
4359 /*@C
4360   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4361 
4362   Not collective
4363 
4364   Input Parameters:
4365 + dm - The DM
4366 . section - The section describing the layout in v, or NULL to use the default section
4367 . v - The local vector
4368 . point - The point in the DM
4369 . csize - The size of the input values array, or NULL
4370 - values - An array to use for the values, or NULL to have it allocated automatically
4371 
4372   Output Parameters:
4373 + csize - The number of values in the closure
4374 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4375 
4376 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4377 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4378 $ assembly function, and a user may already have allocated storage for this operation.
4379 $
4380 $ A typical use could be
4381 $
4382 $  values = NULL;
4383 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4384 $  for (cl = 0; cl < clSize; ++cl) {
4385 $    <Compute on closure>
4386 $  }
4387 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4388 $
4389 $ or
4390 $
4391 $  PetscMalloc1(clMaxSize, &values);
4392 $  for (p = pStart; p < pEnd; ++p) {
4393 $    clSize = clMaxSize;
4394 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4395 $    for (cl = 0; cl < clSize; ++cl) {
4396 $      <Compute on closure>
4397 $    }
4398 $  }
4399 $  PetscFree(values);
4400 
4401   Fortran Notes:
4402   Since it returns an array, this routine is only available in Fortran 90, and you must
4403   include petsc.h90 in your code.
4404 
4405   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4406 
4407   Level: intermediate
4408 
4409 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4410 @*/
4411 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4412 {
4413   PetscSection       clSection;
4414   IS                 clPoints;
4415   PetscScalar       *array;
4416   const PetscScalar *vArray;
4417   PetscInt          *points = NULL;
4418   const PetscInt    *clp, *perm;
4419   PetscInt           depth, numFields, numPoints, size;
4420   PetscErrorCode     ierr;
4421 
4422   PetscFunctionBeginHot;
4423   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4424   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4425   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4426   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4427   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4428   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4429   if (depth == 1 && numFields < 2) {
4430     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4431     PetscFunctionReturn(0);
4432   }
4433   /* Get points */
4434   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4435   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4436   /* Get array */
4437   if (!values || !*values) {
4438     PetscInt asize = 0, dof, p;
4439 
4440     for (p = 0; p < numPoints*2; p += 2) {
4441       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4442       asize += dof;
4443     }
4444     if (!values) {
4445       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4446       if (csize) *csize = asize;
4447       PetscFunctionReturn(0);
4448     }
4449     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4450   } else {
4451     array = *values;
4452   }
4453   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4454   /* Get values */
4455   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4456   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4457   /* Cleanup points */
4458   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4459   /* Cleanup array */
4460   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4461   if (!*values) {
4462     if (csize) *csize = size;
4463     *values = array;
4464   } else {
4465     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4466     *csize = size;
4467   }
4468   PetscFunctionReturn(0);
4469 }
4470 
4471 /*@C
4472   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4473 
4474   Not collective
4475 
4476   Input Parameters:
4477 + dm - The DM
4478 . section - The section describing the layout in v, or NULL to use the default section
4479 . v - The local vector
4480 . point - The point in the DM
4481 . csize - The number of values in the closure, or NULL
4482 - values - The array of values, which is a borrowed array and should not be freed
4483 
4484   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4485 
4486   Fortran Notes:
4487   Since it returns an array, this routine is only available in Fortran 90, and you must
4488   include petsc.h90 in your code.
4489 
4490   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4491 
4492   Level: intermediate
4493 
4494 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4495 @*/
4496 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4497 {
4498   PetscInt       size = 0;
4499   PetscErrorCode ierr;
4500 
4501   PetscFunctionBegin;
4502   /* Should work without recalculating size */
4503   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4504   *values = NULL;
4505   PetscFunctionReturn(0);
4506 }
4507 
4508 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4509 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4510 
4511 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4512 {
4513   PetscInt        cdof;   /* The number of constraints on this point */
4514   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4515   PetscScalar    *a;
4516   PetscInt        off, cind = 0, k;
4517   PetscErrorCode  ierr;
4518 
4519   PetscFunctionBegin;
4520   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4521   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4522   a    = &array[off];
4523   if (!cdof || setBC) {
4524     if (clperm) {
4525       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4526       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4527     } else {
4528       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4529       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4530     }
4531   } else {
4532     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4533     if (clperm) {
4534       if (perm) {for (k = 0; k < dof; ++k) {
4535           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4536           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4537         }
4538       } else {
4539         for (k = 0; k < dof; ++k) {
4540           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4541           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4542         }
4543       }
4544     } else {
4545       if (perm) {
4546         for (k = 0; k < dof; ++k) {
4547           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4548           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4549         }
4550       } else {
4551         for (k = 0; k < dof; ++k) {
4552           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4553           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4554         }
4555       }
4556     }
4557   }
4558   PetscFunctionReturn(0);
4559 }
4560 
4561 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4562 {
4563   PetscInt        cdof;   /* The number of constraints on this point */
4564   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4565   PetscScalar    *a;
4566   PetscInt        off, cind = 0, k;
4567   PetscErrorCode  ierr;
4568 
4569   PetscFunctionBegin;
4570   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4571   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4572   a    = &array[off];
4573   if (cdof) {
4574     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4575     if (clperm) {
4576       if (perm) {
4577         for (k = 0; k < dof; ++k) {
4578           if ((cind < cdof) && (k == cdofs[cind])) {
4579             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4580             cind++;
4581           }
4582         }
4583       } else {
4584         for (k = 0; k < dof; ++k) {
4585           if ((cind < cdof) && (k == cdofs[cind])) {
4586             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4587             cind++;
4588           }
4589         }
4590       }
4591     } else {
4592       if (perm) {
4593         for (k = 0; k < dof; ++k) {
4594           if ((cind < cdof) && (k == cdofs[cind])) {
4595             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4596             cind++;
4597           }
4598         }
4599       } else {
4600         for (k = 0; k < dof; ++k) {
4601           if ((cind < cdof) && (k == cdofs[cind])) {
4602             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4603             cind++;
4604           }
4605         }
4606       }
4607     }
4608   }
4609   PetscFunctionReturn(0);
4610 }
4611 
4612 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4613 {
4614   PetscScalar    *a;
4615   PetscInt        fdof, foff, fcdof, foffset = *offset;
4616   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4617   PetscInt        cind = 0, b;
4618   PetscErrorCode  ierr;
4619 
4620   PetscFunctionBegin;
4621   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4622   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4623   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4624   a    = &array[foff];
4625   if (!fcdof || setBC) {
4626     if (clperm) {
4627       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4628       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4629     } else {
4630       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4631       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4632     }
4633   } else {
4634     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4635     if (clperm) {
4636       if (perm) {
4637         for (b = 0; b < fdof; b++) {
4638           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4639           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4640         }
4641       } else {
4642         for (b = 0; b < fdof; b++) {
4643           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4644           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4645         }
4646       }
4647     } else {
4648       if (perm) {
4649         for (b = 0; b < fdof; b++) {
4650           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4651           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4652         }
4653       } else {
4654         for (b = 0; b < fdof; b++) {
4655           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4656           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4657         }
4658       }
4659     }
4660   }
4661   *offset += fdof;
4662   PetscFunctionReturn(0);
4663 }
4664 
4665 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4666 {
4667   PetscScalar    *a;
4668   PetscInt        fdof, foff, fcdof, foffset = *offset;
4669   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4670   PetscInt        cind = 0, ncind = 0, b;
4671   PetscBool       ncSet, fcSet;
4672   PetscErrorCode  ierr;
4673 
4674   PetscFunctionBegin;
4675   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4676   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4677   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4678   a    = &array[foff];
4679   if (fcdof) {
4680     /* We just override fcdof and fcdofs with Ncc and comps */
4681     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4682     if (clperm) {
4683       if (perm) {
4684         if (comps) {
4685           for (b = 0; b < fdof; b++) {
4686             ncSet = fcSet = PETSC_FALSE;
4687             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4688             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4689             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4690           }
4691         } else {
4692           for (b = 0; b < fdof; b++) {
4693             if ((cind < fcdof) && (b == fcdofs[cind])) {
4694               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4695               ++cind;
4696             }
4697           }
4698         }
4699       } else {
4700         if (comps) {
4701           for (b = 0; b < fdof; b++) {
4702             ncSet = fcSet = PETSC_FALSE;
4703             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4704             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4705             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4706           }
4707         } else {
4708           for (b = 0; b < fdof; b++) {
4709             if ((cind < fcdof) && (b == fcdofs[cind])) {
4710               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4711               ++cind;
4712             }
4713           }
4714         }
4715       }
4716     } else {
4717       if (perm) {
4718         if (comps) {
4719           for (b = 0; b < fdof; b++) {
4720             ncSet = fcSet = PETSC_FALSE;
4721             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4722             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4723             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4724           }
4725         } else {
4726           for (b = 0; b < fdof; b++) {
4727             if ((cind < fcdof) && (b == fcdofs[cind])) {
4728               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4729               ++cind;
4730             }
4731           }
4732         }
4733       } else {
4734         if (comps) {
4735           for (b = 0; b < fdof; b++) {
4736             ncSet = fcSet = PETSC_FALSE;
4737             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4738             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4739             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4740           }
4741         } else {
4742           for (b = 0; b < fdof; b++) {
4743             if ((cind < fcdof) && (b == fcdofs[cind])) {
4744               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4745               ++cind;
4746             }
4747           }
4748         }
4749       }
4750     }
4751   }
4752   *offset += fdof;
4753   PetscFunctionReturn(0);
4754 }
4755 
4756 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4757 {
4758   PetscScalar    *array;
4759   const PetscInt *cone, *coneO;
4760   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4761   PetscErrorCode  ierr;
4762 
4763   PetscFunctionBeginHot;
4764   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4765   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4766   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4767   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4768   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4769   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4770     const PetscInt cp = !p ? point : cone[p-1];
4771     const PetscInt o  = !p ? 0     : coneO[p-1];
4772 
4773     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4774     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4775     /* ADD_VALUES */
4776     {
4777       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4778       PetscScalar    *a;
4779       PetscInt        cdof, coff, cind = 0, k;
4780 
4781       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4782       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4783       a    = &array[coff];
4784       if (!cdof) {
4785         if (o >= 0) {
4786           for (k = 0; k < dof; ++k) {
4787             a[k] += values[off+k];
4788           }
4789         } else {
4790           for (k = 0; k < dof; ++k) {
4791             a[k] += values[off+dof-k-1];
4792           }
4793         }
4794       } else {
4795         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4796         if (o >= 0) {
4797           for (k = 0; k < dof; ++k) {
4798             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4799             a[k] += values[off+k];
4800           }
4801         } else {
4802           for (k = 0; k < dof; ++k) {
4803             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4804             a[k] += values[off+dof-k-1];
4805           }
4806         }
4807       }
4808     }
4809   }
4810   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4811   PetscFunctionReturn(0);
4812 }
4813 
4814 /*@C
4815   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4816 
4817   Not collective
4818 
4819   Input Parameters:
4820 + dm - The DM
4821 . section - The section describing the layout in v, or NULL to use the default section
4822 . v - The local vector
4823 . point - The point in the DM
4824 . values - The array of values
4825 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4826          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4827 
4828   Fortran Notes:
4829   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4830 
4831   Level: intermediate
4832 
4833 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4834 @*/
4835 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4836 {
4837   PetscSection    clSection;
4838   IS              clPoints;
4839   PetscScalar    *array;
4840   PetscInt       *points = NULL;
4841   const PetscInt *clp, *clperm;
4842   PetscInt        depth, numFields, numPoints, p;
4843   PetscErrorCode  ierr;
4844 
4845   PetscFunctionBeginHot;
4846   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4847   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4848   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4849   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4850   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4851   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4852   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4853     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4854     PetscFunctionReturn(0);
4855   }
4856   /* Get points */
4857   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4858   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4859   /* Get array */
4860   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4861   /* Get values */
4862   if (numFields > 0) {
4863     PetscInt offset = 0, f;
4864     for (f = 0; f < numFields; ++f) {
4865       const PetscInt    **perms = NULL;
4866       const PetscScalar **flips = NULL;
4867 
4868       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4869       switch (mode) {
4870       case INSERT_VALUES:
4871         for (p = 0; p < numPoints; p++) {
4872           const PetscInt    point = points[2*p];
4873           const PetscInt    *perm = perms ? perms[p] : NULL;
4874           const PetscScalar *flip = flips ? flips[p] : NULL;
4875           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4876         } break;
4877       case INSERT_ALL_VALUES:
4878         for (p = 0; p < numPoints; p++) {
4879           const PetscInt    point = points[2*p];
4880           const PetscInt    *perm = perms ? perms[p] : NULL;
4881           const PetscScalar *flip = flips ? flips[p] : NULL;
4882           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4883         } break;
4884       case INSERT_BC_VALUES:
4885         for (p = 0; p < numPoints; p++) {
4886           const PetscInt    point = points[2*p];
4887           const PetscInt    *perm = perms ? perms[p] : NULL;
4888           const PetscScalar *flip = flips ? flips[p] : NULL;
4889           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4890         } break;
4891       case ADD_VALUES:
4892         for (p = 0; p < numPoints; p++) {
4893           const PetscInt    point = points[2*p];
4894           const PetscInt    *perm = perms ? perms[p] : NULL;
4895           const PetscScalar *flip = flips ? flips[p] : NULL;
4896           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4897         } break;
4898       case ADD_ALL_VALUES:
4899         for (p = 0; p < numPoints; p++) {
4900           const PetscInt    point = points[2*p];
4901           const PetscInt    *perm = perms ? perms[p] : NULL;
4902           const PetscScalar *flip = flips ? flips[p] : NULL;
4903           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4904         } break;
4905       case ADD_BC_VALUES:
4906         for (p = 0; p < numPoints; p++) {
4907           const PetscInt    point = points[2*p];
4908           const PetscInt    *perm = perms ? perms[p] : NULL;
4909           const PetscScalar *flip = flips ? flips[p] : NULL;
4910           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4911         } break;
4912       default:
4913         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4914       }
4915       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4916     }
4917   } else {
4918     PetscInt dof, off;
4919     const PetscInt    **perms = NULL;
4920     const PetscScalar **flips = NULL;
4921 
4922     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4923     switch (mode) {
4924     case INSERT_VALUES:
4925       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4926         const PetscInt    point = points[2*p];
4927         const PetscInt    *perm = perms ? perms[p] : NULL;
4928         const PetscScalar *flip = flips ? flips[p] : NULL;
4929         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4930         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4931       } break;
4932     case INSERT_ALL_VALUES:
4933       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4934         const PetscInt    point = points[2*p];
4935         const PetscInt    *perm = perms ? perms[p] : NULL;
4936         const PetscScalar *flip = flips ? flips[p] : NULL;
4937         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4938         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4939       } break;
4940     case INSERT_BC_VALUES:
4941       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4942         const PetscInt    point = points[2*p];
4943         const PetscInt    *perm = perms ? perms[p] : NULL;
4944         const PetscScalar *flip = flips ? flips[p] : NULL;
4945         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4946         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4947       } break;
4948     case ADD_VALUES:
4949       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4950         const PetscInt    point = points[2*p];
4951         const PetscInt    *perm = perms ? perms[p] : NULL;
4952         const PetscScalar *flip = flips ? flips[p] : NULL;
4953         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4954         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4955       } break;
4956     case ADD_ALL_VALUES:
4957       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4958         const PetscInt    point = points[2*p];
4959         const PetscInt    *perm = perms ? perms[p] : NULL;
4960         const PetscScalar *flip = flips ? flips[p] : NULL;
4961         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4962         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4963       } break;
4964     case ADD_BC_VALUES:
4965       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4966         const PetscInt    point = points[2*p];
4967         const PetscInt    *perm = perms ? perms[p] : NULL;
4968         const PetscScalar *flip = flips ? flips[p] : NULL;
4969         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4970         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4971       } break;
4972     default:
4973       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4974     }
4975     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4976   }
4977   /* Cleanup points */
4978   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4979   /* Cleanup array */
4980   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4981   PetscFunctionReturn(0);
4982 }
4983 
4984 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4985 {
4986   PetscSection      clSection;
4987   IS                clPoints;
4988   PetscScalar       *array;
4989   PetscInt          *points = NULL;
4990   const PetscInt    *clp, *clperm;
4991   PetscInt          numFields, numPoints, p;
4992   PetscInt          offset = 0, f;
4993   PetscErrorCode    ierr;
4994 
4995   PetscFunctionBeginHot;
4996   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4997   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
4998   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4999   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
5000   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5001   /* Get points */
5002   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5003   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5004   /* Get array */
5005   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
5006   /* Get values */
5007   for (f = 0; f < numFields; ++f) {
5008     const PetscInt    **perms = NULL;
5009     const PetscScalar **flips = NULL;
5010 
5011     if (!fieldActive[f]) {
5012       for (p = 0; p < numPoints*2; p += 2) {
5013         PetscInt fdof;
5014         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5015         offset += fdof;
5016       }
5017       continue;
5018     }
5019     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5020     switch (mode) {
5021     case INSERT_VALUES:
5022       for (p = 0; p < numPoints; p++) {
5023         const PetscInt    point = points[2*p];
5024         const PetscInt    *perm = perms ? perms[p] : NULL;
5025         const PetscScalar *flip = flips ? flips[p] : NULL;
5026         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
5027       } break;
5028     case INSERT_ALL_VALUES:
5029       for (p = 0; p < numPoints; p++) {
5030         const PetscInt    point = points[2*p];
5031         const PetscInt    *perm = perms ? perms[p] : NULL;
5032         const PetscScalar *flip = flips ? flips[p] : NULL;
5033         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
5034         } break;
5035     case INSERT_BC_VALUES:
5036       for (p = 0; p < numPoints; p++) {
5037         const PetscInt    point = points[2*p];
5038         const PetscInt    *perm = perms ? perms[p] : NULL;
5039         const PetscScalar *flip = flips ? flips[p] : NULL;
5040         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
5041       } break;
5042     case ADD_VALUES:
5043       for (p = 0; p < numPoints; p++) {
5044         const PetscInt    point = points[2*p];
5045         const PetscInt    *perm = perms ? perms[p] : NULL;
5046         const PetscScalar *flip = flips ? flips[p] : NULL;
5047         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
5048       } break;
5049     case ADD_ALL_VALUES:
5050       for (p = 0; p < numPoints; p++) {
5051         const PetscInt    point = points[2*p];
5052         const PetscInt    *perm = perms ? perms[p] : NULL;
5053         const PetscScalar *flip = flips ? flips[p] : NULL;
5054         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
5055       } break;
5056     default:
5057       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
5058     }
5059     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
5060   }
5061   /* Cleanup points */
5062   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5063   /* Cleanup array */
5064   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
5065   PetscFunctionReturn(0);
5066 }
5067 
5068 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
5069 {
5070   PetscMPIInt    rank;
5071   PetscInt       i, j;
5072   PetscErrorCode ierr;
5073 
5074   PetscFunctionBegin;
5075   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
5076   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
5077   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
5078   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
5079   numCIndices = numCIndices ? numCIndices : numRIndices;
5080   for (i = 0; i < numRIndices; i++) {
5081     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
5082     for (j = 0; j < numCIndices; j++) {
5083 #if defined(PETSC_USE_COMPLEX)
5084       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
5085 #else
5086       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
5087 #endif
5088     }
5089     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
5090   }
5091   PetscFunctionReturn(0);
5092 }
5093 
5094 /*
5095   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
5096 
5097   Input Parameters:
5098 + section - The section for this data layout
5099 . point   - The point contributing dofs with these indices
5100 . off     - The global offset of this point
5101 . loff    - The local offset of each field
5102 . setBC   - The flag determining whether to include indices of bounsary values
5103 . perm    - A permutation of the dofs on this point, or NULL
5104 - indperm - A permutation of the entire indices array, or NULL
5105 
5106   Output Parameter:
5107 . indices - Indices for dofs on this point
5108 
5109   Level: developer
5110 
5111   Note: The indices could be local or global, depending on the value of 'off'.
5112 */
5113 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
5114 {
5115   PetscInt        dof;   /* The number of unknowns on this point */
5116   PetscInt        cdof;  /* The number of constraints on this point */
5117   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
5118   PetscInt        cind = 0, k;
5119   PetscErrorCode  ierr;
5120 
5121   PetscFunctionBegin;
5122   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
5123   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
5124   if (!cdof || setBC) {
5125     for (k = 0; k < dof; ++k) {
5126       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5127       const PetscInt ind    = indperm ? indperm[preind] : preind;
5128 
5129       indices[ind] = off + k;
5130     }
5131   } else {
5132     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
5133     for (k = 0; k < dof; ++k) {
5134       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
5135       const PetscInt ind    = indperm ? indperm[preind] : preind;
5136 
5137       if ((cind < cdof) && (k == cdofs[cind])) {
5138         /* Insert check for returning constrained indices */
5139         indices[ind] = -(off+k+1);
5140         ++cind;
5141       } else {
5142         indices[ind] = off+k-cind;
5143       }
5144     }
5145   }
5146   *loff += dof;
5147   PetscFunctionReturn(0);
5148 }
5149 
5150 /*
5151   This version only believes the point offset from the globalSection
5152 
5153  . off - The global offset of this point
5154 */
5155 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5156 {
5157   PetscInt       numFields, foff, f;
5158   PetscErrorCode ierr;
5159 
5160   PetscFunctionBegin;
5161   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5162   for (f = 0, foff = 0; f < numFields; ++f) {
5163     PetscInt        fdof, cfdof;
5164     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5165     PetscInt        cind = 0, b;
5166     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5167 
5168     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5169     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5170     if (!cfdof || setBC) {
5171       for (b = 0; b < fdof; ++b) {
5172         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5173         const PetscInt ind    = indperm ? indperm[preind] : preind;
5174 
5175         indices[ind] = off+foff+b;
5176       }
5177     } else {
5178       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5179       for (b = 0; b < fdof; ++b) {
5180         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5181         const PetscInt ind    = indperm ? indperm[preind] : preind;
5182 
5183         if ((cind < cfdof) && (b == fcdofs[cind])) {
5184           indices[ind] = -(off+foff+b+1);
5185           ++cind;
5186         } else {
5187           indices[ind] = off+foff+b-cind;
5188         }
5189       }
5190     }
5191     foff     += (setBC ? fdof : (fdof - cfdof));
5192     foffs[f] += fdof;
5193   }
5194   PetscFunctionReturn(0);
5195 }
5196 
5197 /*
5198   This version believes the globalSection offsets for each field, rather than just the point offset
5199 
5200  . foffs - The offset into 'indices' for each field, since it is segregated by field
5201 */
5202 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5203 {
5204   PetscInt       numFields, foff, f;
5205   PetscErrorCode ierr;
5206 
5207   PetscFunctionBegin;
5208   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5209   for (f = 0; f < numFields; ++f) {
5210     PetscInt        fdof, cfdof;
5211     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5212     PetscInt        cind = 0, b;
5213     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5214 
5215     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5216     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5217     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5218     if (!cfdof || setBC) {
5219       for (b = 0; b < fdof; ++b) {
5220         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5221         const PetscInt ind    = indperm ? indperm[preind] : preind;
5222 
5223         indices[ind] = foff+b;
5224       }
5225     } else {
5226       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5227       for (b = 0; b < fdof; ++b) {
5228         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5229         const PetscInt ind    = indperm ? indperm[preind] : preind;
5230 
5231         if ((cind < cfdof) && (b == fcdofs[cind])) {
5232           indices[ind] = -(foff+b+1);
5233           ++cind;
5234         } else {
5235           indices[ind] = foff+b-cind;
5236         }
5237       }
5238     }
5239     foffs[f] += fdof;
5240   }
5241   PetscFunctionReturn(0);
5242 }
5243 
5244 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5245 {
5246   Mat             cMat;
5247   PetscSection    aSec, cSec;
5248   IS              aIS;
5249   PetscInt        aStart = -1, aEnd = -1;
5250   const PetscInt  *anchors;
5251   PetscInt        numFields, f, p, q, newP = 0;
5252   PetscInt        newNumPoints = 0, newNumIndices = 0;
5253   PetscInt        *newPoints, *indices, *newIndices;
5254   PetscInt        maxAnchor, maxDof;
5255   PetscInt        newOffsets[32];
5256   PetscInt        *pointMatOffsets[32];
5257   PetscInt        *newPointOffsets[32];
5258   PetscScalar     *pointMat[32];
5259   PetscScalar     *newValues=NULL,*tmpValues;
5260   PetscBool       anyConstrained = PETSC_FALSE;
5261   PetscErrorCode  ierr;
5262 
5263   PetscFunctionBegin;
5264   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5265   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5266   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5267 
5268   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5269   /* if there are point-to-point constraints */
5270   if (aSec) {
5271     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5272     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5273     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5274     /* figure out how many points are going to be in the new element matrix
5275      * (we allow double counting, because it's all just going to be summed
5276      * into the global matrix anyway) */
5277     for (p = 0; p < 2*numPoints; p+=2) {
5278       PetscInt b    = points[p];
5279       PetscInt bDof = 0, bSecDof;
5280 
5281       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5282       if (!bSecDof) {
5283         continue;
5284       }
5285       if (b >= aStart && b < aEnd) {
5286         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5287       }
5288       if (bDof) {
5289         /* this point is constrained */
5290         /* it is going to be replaced by its anchors */
5291         PetscInt bOff, q;
5292 
5293         anyConstrained = PETSC_TRUE;
5294         newNumPoints  += bDof;
5295         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5296         for (q = 0; q < bDof; q++) {
5297           PetscInt a = anchors[bOff + q];
5298           PetscInt aDof;
5299 
5300           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5301           newNumIndices += aDof;
5302           for (f = 0; f < numFields; ++f) {
5303             PetscInt fDof;
5304 
5305             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5306             newOffsets[f+1] += fDof;
5307           }
5308         }
5309       }
5310       else {
5311         /* this point is not constrained */
5312         newNumPoints++;
5313         newNumIndices += bSecDof;
5314         for (f = 0; f < numFields; ++f) {
5315           PetscInt fDof;
5316 
5317           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5318           newOffsets[f+1] += fDof;
5319         }
5320       }
5321     }
5322   }
5323   if (!anyConstrained) {
5324     if (outNumPoints)  *outNumPoints  = 0;
5325     if (outNumIndices) *outNumIndices = 0;
5326     if (outPoints)     *outPoints     = NULL;
5327     if (outValues)     *outValues     = NULL;
5328     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5329     PetscFunctionReturn(0);
5330   }
5331 
5332   if (outNumPoints)  *outNumPoints  = newNumPoints;
5333   if (outNumIndices) *outNumIndices = newNumIndices;
5334 
5335   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5336 
5337   if (!outPoints && !outValues) {
5338     if (offsets) {
5339       for (f = 0; f <= numFields; f++) {
5340         offsets[f] = newOffsets[f];
5341       }
5342     }
5343     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5344     PetscFunctionReturn(0);
5345   }
5346 
5347   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5348 
5349   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5350 
5351   /* workspaces */
5352   if (numFields) {
5353     for (f = 0; f < numFields; f++) {
5354       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5355       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5356     }
5357   }
5358   else {
5359     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5360     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5361   }
5362 
5363   /* get workspaces for the point-to-point matrices */
5364   if (numFields) {
5365     PetscInt totalOffset, totalMatOffset;
5366 
5367     for (p = 0; p < numPoints; p++) {
5368       PetscInt b    = points[2*p];
5369       PetscInt bDof = 0, bSecDof;
5370 
5371       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5372       if (!bSecDof) {
5373         for (f = 0; f < numFields; f++) {
5374           newPointOffsets[f][p + 1] = 0;
5375           pointMatOffsets[f][p + 1] = 0;
5376         }
5377         continue;
5378       }
5379       if (b >= aStart && b < aEnd) {
5380         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5381       }
5382       if (bDof) {
5383         for (f = 0; f < numFields; f++) {
5384           PetscInt fDof, q, bOff, allFDof = 0;
5385 
5386           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5387           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5388           for (q = 0; q < bDof; q++) {
5389             PetscInt a = anchors[bOff + q];
5390             PetscInt aFDof;
5391 
5392             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5393             allFDof += aFDof;
5394           }
5395           newPointOffsets[f][p+1] = allFDof;
5396           pointMatOffsets[f][p+1] = fDof * allFDof;
5397         }
5398       }
5399       else {
5400         for (f = 0; f < numFields; f++) {
5401           PetscInt fDof;
5402 
5403           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5404           newPointOffsets[f][p+1] = fDof;
5405           pointMatOffsets[f][p+1] = 0;
5406         }
5407       }
5408     }
5409     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5410       newPointOffsets[f][0] = totalOffset;
5411       pointMatOffsets[f][0] = totalMatOffset;
5412       for (p = 0; p < numPoints; p++) {
5413         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5414         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5415       }
5416       totalOffset    = newPointOffsets[f][numPoints];
5417       totalMatOffset = pointMatOffsets[f][numPoints];
5418       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5419     }
5420   }
5421   else {
5422     for (p = 0; p < numPoints; p++) {
5423       PetscInt b    = points[2*p];
5424       PetscInt bDof = 0, bSecDof;
5425 
5426       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5427       if (!bSecDof) {
5428         newPointOffsets[0][p + 1] = 0;
5429         pointMatOffsets[0][p + 1] = 0;
5430         continue;
5431       }
5432       if (b >= aStart && b < aEnd) {
5433         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5434       }
5435       if (bDof) {
5436         PetscInt bOff, q, allDof = 0;
5437 
5438         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5439         for (q = 0; q < bDof; q++) {
5440           PetscInt a = anchors[bOff + q], aDof;
5441 
5442           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5443           allDof += aDof;
5444         }
5445         newPointOffsets[0][p+1] = allDof;
5446         pointMatOffsets[0][p+1] = bSecDof * allDof;
5447       }
5448       else {
5449         newPointOffsets[0][p+1] = bSecDof;
5450         pointMatOffsets[0][p+1] = 0;
5451       }
5452     }
5453     newPointOffsets[0][0] = 0;
5454     pointMatOffsets[0][0] = 0;
5455     for (p = 0; p < numPoints; p++) {
5456       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5457       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5458     }
5459     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5460   }
5461 
5462   /* output arrays */
5463   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5464 
5465   /* get the point-to-point matrices; construct newPoints */
5466   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5467   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5468   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5469   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5470   if (numFields) {
5471     for (p = 0, newP = 0; p < numPoints; p++) {
5472       PetscInt b    = points[2*p];
5473       PetscInt o    = points[2*p+1];
5474       PetscInt bDof = 0, bSecDof;
5475 
5476       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5477       if (!bSecDof) {
5478         continue;
5479       }
5480       if (b >= aStart && b < aEnd) {
5481         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5482       }
5483       if (bDof) {
5484         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5485 
5486         fStart[0] = 0;
5487         fEnd[0]   = 0;
5488         for (f = 0; f < numFields; f++) {
5489           PetscInt fDof;
5490 
5491           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5492           fStart[f+1] = fStart[f] + fDof;
5493           fEnd[f+1]   = fStart[f+1];
5494         }
5495         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5496         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5497 
5498         fAnchorStart[0] = 0;
5499         fAnchorEnd[0]   = 0;
5500         for (f = 0; f < numFields; f++) {
5501           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5502 
5503           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5504           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5505         }
5506         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5507         for (q = 0; q < bDof; q++) {
5508           PetscInt a = anchors[bOff + q], aOff;
5509 
5510           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5511           newPoints[2*(newP + q)]     = a;
5512           newPoints[2*(newP + q) + 1] = 0;
5513           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5514           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5515         }
5516         newP += bDof;
5517 
5518         if (outValues) {
5519           /* get the point-to-point submatrix */
5520           for (f = 0; f < numFields; f++) {
5521             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5522           }
5523         }
5524       }
5525       else {
5526         newPoints[2 * newP]     = b;
5527         newPoints[2 * newP + 1] = o;
5528         newP++;
5529       }
5530     }
5531   } else {
5532     for (p = 0; p < numPoints; p++) {
5533       PetscInt b    = points[2*p];
5534       PetscInt o    = points[2*p+1];
5535       PetscInt bDof = 0, bSecDof;
5536 
5537       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5538       if (!bSecDof) {
5539         continue;
5540       }
5541       if (b >= aStart && b < aEnd) {
5542         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5543       }
5544       if (bDof) {
5545         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5546 
5547         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5548         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5549 
5550         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5551         for (q = 0; q < bDof; q++) {
5552           PetscInt a = anchors[bOff + q], aOff;
5553 
5554           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5555 
5556           newPoints[2*(newP + q)]     = a;
5557           newPoints[2*(newP + q) + 1] = 0;
5558           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5559           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5560         }
5561         newP += bDof;
5562 
5563         /* get the point-to-point submatrix */
5564         if (outValues) {
5565           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5566         }
5567       }
5568       else {
5569         newPoints[2 * newP]     = b;
5570         newPoints[2 * newP + 1] = o;
5571         newP++;
5572       }
5573     }
5574   }
5575 
5576   if (outValues) {
5577     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5578     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5579     /* multiply constraints on the right */
5580     if (numFields) {
5581       for (f = 0; f < numFields; f++) {
5582         PetscInt oldOff = offsets[f];
5583 
5584         for (p = 0; p < numPoints; p++) {
5585           PetscInt cStart = newPointOffsets[f][p];
5586           PetscInt b      = points[2 * p];
5587           PetscInt c, r, k;
5588           PetscInt dof;
5589 
5590           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5591           if (!dof) {
5592             continue;
5593           }
5594           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5595             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5596             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5597 
5598             for (r = 0; r < numIndices; r++) {
5599               for (c = 0; c < nCols; c++) {
5600                 for (k = 0; k < dof; k++) {
5601                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5602                 }
5603               }
5604             }
5605           }
5606           else {
5607             /* copy this column as is */
5608             for (r = 0; r < numIndices; r++) {
5609               for (c = 0; c < dof; c++) {
5610                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5611               }
5612             }
5613           }
5614           oldOff += dof;
5615         }
5616       }
5617     }
5618     else {
5619       PetscInt oldOff = 0;
5620       for (p = 0; p < numPoints; p++) {
5621         PetscInt cStart = newPointOffsets[0][p];
5622         PetscInt b      = points[2 * p];
5623         PetscInt c, r, k;
5624         PetscInt dof;
5625 
5626         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5627         if (!dof) {
5628           continue;
5629         }
5630         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5631           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5632           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5633 
5634           for (r = 0; r < numIndices; r++) {
5635             for (c = 0; c < nCols; c++) {
5636               for (k = 0; k < dof; k++) {
5637                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5638               }
5639             }
5640           }
5641         }
5642         else {
5643           /* copy this column as is */
5644           for (r = 0; r < numIndices; r++) {
5645             for (c = 0; c < dof; c++) {
5646               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5647             }
5648           }
5649         }
5650         oldOff += dof;
5651       }
5652     }
5653 
5654     if (multiplyLeft) {
5655       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5656       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5657       /* multiply constraints transpose on the left */
5658       if (numFields) {
5659         for (f = 0; f < numFields; f++) {
5660           PetscInt oldOff = offsets[f];
5661 
5662           for (p = 0; p < numPoints; p++) {
5663             PetscInt rStart = newPointOffsets[f][p];
5664             PetscInt b      = points[2 * p];
5665             PetscInt c, r, k;
5666             PetscInt dof;
5667 
5668             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5669             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5670               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5671               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5672 
5673               for (r = 0; r < nRows; r++) {
5674                 for (c = 0; c < newNumIndices; c++) {
5675                   for (k = 0; k < dof; k++) {
5676                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5677                   }
5678                 }
5679               }
5680             }
5681             else {
5682               /* copy this row as is */
5683               for (r = 0; r < dof; r++) {
5684                 for (c = 0; c < newNumIndices; c++) {
5685                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5686                 }
5687               }
5688             }
5689             oldOff += dof;
5690           }
5691         }
5692       }
5693       else {
5694         PetscInt oldOff = 0;
5695 
5696         for (p = 0; p < numPoints; p++) {
5697           PetscInt rStart = newPointOffsets[0][p];
5698           PetscInt b      = points[2 * p];
5699           PetscInt c, r, k;
5700           PetscInt dof;
5701 
5702           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5703           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5704             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5705             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5706 
5707             for (r = 0; r < nRows; r++) {
5708               for (c = 0; c < newNumIndices; c++) {
5709                 for (k = 0; k < dof; k++) {
5710                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5711                 }
5712               }
5713             }
5714           }
5715           else {
5716             /* copy this row as is */
5717             for (r = 0; r < dof; r++) {
5718               for (c = 0; c < newNumIndices; c++) {
5719                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5720               }
5721             }
5722           }
5723           oldOff += dof;
5724         }
5725       }
5726 
5727       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5728     }
5729     else {
5730       newValues = tmpValues;
5731     }
5732   }
5733 
5734   /* clean up */
5735   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5736   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5737 
5738   if (numFields) {
5739     for (f = 0; f < numFields; f++) {
5740       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5741       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5742       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5743     }
5744   }
5745   else {
5746     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5747     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5748     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5749   }
5750   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5751 
5752   /* output */
5753   if (outPoints) {
5754     *outPoints = newPoints;
5755   }
5756   else {
5757     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5758   }
5759   if (outValues) {
5760     *outValues = newValues;
5761   }
5762   for (f = 0; f <= numFields; f++) {
5763     offsets[f] = newOffsets[f];
5764   }
5765   PetscFunctionReturn(0);
5766 }
5767 
5768 /*@C
5769   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5770 
5771   Not collective
5772 
5773   Input Parameters:
5774 + dm - The DM
5775 . section - The section describing the layout in v, or NULL to use the default section
5776 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5777 - point - The mesh point
5778 
5779   Output parameters:
5780 + numIndices - The number of indices
5781 . indices - The indices
5782 - outOffsets - Field offset if not NULL
5783 
5784   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5785 
5786   Level: advanced
5787 
5788 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5789 @*/
5790 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5791 {
5792   PetscSection    clSection;
5793   IS              clPoints;
5794   const PetscInt *clp, *clperm;
5795   const PetscInt  **perms[32] = {NULL};
5796   PetscInt       *points = NULL, *pointsNew;
5797   PetscInt        numPoints, numPointsNew;
5798   PetscInt        offsets[32];
5799   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5800   PetscErrorCode  ierr;
5801 
5802   PetscFunctionBegin;
5803   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5804   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5805   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5806   if (numIndices) PetscValidPointer(numIndices, 4);
5807   PetscValidPointer(indices, 5);
5808   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5809   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5810   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5811   /* Get points in closure */
5812   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5813   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5814   /* Get number of indices and indices per field */
5815   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5816     PetscInt dof, fdof;
5817 
5818     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5819     for (f = 0; f < Nf; ++f) {
5820       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5821       offsets[f+1] += fdof;
5822     }
5823     Nind += dof;
5824   }
5825   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5826   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5827   if (!Nf) offsets[1] = Nind;
5828   /* Get dual space symmetries */
5829   for (f = 0; f < PetscMax(1,Nf); f++) {
5830     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5831     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5832   }
5833   /* Correct for hanging node constraints */
5834   {
5835     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5836     if (numPointsNew) {
5837       for (f = 0; f < PetscMax(1,Nf); f++) {
5838         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5839         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5840       }
5841       for (f = 0; f < PetscMax(1,Nf); f++) {
5842         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5843         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5844       }
5845       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5846       numPoints = numPointsNew;
5847       Nind      = NindNew;
5848       points    = pointsNew;
5849     }
5850   }
5851   /* Calculate indices */
5852   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5853   if (Nf) {
5854     if (outOffsets) {
5855       PetscInt f;
5856 
5857       for (f = 0; f <= Nf; f++) {
5858         outOffsets[f] = offsets[f];
5859       }
5860     }
5861     for (p = 0; p < numPoints; p++) {
5862       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5863       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);
5864     }
5865   } else {
5866     for (p = 0, off = 0; p < numPoints; p++) {
5867       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5868 
5869       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5870       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);
5871     }
5872   }
5873   /* Cleanup points */
5874   for (f = 0; f < PetscMax(1,Nf); f++) {
5875     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5876     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5877   }
5878   if (numPointsNew) {
5879     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5880   } else {
5881     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5882   }
5883   if (numIndices) *numIndices = Nind;
5884   PetscFunctionReturn(0);
5885 }
5886 
5887 /*@C
5888   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5889 
5890   Not collective
5891 
5892   Input Parameters:
5893 + dm - The DM
5894 . section - The section describing the layout in v, or NULL to use the default section
5895 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5896 . point - The mesh point
5897 . numIndices - The number of indices
5898 . indices - The indices
5899 - outOffsets - Field offset if not NULL
5900 
5901   Level: advanced
5902 
5903 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5904 @*/
5905 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5906 {
5907   PetscErrorCode ierr;
5908 
5909   PetscFunctionBegin;
5910   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5911   PetscValidPointer(indices, 5);
5912   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5913   PetscFunctionReturn(0);
5914 }
5915 
5916 /*@C
5917   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5918 
5919   Not collective
5920 
5921   Input Parameters:
5922 + dm - The DM
5923 . section - The section describing the layout in v, or NULL to use the default section
5924 . globalSection - The section describing the layout in v, or NULL to use the default global section
5925 . A - The matrix
5926 . point - The point in the DM
5927 . values - The array of values
5928 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5929 
5930   Fortran Notes:
5931   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5932 
5933   Level: intermediate
5934 
5935 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5936 @*/
5937 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5938 {
5939   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5940   PetscSection        clSection;
5941   IS                  clPoints;
5942   PetscInt           *points = NULL, *newPoints;
5943   const PetscInt     *clp, *clperm;
5944   PetscInt           *indices;
5945   PetscInt            offsets[32];
5946   const PetscInt    **perms[32] = {NULL};
5947   const PetscScalar **flips[32] = {NULL};
5948   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5949   PetscScalar        *valCopy = NULL;
5950   PetscScalar        *newValues;
5951   PetscErrorCode      ierr;
5952 
5953   PetscFunctionBegin;
5954   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5955   if (!section) {ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);}
5956   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5957   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5958   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5959   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5960   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5961   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5962   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5963   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5964   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5965   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5966     PetscInt fdof;
5967 
5968     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5969     for (f = 0; f < numFields; ++f) {
5970       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5971       offsets[f+1] += fdof;
5972     }
5973     numIndices += dof;
5974   }
5975   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5976 
5977   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5978   /* Get symmetries */
5979   for (f = 0; f < PetscMax(1,numFields); f++) {
5980     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5981     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5982     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5983       PetscInt foffset = offsets[f];
5984 
5985       for (p = 0; p < numPoints; p++) {
5986         PetscInt point          = points[2*p], fdof;
5987         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5988 
5989         if (!numFields) {
5990           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5991         } else {
5992           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5993         }
5994         if (flip) {
5995           PetscInt i, j, k;
5996 
5997           if (!valCopy) {
5998             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5999             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
6000             values = valCopy;
6001           }
6002           for (i = 0; i < fdof; i++) {
6003             PetscScalar fval = flip[i];
6004 
6005             for (k = 0; k < numIndices; k++) {
6006               valCopy[numIndices * (foffset + i) + k] *= fval;
6007               valCopy[numIndices * k + (foffset + i)] *= fval;
6008             }
6009           }
6010         }
6011         foffset += fdof;
6012       }
6013     }
6014   }
6015   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
6016   if (newNumPoints) {
6017     if (valCopy) {
6018       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6019     }
6020     for (f = 0; f < PetscMax(1,numFields); f++) {
6021       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6022       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6023     }
6024     for (f = 0; f < PetscMax(1,numFields); f++) {
6025       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6026       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
6027     }
6028     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6029     numPoints  = newNumPoints;
6030     numIndices = newNumIndices;
6031     points     = newPoints;
6032     values     = newValues;
6033   }
6034   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6035   if (numFields) {
6036     PetscBool useFieldOffsets;
6037 
6038     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
6039     if (useFieldOffsets) {
6040       for (p = 0; p < numPoints; p++) {
6041         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
6042       }
6043     } else {
6044       for (p = 0; p < numPoints; p++) {
6045         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6046         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
6047       }
6048     }
6049   } else {
6050     for (p = 0, off = 0; p < numPoints; p++) {
6051       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
6052       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
6053       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
6054     }
6055   }
6056   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
6057   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
6058   if (mesh->printFEM > 1) {
6059     PetscInt i;
6060     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
6061     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
6062     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6063   }
6064   if (ierr) {
6065     PetscMPIInt    rank;
6066     PetscErrorCode ierr2;
6067 
6068     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6069     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6070     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
6071     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
6072     CHKERRQ(ierr);
6073   }
6074   for (f = 0; f < PetscMax(1,numFields); f++) {
6075     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6076     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
6077   }
6078   if (newNumPoints) {
6079     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
6080     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
6081   }
6082   else {
6083     if (valCopy) {
6084       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
6085     }
6086     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
6087   }
6088   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
6089   PetscFunctionReturn(0);
6090 }
6091 
6092 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
6093 {
6094   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
6095   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
6096   PetscInt       *cpoints = NULL;
6097   PetscInt       *findices, *cindices;
6098   const PetscInt *fclperm, *cclperm;
6099   PetscInt        foffsets[32], coffsets[32];
6100   CellRefiner     cellRefiner;
6101   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6102   PetscErrorCode  ierr;
6103 
6104   PetscFunctionBegin;
6105   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6106   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6107   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6108   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6109   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6110   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6111   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6112   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6113   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6114   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6115   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
6116   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6117   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6118   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6119   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6120   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6121   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6122   /* Column indices */
6123   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6124   maxFPoints = numCPoints;
6125   /* Compress out points not in the section */
6126   /*   TODO: Squeeze out points with 0 dof as well */
6127   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6128   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6129     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6130       cpoints[q*2]   = cpoints[p];
6131       cpoints[q*2+1] = cpoints[p+1];
6132       ++q;
6133     }
6134   }
6135   numCPoints = q;
6136   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6137     PetscInt fdof;
6138 
6139     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6140     if (!dof) continue;
6141     for (f = 0; f < numFields; ++f) {
6142       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6143       coffsets[f+1] += fdof;
6144     }
6145     numCIndices += dof;
6146   }
6147   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6148   /* Row indices */
6149   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6150   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6151   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6152   for (r = 0, q = 0; r < numSubcells; ++r) {
6153     /* TODO Map from coarse to fine cells */
6154     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6155     /* Compress out points not in the section */
6156     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6157     for (p = 0; p < numFPoints*2; p += 2) {
6158       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6159         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6160         if (!dof) continue;
6161         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6162         if (s < q) continue;
6163         ftotpoints[q*2]   = fpoints[p];
6164         ftotpoints[q*2+1] = fpoints[p+1];
6165         ++q;
6166       }
6167     }
6168     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6169   }
6170   numFPoints = q;
6171   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6172     PetscInt fdof;
6173 
6174     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6175     if (!dof) continue;
6176     for (f = 0; f < numFields; ++f) {
6177       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6178       foffsets[f+1] += fdof;
6179     }
6180     numFIndices += dof;
6181   }
6182   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6183 
6184   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6185   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6186   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6187   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6188   if (numFields) {
6189     const PetscInt **permsF[32] = {NULL};
6190     const PetscInt **permsC[32] = {NULL};
6191 
6192     for (f = 0; f < numFields; f++) {
6193       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6194       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6195     }
6196     for (p = 0; p < numFPoints; p++) {
6197       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6198       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6199     }
6200     for (p = 0; p < numCPoints; p++) {
6201       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6202       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6203     }
6204     for (f = 0; f < numFields; f++) {
6205       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6206       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6207     }
6208   } else {
6209     const PetscInt **permsF = NULL;
6210     const PetscInt **permsC = NULL;
6211 
6212     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6213     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6214     for (p = 0, off = 0; p < numFPoints; p++) {
6215       const PetscInt *perm = permsF ? permsF[p] : NULL;
6216 
6217       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6218       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6219     }
6220     for (p = 0, off = 0; p < numCPoints; p++) {
6221       const PetscInt *perm = permsC ? permsC[p] : NULL;
6222 
6223       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6224       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6225     }
6226     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6227     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6228   }
6229   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6230   /* TODO: flips */
6231   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6232   if (ierr) {
6233     PetscMPIInt    rank;
6234     PetscErrorCode ierr2;
6235 
6236     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6237     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6238     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6239     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6240     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6241     CHKERRQ(ierr);
6242   }
6243   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6244   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6245   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6246   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6247   PetscFunctionReturn(0);
6248 }
6249 
6250 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6251 {
6252   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6253   PetscInt      *cpoints = NULL;
6254   PetscInt       foffsets[32], coffsets[32];
6255   const PetscInt *fclperm, *cclperm;
6256   CellRefiner    cellRefiner;
6257   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6258   PetscErrorCode ierr;
6259 
6260   PetscFunctionBegin;
6261   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6262   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6263   if (!fsection) {ierr = DMGetLocalSection(dmf, &fsection);CHKERRQ(ierr);}
6264   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6265   if (!csection) {ierr = DMGetLocalSection(dmc, &csection);CHKERRQ(ierr);}
6266   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6267   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6268   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6269   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6270   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6271   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6272   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6273   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6274   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6275   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6276   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6277   /* Column indices */
6278   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6279   maxFPoints = numCPoints;
6280   /* Compress out points not in the section */
6281   /*   TODO: Squeeze out points with 0 dof as well */
6282   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6283   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6284     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6285       cpoints[q*2]   = cpoints[p];
6286       cpoints[q*2+1] = cpoints[p+1];
6287       ++q;
6288     }
6289   }
6290   numCPoints = q;
6291   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6292     PetscInt fdof;
6293 
6294     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6295     if (!dof) continue;
6296     for (f = 0; f < numFields; ++f) {
6297       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6298       coffsets[f+1] += fdof;
6299     }
6300     numCIndices += dof;
6301   }
6302   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6303   /* Row indices */
6304   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6305   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6306   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6307   for (r = 0, q = 0; r < numSubcells; ++r) {
6308     /* TODO Map from coarse to fine cells */
6309     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6310     /* Compress out points not in the section */
6311     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6312     for (p = 0; p < numFPoints*2; p += 2) {
6313       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6314         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6315         if (!dof) continue;
6316         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6317         if (s < q) continue;
6318         ftotpoints[q*2]   = fpoints[p];
6319         ftotpoints[q*2+1] = fpoints[p+1];
6320         ++q;
6321       }
6322     }
6323     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6324   }
6325   numFPoints = q;
6326   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6327     PetscInt fdof;
6328 
6329     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6330     if (!dof) continue;
6331     for (f = 0; f < numFields; ++f) {
6332       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6333       foffsets[f+1] += fdof;
6334     }
6335     numFIndices += dof;
6336   }
6337   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6338 
6339   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6340   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6341   if (numFields) {
6342     const PetscInt **permsF[32] = {NULL};
6343     const PetscInt **permsC[32] = {NULL};
6344 
6345     for (f = 0; f < numFields; f++) {
6346       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6347       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6348     }
6349     for (p = 0; p < numFPoints; p++) {
6350       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6351       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6352     }
6353     for (p = 0; p < numCPoints; p++) {
6354       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6355       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6356     }
6357     for (f = 0; f < numFields; f++) {
6358       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6359       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6360     }
6361   } else {
6362     const PetscInt **permsF = NULL;
6363     const PetscInt **permsC = NULL;
6364 
6365     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6366     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6367     for (p = 0, off = 0; p < numFPoints; p++) {
6368       const PetscInt *perm = permsF ? permsF[p] : NULL;
6369 
6370       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6371       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6372     }
6373     for (p = 0, off = 0; p < numCPoints; p++) {
6374       const PetscInt *perm = permsC ? permsC[p] : NULL;
6375 
6376       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6377       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6378     }
6379     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6380     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6381   }
6382   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6383   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6384   PetscFunctionReturn(0);
6385 }
6386 
6387 /*@
6388   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6389 
6390   Input Parameter:
6391 . dm - The DMPlex object
6392 
6393   Output Parameters:
6394 + cMax - The first hybrid cell
6395 . fMax - The first hybrid face
6396 . eMax - The first hybrid edge
6397 - vMax - The first hybrid vertex
6398 
6399   Level: developer
6400 
6401 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6402 @*/
6403 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6404 {
6405   DM_Plex       *mesh = (DM_Plex*) dm->data;
6406   PetscInt       dim;
6407   PetscErrorCode ierr;
6408 
6409   PetscFunctionBegin;
6410   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6411   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6412   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6413   if (cMax) *cMax = mesh->hybridPointMax[dim];
6414   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6415   if (eMax) *eMax = mesh->hybridPointMax[1];
6416   if (vMax) *vMax = mesh->hybridPointMax[0];
6417   PetscFunctionReturn(0);
6418 }
6419 
6420 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6421 {
6422   IS             is, his;
6423   PetscInt       first = 0, stride;
6424   PetscBool      isStride;
6425   PetscErrorCode ierr;
6426 
6427   PetscFunctionBegin;
6428   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6429   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6430   if (isStride) {
6431     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6432   }
6433   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6434   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6435   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6436   ierr = ISDestroy(&his);CHKERRQ(ierr);
6437   ierr = ISDestroy(&is);CHKERRQ(ierr);
6438   PetscFunctionReturn(0);
6439 }
6440 
6441 /*@
6442   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6443 
6444   Input Parameters:
6445 + dm   - The DMPlex object
6446 . cMax - The first hybrid cell
6447 . fMax - The first hybrid face
6448 . eMax - The first hybrid edge
6449 - vMax - The first hybrid vertex
6450 
6451   Level: developer
6452 
6453 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6454 @*/
6455 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6456 {
6457   DM_Plex       *mesh = (DM_Plex*) dm->data;
6458   PetscInt       dim;
6459   PetscErrorCode ierr;
6460 
6461   PetscFunctionBegin;
6462   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6463   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6464   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6465   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6466   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6467   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6468   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6469   PetscFunctionReturn(0);
6470 }
6471 
6472 /*@C
6473   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6474 
6475   Input Parameter:
6476 . dm   - The DMPlex object
6477 
6478   Output Parameter:
6479 . cellHeight - The height of a cell
6480 
6481   Level: developer
6482 
6483 .seealso DMPlexSetVTKCellHeight()
6484 @*/
6485 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6486 {
6487   DM_Plex *mesh = (DM_Plex*) dm->data;
6488 
6489   PetscFunctionBegin;
6490   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6491   PetscValidPointer(cellHeight, 2);
6492   *cellHeight = mesh->vtkCellHeight;
6493   PetscFunctionReturn(0);
6494 }
6495 
6496 /*@C
6497   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6498 
6499   Input Parameters:
6500 + dm   - The DMPlex object
6501 - cellHeight - The height of a cell
6502 
6503   Level: developer
6504 
6505 .seealso DMPlexGetVTKCellHeight()
6506 @*/
6507 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6508 {
6509   DM_Plex *mesh = (DM_Plex*) dm->data;
6510 
6511   PetscFunctionBegin;
6512   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6513   mesh->vtkCellHeight = cellHeight;
6514   PetscFunctionReturn(0);
6515 }
6516 
6517 /* We can easily have a form that takes an IS instead */
6518 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6519 {
6520   PetscSection   section, globalSection;
6521   PetscInt      *numbers, p;
6522   PetscErrorCode ierr;
6523 
6524   PetscFunctionBegin;
6525   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6526   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6527   for (p = pStart; p < pEnd; ++p) {
6528     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6529   }
6530   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6531   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6532   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6533   for (p = pStart; p < pEnd; ++p) {
6534     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6535     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6536     else                       numbers[p-pStart] += shift;
6537   }
6538   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6539   if (globalSize) {
6540     PetscLayout layout;
6541     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6542     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6543     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6544   }
6545   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6546   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6547   PetscFunctionReturn(0);
6548 }
6549 
6550 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6551 {
6552   PetscInt       cellHeight, cStart, cEnd, cMax;
6553   PetscErrorCode ierr;
6554 
6555   PetscFunctionBegin;
6556   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6557   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6558   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6559   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6560   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6561   PetscFunctionReturn(0);
6562 }
6563 
6564 /*@
6565   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6566 
6567   Input Parameter:
6568 . dm   - The DMPlex object
6569 
6570   Output Parameter:
6571 . globalCellNumbers - Global cell numbers for all cells on this process
6572 
6573   Level: developer
6574 
6575 .seealso DMPlexGetVertexNumbering()
6576 @*/
6577 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6578 {
6579   DM_Plex       *mesh = (DM_Plex*) dm->data;
6580   PetscErrorCode ierr;
6581 
6582   PetscFunctionBegin;
6583   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6584   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6585   *globalCellNumbers = mesh->globalCellNumbers;
6586   PetscFunctionReturn(0);
6587 }
6588 
6589 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6590 {
6591   PetscInt       vStart, vEnd, vMax;
6592   PetscErrorCode ierr;
6593 
6594   PetscFunctionBegin;
6595   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6596   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6597   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6598   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6599   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6600   PetscFunctionReturn(0);
6601 }
6602 
6603 /*@
6604   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
6605 
6606   Input Parameter:
6607 . dm   - The DMPlex object
6608 
6609   Output Parameter:
6610 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6611 
6612   Level: developer
6613 
6614 .seealso DMPlexGetCellNumbering()
6615 @*/
6616 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6617 {
6618   DM_Plex       *mesh = (DM_Plex*) dm->data;
6619   PetscErrorCode ierr;
6620 
6621   PetscFunctionBegin;
6622   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6623   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6624   *globalVertexNumbers = mesh->globalVertexNumbers;
6625   PetscFunctionReturn(0);
6626 }
6627 
6628 /*@
6629   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6630 
6631   Input Parameter:
6632 . dm   - The DMPlex object
6633 
6634   Output Parameter:
6635 . globalPointNumbers - Global numbers for all points on this process
6636 
6637   Level: developer
6638 
6639 .seealso DMPlexGetCellNumbering()
6640 @*/
6641 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6642 {
6643   IS             nums[4];
6644   PetscInt       depths[4], gdepths[4], starts[4];
6645   PetscInt       depth, d, shift = 0;
6646   PetscErrorCode ierr;
6647 
6648   PetscFunctionBegin;
6649   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6650   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6651   /* For unstratified meshes use dim instead of depth */
6652   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6653   for (d = 0; d <= depth; ++d) {
6654     PetscInt end;
6655 
6656     depths[d] = depth-d;
6657     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6658     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6659   }
6660   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6661   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6662   for (d = 0; d <= depth; ++d) {
6663     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6664   }
6665   for (d = 0; d <= depth; ++d) {
6666     PetscInt pStart, pEnd, gsize;
6667 
6668     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6669     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6670     shift += gsize;
6671   }
6672   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6673   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6674   PetscFunctionReturn(0);
6675 }
6676 
6677 
6678 /*@
6679   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6680 
6681   Input Parameter:
6682 . dm - The DMPlex object
6683 
6684   Output Parameter:
6685 . ranks - The rank field
6686 
6687   Options Database Keys:
6688 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6689 
6690   Level: intermediate
6691 
6692 .seealso: DMView()
6693 @*/
6694 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6695 {
6696   DM             rdm;
6697   PetscFE        fe;
6698   PetscScalar   *r;
6699   PetscMPIInt    rank;
6700   PetscInt       dim, cStart, cEnd, c;
6701   PetscErrorCode ierr;
6702 
6703   PetscFunctionBeginUser;
6704   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6705   PetscValidPointer(ranks, 2);
6706   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6707   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6708   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6709   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6710   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6711   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6712   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6713   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6714   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6715   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6716   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6717   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6718   for (c = cStart; c < cEnd; ++c) {
6719     PetscScalar *lr;
6720 
6721     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6722     *lr = rank;
6723   }
6724   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6725   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6726   PetscFunctionReturn(0);
6727 }
6728 
6729 /*@
6730   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6731 
6732   Input Parameters:
6733 + dm    - The DMPlex
6734 - label - The DMLabel
6735 
6736   Output Parameter:
6737 . val - The label value field
6738 
6739   Options Database Keys:
6740 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6741 
6742   Level: intermediate
6743 
6744 .seealso: DMView()
6745 @*/
6746 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6747 {
6748   DM             rdm;
6749   PetscFE        fe;
6750   PetscScalar   *v;
6751   PetscInt       dim, cStart, cEnd, c;
6752   PetscErrorCode ierr;
6753 
6754   PetscFunctionBeginUser;
6755   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6756   PetscValidPointer(label, 2);
6757   PetscValidPointer(val, 3);
6758   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6759   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6760   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6761   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6762   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6763   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6764   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6765   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6766   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6767   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6768   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6769   for (c = cStart; c < cEnd; ++c) {
6770     PetscScalar *lv;
6771     PetscInt     cval;
6772 
6773     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6774     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6775     *lv = cval;
6776   }
6777   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6778   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6779   PetscFunctionReturn(0);
6780 }
6781 
6782 /*@
6783   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6784 
6785   Input Parameter:
6786 . dm - The DMPlex object
6787 
6788   Note: This is a useful diagnostic when creating meshes programmatically.
6789 
6790   Level: developer
6791 
6792 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6793 @*/
6794 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6795 {
6796   PetscSection    coneSection, supportSection;
6797   const PetscInt *cone, *support;
6798   PetscInt        coneSize, c, supportSize, s;
6799   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6800   PetscBool       storagecheck = PETSC_TRUE;
6801   PetscErrorCode  ierr;
6802 
6803   PetscFunctionBegin;
6804   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6805   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6806   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6807   /* Check that point p is found in the support of its cone points, and vice versa */
6808   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6809   for (p = pStart; p < pEnd; ++p) {
6810     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6811     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6812     for (c = 0; c < coneSize; ++c) {
6813       PetscBool dup = PETSC_FALSE;
6814       PetscInt  d;
6815       for (d = c-1; d >= 0; --d) {
6816         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6817       }
6818       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6819       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6820       for (s = 0; s < supportSize; ++s) {
6821         if (support[s] == p) break;
6822       }
6823       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6824         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6825         for (s = 0; s < coneSize; ++s) {
6826           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6827         }
6828         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6829         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6830         for (s = 0; s < supportSize; ++s) {
6831           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6832         }
6833         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6834         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6835         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6836       }
6837     }
6838     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6839     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6840     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6841     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6842     for (s = 0; s < supportSize; ++s) {
6843       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6844       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6845       for (c = 0; c < coneSize; ++c) {
6846         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6847         if (cone[c] != pp) { c = 0; break; }
6848         if (cone[c] == p) break;
6849       }
6850       if (c >= coneSize) {
6851         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6852         for (c = 0; c < supportSize; ++c) {
6853           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6854         }
6855         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6856         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6857         for (c = 0; c < coneSize; ++c) {
6858           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6859         }
6860         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6861         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6862       }
6863     }
6864   }
6865   if (storagecheck) {
6866     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6867     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6868     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6869   }
6870   PetscFunctionReturn(0);
6871 }
6872 
6873 /*@
6874   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6875 
6876   Input Parameters:
6877 + dm - The DMPlex object
6878 - cellHeight - Normally 0
6879 
6880   Note: This is a useful diagnostic when creating meshes programmatically.
6881   Currently applicable only to homogeneous simplex or tensor meshes.
6882 
6883   Level: developer
6884 
6885 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6886 @*/
6887 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6888 {
6889   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6890   PetscBool      isSimplex = PETSC_FALSE;
6891   PetscErrorCode ierr;
6892 
6893   PetscFunctionBegin;
6894   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6895   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6896   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6897   if (cStart < cEnd) {
6898     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6899     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6900   }
6901   switch (dim) {
6902   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6903   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6904   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6905   default:
6906     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6907   }
6908   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6909   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6910   cMax = cMax >= 0 ? cMax : cEnd;
6911   for (c = cStart; c < cMax; ++c) {
6912     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6913 
6914     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6915     for (cl = 0; cl < closureSize*2; cl += 2) {
6916       const PetscInt p = closure[cl];
6917       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6918     }
6919     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6920     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6921   }
6922   for (c = cMax; c < cEnd; ++c) {
6923     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6924 
6925     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6926     for (cl = 0; cl < closureSize*2; cl += 2) {
6927       const PetscInt p = closure[cl];
6928       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6929     }
6930     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6931     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6932   }
6933   PetscFunctionReturn(0);
6934 }
6935 
6936 /*@
6937   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6938 
6939   Input Parameters:
6940 + dm - The DMPlex object
6941 - cellHeight - Normally 0
6942 
6943   Note: This is a useful diagnostic when creating meshes programmatically.
6944 
6945   Level: developer
6946 
6947 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6948 @*/
6949 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6950 {
6951   PetscInt       pMax[4];
6952   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6953   PetscErrorCode ierr;
6954 
6955   PetscFunctionBegin;
6956   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6957   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6958   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6959   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6960   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6961   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6962     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6963     for (c = cStart; c < cEnd; ++c) {
6964       const PetscInt *cone, *ornt, *faces;
6965       PetscInt        numFaces, faceSize, coneSize,f;
6966       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6967 
6968       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6969       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6970       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6971       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6972       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6973       for (cl = 0; cl < closureSize*2; cl += 2) {
6974         const PetscInt p = closure[cl];
6975         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6976       }
6977       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6978       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6979       for (f = 0; f < numFaces; ++f) {
6980         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6981 
6982         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6983         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6984           const PetscInt p = fclosure[cl];
6985           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6986         }
6987         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6988         for (v = 0; v < fnumCorners; ++v) {
6989           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6990         }
6991         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6992       }
6993       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6994       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6995     }
6996   }
6997   PetscFunctionReturn(0);
6998 }
6999 
7000 /*@
7001   DMPlexCheckGeometry - Check the geometry of mesh cells
7002 
7003   Input Parameter:
7004 . dm - The DMPlex object
7005 
7006   Note: This is a useful diagnostic when creating meshes programmatically.
7007 
7008   Level: developer
7009 
7010 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
7011 @*/
7012 PetscErrorCode DMPlexCheckGeometry(DM dm)
7013 {
7014   PetscReal      detJ, J[9], refVol = 1.0;
7015   PetscReal      vol;
7016   PetscInt       dim, depth, d, cStart, cEnd, c, cMax;
7017   PetscErrorCode ierr;
7018 
7019   PetscFunctionBegin;
7020   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
7021   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7022   for (d = 0; d < dim; ++d) refVol *= 2.0;
7023   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
7024   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7025   cMax = cMax < 0 ? cEnd : cMax;
7026   for (c = cStart; c < cMax; ++c) {
7027     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
7028     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
7029     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
7030     if (depth > 1) {
7031       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
7032       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
7033       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
7034     }
7035   }
7036   PetscFunctionReturn(0);
7037 }
7038 
7039 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
7040 {
7041   PetscInt i,l,n;
7042   const PetscInt *cone;
7043   PetscErrorCode ierr;
7044 
7045   PetscFunctionBegin;
7046   *missingPoint = -1;
7047   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
7048   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
7049   for (i=0; i<n; i++) {
7050     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
7051     if (l < 0) {
7052       *missingPoint = cone[i];
7053       break;
7054     }
7055   }
7056   PetscFunctionReturn(0);
7057 }
7058 
7059 /*@
7060   DMPlexCheckPointSF - Check that several necessary conditions are met for the point SF of this plex.
7061 
7062   Input Parameters:
7063 . dm - The DMPlex object
7064 
7065   Notes:
7066   This is mainly intended for debugging/testing purposes.
7067   It currently checks only meshes with no partition overlapping.
7068 
7069   Level: developer
7070 
7071 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7072 @*/
7073 PetscErrorCode DMPlexCheckPointSF(DM dm)
7074 {
7075   PetscSF         sf;
7076   PetscInt        d,depth,i,nleaves,p,plo,phi,missingPoint;
7077   const PetscInt *locals;
7078   PetscErrorCode  ierr;
7079 
7080   PetscFunctionBegin;
7081   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7082   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
7083   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
7084   ierr = DMPlexGetOverlap(dm, &d);CHKERRQ(ierr);
7085   if (d) {
7086     ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Warning: DMPlexCheckPointSF() is currently not implemented for meshes with partition overlapping");
7087     PetscFunctionReturn(0);
7088   }
7089   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
7090 
7091   /* 1) check there are no faces in 2D, cells in 3D, in interface */
7092   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
7093   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
7094   for (i=0; i<nleaves; i++) {
7095     p = locals[i];
7096     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
7097   }
7098 
7099   /* 2) if some point is in interface, then all its cone points must be also in interface  */
7100   for (i=0; i<nleaves; i++) {
7101     p = locals[i];
7102     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
7103     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
7104   }
7105   PetscFunctionReturn(0);
7106 }
7107 
7108 typedef struct cell_stats
7109 {
7110   PetscReal min, max, sum, squaresum;
7111   PetscInt  count;
7112 } cell_stats_t;
7113 
7114 static void MPIAPI cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
7115 {
7116   PetscInt i, N = *len;
7117 
7118   for (i = 0; i < N; i++) {
7119     cell_stats_t *A = (cell_stats_t *) a;
7120     cell_stats_t *B = (cell_stats_t *) b;
7121 
7122     B->min = PetscMin(A->min,B->min);
7123     B->max = PetscMax(A->max,B->max);
7124     B->sum += A->sum;
7125     B->squaresum += A->squaresum;
7126     B->count += A->count;
7127   }
7128 }
7129 
7130 /*@
7131   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
7132 
7133   Collective on dm
7134 
7135   Input Parameters:
7136 + dm        - The DMPlex object
7137 . output    - If true, statistics will be displayed on stdout
7138 - condLimit - Display all cells above this condition number, or PETSC_DETERMINE for no cell output
7139 
7140   Note: This is mainly intended for debugging/testing purposes.
7141 
7142   Level: developer
7143 
7144 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
7145 @*/
7146 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
7147 {
7148   DM             dmCoarse;
7149   cell_stats_t   stats, globalStats;
7150   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
7151   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
7152   PetscReal      limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
7153   PetscInt       cdim, cStart, cEnd, cMax, c, eStart, eEnd, count = 0;
7154   PetscMPIInt    rank,size;
7155   PetscErrorCode ierr;
7156 
7157   PetscFunctionBegin;
7158   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7159   stats.min   = PETSC_MAX_REAL;
7160   stats.max   = PETSC_MIN_REAL;
7161   stats.sum   = stats.squaresum = 0.;
7162   stats.count = 0;
7163 
7164   ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
7165   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
7166   ierr = DMGetCoordinateDim(dm,&cdim);CHKERRQ(ierr);
7167   ierr = PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ);CHKERRQ(ierr);
7168   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7169   ierr = DMPlexGetDepthStratum(dm,1,&eStart,&eEnd);CHKERRQ(ierr);
7170   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7171   cMax = cMax < 0 ? cEnd : cMax;
7172   for (c = cStart; c < cMax; c++) {
7173     PetscInt  i;
7174     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7175 
7176     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7177     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7178     for (i = 0; i < PetscSqr(cdim); ++i) {
7179       frobJ    += J[i] * J[i];
7180       frobInvJ += invJ[i] * invJ[i];
7181     }
7182     cond2 = frobJ * frobInvJ;
7183     cond  = PetscSqrtReal(cond2);
7184 
7185     stats.min        = PetscMin(stats.min,cond);
7186     stats.max        = PetscMax(stats.max,cond);
7187     stats.sum       += cond;
7188     stats.squaresum += cond2;
7189     stats.count++;
7190     if (output && cond > limit) {
7191       PetscSection coordSection;
7192       Vec          coordsLocal;
7193       PetscScalar *coords = NULL;
7194       PetscInt     Nv, d, clSize, cl, *closure = NULL;
7195 
7196       ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
7197       ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
7198       ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7199       ierr = PetscSynchronizedPrintf(comm, "[%d] Cell %D cond %g\n", rank, c, (double) cond);CHKERRQ(ierr);
7200       for (i = 0; i < Nv/cdim; ++i) {
7201         ierr = PetscSynchronizedPrintf(comm, "  Vertex %D: (", i);CHKERRQ(ierr);
7202         for (d = 0; d < cdim; ++d) {
7203           if (d > 0) {ierr = PetscSynchronizedPrintf(comm, ", ");CHKERRQ(ierr);}
7204           ierr = PetscSynchronizedPrintf(comm, "%g", (double) PetscRealPart(coords[i*cdim+d]));CHKERRQ(ierr);
7205         }
7206         ierr = PetscSynchronizedPrintf(comm, ")\n");CHKERRQ(ierr);
7207       }
7208       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7209       for (cl = 0; cl < clSize*2; cl += 2) {
7210         const PetscInt edge = closure[cl];
7211 
7212         if ((edge >= eStart) && (edge < eEnd)) {
7213           PetscReal len;
7214 
7215           ierr = DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL);CHKERRQ(ierr);
7216           ierr = PetscSynchronizedPrintf(comm, "  Edge %D: length %g\n", edge, (double) len);CHKERRQ(ierr);
7217         }
7218       }
7219       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
7220       ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords);CHKERRQ(ierr);
7221     }
7222   }
7223   if (output) {ierr = PetscSynchronizedFlush(comm, NULL);CHKERRQ(ierr);}
7224 
7225   if (size > 1) {
7226     PetscMPIInt   blockLengths[2] = {4,1};
7227     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7228     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7229     MPI_Op        statReduce;
7230 
7231     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7232     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7233     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7234     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7235     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7236     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7237   } else {
7238     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7239   }
7240   if (!rank) {
7241     count = globalStats.count;
7242     min   = globalStats.min;
7243     max   = globalStats.max;
7244     mean  = globalStats.sum / globalStats.count;
7245     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7246   }
7247 
7248   if (output) {
7249     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7250   }
7251   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7252 
7253   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7254   if (dmCoarse) {
7255     PetscBool isplex;
7256 
7257     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7258     if (isplex) {
7259       ierr = DMPlexCheckCellShape(dmCoarse,output,condLimit);CHKERRQ(ierr);
7260     }
7261   }
7262   PetscFunctionReturn(0);
7263 }
7264 
7265 /* Pointwise interpolation
7266      Just code FEM for now
7267      u^f = I u^c
7268      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7269      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7270      I_{ij} = psi^f_i phi^c_j
7271 */
7272 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7273 {
7274   PetscSection   gsc, gsf;
7275   PetscInt       m, n;
7276   void          *ctx;
7277   DM             cdm;
7278   PetscBool      regular, ismatis;
7279   PetscErrorCode ierr;
7280 
7281   PetscFunctionBegin;
7282   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7283   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7284   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7285   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7286 
7287   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7288   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7289   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7290   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7291   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7292 
7293   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7294   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7295   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7296   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7297   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7298   if (scaling) {
7299     /* Use naive scaling */
7300     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7301   }
7302   PetscFunctionReturn(0);
7303 }
7304 
7305 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7306 {
7307   PetscErrorCode ierr;
7308   VecScatter     ctx;
7309 
7310   PetscFunctionBegin;
7311   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7312   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7313   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7314   PetscFunctionReturn(0);
7315 }
7316 
7317 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7318 {
7319   PetscSection   gsc, gsf;
7320   PetscInt       m, n;
7321   void          *ctx;
7322   DM             cdm;
7323   PetscBool      regular;
7324   PetscErrorCode ierr;
7325 
7326   PetscFunctionBegin;
7327   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7328   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7329   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7330   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7331 
7332   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7333   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7334   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7335   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7336 
7337   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7338   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7339   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7340   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7341   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7342   PetscFunctionReturn(0);
7343 }
7344 
7345 /*@
7346   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7347 
7348   Input Parameter:
7349 . dm - The DMPlex object
7350 
7351   Output Parameter:
7352 . regular - The flag
7353 
7354   Level: intermediate
7355 
7356 .seealso: DMPlexSetRegularRefinement()
7357 @*/
7358 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7359 {
7360   PetscFunctionBegin;
7361   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7362   PetscValidPointer(regular, 2);
7363   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7364   PetscFunctionReturn(0);
7365 }
7366 
7367 /*@
7368   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7369 
7370   Input Parameters:
7371 + dm - The DMPlex object
7372 - regular - The flag
7373 
7374   Level: intermediate
7375 
7376 .seealso: DMPlexGetRegularRefinement()
7377 @*/
7378 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7379 {
7380   PetscFunctionBegin;
7381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7382   ((DM_Plex *) dm->data)->regularRefinement = regular;
7383   PetscFunctionReturn(0);
7384 }
7385 
7386 /* anchors */
7387 /*@
7388   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7389   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7390 
7391   not collective
7392 
7393   Input Parameters:
7394 . dm - The DMPlex object
7395 
7396   Output Parameters:
7397 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7398 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7399 
7400 
7401   Level: intermediate
7402 
7403 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7404 @*/
7405 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7406 {
7407   DM_Plex *plex = (DM_Plex *)dm->data;
7408   PetscErrorCode ierr;
7409 
7410   PetscFunctionBegin;
7411   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7412   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7413   if (anchorSection) *anchorSection = plex->anchorSection;
7414   if (anchorIS) *anchorIS = plex->anchorIS;
7415   PetscFunctionReturn(0);
7416 }
7417 
7418 /*@
7419   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7420   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7421   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7422 
7423   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7424   DMGetConstraints() and filling in the entries in the constraint matrix.
7425 
7426   collective on dm
7427 
7428   Input Parameters:
7429 + dm - The DMPlex object
7430 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7431 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7432 
7433   The reference counts of anchorSection and anchorIS are incremented.
7434 
7435   Level: intermediate
7436 
7437 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7438 @*/
7439 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7440 {
7441   DM_Plex        *plex = (DM_Plex *)dm->data;
7442   PetscMPIInt    result;
7443   PetscErrorCode ierr;
7444 
7445   PetscFunctionBegin;
7446   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7447   if (anchorSection) {
7448     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7449     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7450     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7451   }
7452   if (anchorIS) {
7453     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7454     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7455     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7456   }
7457 
7458   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7459   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7460   plex->anchorSection = anchorSection;
7461 
7462   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7463   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7464   plex->anchorIS = anchorIS;
7465 
7466 #if defined(PETSC_USE_DEBUG)
7467   if (anchorIS && anchorSection) {
7468     PetscInt size, a, pStart, pEnd;
7469     const PetscInt *anchors;
7470 
7471     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7472     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7473     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7474     for (a = 0; a < size; a++) {
7475       PetscInt p;
7476 
7477       p = anchors[a];
7478       if (p >= pStart && p < pEnd) {
7479         PetscInt dof;
7480 
7481         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7482         if (dof) {
7483           PetscErrorCode ierr2;
7484 
7485           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7486           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7487         }
7488       }
7489     }
7490     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7491   }
7492 #endif
7493   /* reset the generic constraints */
7494   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7495   PetscFunctionReturn(0);
7496 }
7497 
7498 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7499 {
7500   PetscSection anchorSection;
7501   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7502   PetscErrorCode ierr;
7503 
7504   PetscFunctionBegin;
7505   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7506   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7507   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7508   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7509   if (numFields) {
7510     PetscInt f;
7511     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7512 
7513     for (f = 0; f < numFields; f++) {
7514       PetscInt numComp;
7515 
7516       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7517       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7518     }
7519   }
7520   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7521   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7522   pStart = PetscMax(pStart,sStart);
7523   pEnd   = PetscMin(pEnd,sEnd);
7524   pEnd   = PetscMax(pStart,pEnd);
7525   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7526   for (p = pStart; p < pEnd; p++) {
7527     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7528     if (dof) {
7529       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7530       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7531       for (f = 0; f < numFields; f++) {
7532         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7533         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7534       }
7535     }
7536   }
7537   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7538   PetscFunctionReturn(0);
7539 }
7540 
7541 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7542 {
7543   PetscSection aSec;
7544   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7545   const PetscInt *anchors;
7546   PetscInt numFields, f;
7547   IS aIS;
7548   PetscErrorCode ierr;
7549 
7550   PetscFunctionBegin;
7551   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7552   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7553   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7554   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7555   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7556   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7557   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7558   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7559   /* cSec will be a subset of aSec and section */
7560   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7561   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7562   i[0] = 0;
7563   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7564   for (p = pStart; p < pEnd; p++) {
7565     PetscInt rDof, rOff, r;
7566 
7567     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7568     if (!rDof) continue;
7569     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7570     if (numFields) {
7571       for (f = 0; f < numFields; f++) {
7572         annz = 0;
7573         for (r = 0; r < rDof; r++) {
7574           a = anchors[rOff + r];
7575           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7576           annz += aDof;
7577         }
7578         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7579         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7580         for (q = 0; q < dof; q++) {
7581           i[off + q + 1] = i[off + q] + annz;
7582         }
7583       }
7584     }
7585     else {
7586       annz = 0;
7587       for (q = 0; q < dof; q++) {
7588         a = anchors[off + q];
7589         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7590         annz += aDof;
7591       }
7592       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7593       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7594       for (q = 0; q < dof; q++) {
7595         i[off + q + 1] = i[off + q] + annz;
7596       }
7597     }
7598   }
7599   nnz = i[m];
7600   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7601   offset = 0;
7602   for (p = pStart; p < pEnd; p++) {
7603     if (numFields) {
7604       for (f = 0; f < numFields; f++) {
7605         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7606         for (q = 0; q < dof; q++) {
7607           PetscInt rDof, rOff, r;
7608           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7609           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7610           for (r = 0; r < rDof; r++) {
7611             PetscInt s;
7612 
7613             a = anchors[rOff + r];
7614             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7615             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7616             for (s = 0; s < aDof; s++) {
7617               j[offset++] = aOff + s;
7618             }
7619           }
7620         }
7621       }
7622     }
7623     else {
7624       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7625       for (q = 0; q < dof; q++) {
7626         PetscInt rDof, rOff, r;
7627         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7628         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7629         for (r = 0; r < rDof; r++) {
7630           PetscInt s;
7631 
7632           a = anchors[rOff + r];
7633           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7634           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7635           for (s = 0; s < aDof; s++) {
7636             j[offset++] = aOff + s;
7637           }
7638         }
7639       }
7640     }
7641   }
7642   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7643   ierr = PetscFree(i);CHKERRQ(ierr);
7644   ierr = PetscFree(j);CHKERRQ(ierr);
7645   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7646   PetscFunctionReturn(0);
7647 }
7648 
7649 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7650 {
7651   DM_Plex        *plex = (DM_Plex *)dm->data;
7652   PetscSection   anchorSection, section, cSec;
7653   Mat            cMat;
7654   PetscErrorCode ierr;
7655 
7656   PetscFunctionBegin;
7657   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7658   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7659   if (anchorSection) {
7660     PetscInt Nf;
7661 
7662     ierr = DMGetLocalSection(dm,&section);CHKERRQ(ierr);
7663     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7664     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7665     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7666     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7667     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7668     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7669     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7670   }
7671   PetscFunctionReturn(0);
7672 }
7673 
7674 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7675 {
7676   IS             subis;
7677   PetscSection   section, subsection;
7678   PetscErrorCode ierr;
7679 
7680   PetscFunctionBegin;
7681   ierr = DMGetLocalSection(dm, &section);CHKERRQ(ierr);
7682   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7683   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7684   /* Create subdomain */
7685   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7686   /* Create submodel */
7687   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7688   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7689   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7690   ierr = DMSetLocalSection(*subdm, subsection);CHKERRQ(ierr);
7691   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7692   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7693   /* Create map from submodel to global model */
7694   if (is) {
7695     PetscSection    sectionGlobal, subsectionGlobal;
7696     IS              spIS;
7697     const PetscInt *spmap;
7698     PetscInt       *subIndices;
7699     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7700     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7701 
7702     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7703     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7704     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7705     ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7706     ierr = DMGetGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7707     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7708     for (p = pStart; p < pEnd; ++p) {
7709       PetscInt gdof, pSubSize  = 0;
7710 
7711       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7712       if (gdof > 0) {
7713         for (f = 0; f < Nf; ++f) {
7714           PetscInt fdof, fcdof;
7715 
7716           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7717           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7718           pSubSize += fdof-fcdof;
7719         }
7720         subSize += pSubSize;
7721         if (pSubSize) {
7722           if (bs < 0) {
7723             bs = pSubSize;
7724           } else if (bs != pSubSize) {
7725             /* Layout does not admit a pointwise block size */
7726             bs = 1;
7727           }
7728         }
7729       }
7730     }
7731     /* Must have same blocksize on all procs (some might have no points) */
7732     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7733     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7734     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7735     else                            {bs = bsMinMax[0];}
7736     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7737     for (p = pStart; p < pEnd; ++p) {
7738       PetscInt gdof, goff;
7739 
7740       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7741       if (gdof > 0) {
7742         const PetscInt point = spmap[p];
7743 
7744         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7745         for (f = 0; f < Nf; ++f) {
7746           PetscInt fdof, fcdof, fc, f2, poff = 0;
7747 
7748           /* Can get rid of this loop by storing field information in the global section */
7749           for (f2 = 0; f2 < f; ++f2) {
7750             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7751             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7752             poff += fdof-fcdof;
7753           }
7754           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7755           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7756           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7757             subIndices[subOff] = goff+poff+fc;
7758           }
7759         }
7760       }
7761     }
7762     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7763     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7764     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7765     if (bs > 1) {
7766       /* We need to check that the block size does not come from non-contiguous fields */
7767       PetscInt i, j, set = 1;
7768       for (i = 0; i < subSize; i += bs) {
7769         for (j = 0; j < bs; ++j) {
7770           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7771         }
7772       }
7773       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7774     }
7775     /* Attach nullspace */
7776     for (f = 0; f < Nf; ++f) {
7777       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7778       if ((*subdm)->nullspaceConstructors[f]) break;
7779     }
7780     if (f < Nf) {
7781       MatNullSpace nullSpace;
7782 
7783       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7784       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7785       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7786     }
7787   }
7788   PetscFunctionReturn(0);
7789 }
7790