xref: /petsc/src/dm/impls/plex/plex.c (revision a8158fb5a3692ffc34efd85261445929d2a1359c)
1 #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/isimpl.h>
3 #include <petsc/private/vecimpl.h>
4 #include <petsc/private/glvisvecimpl.h>
5 #include <petscsf.h>
6 #include <petscds.h>
7 #include <petscdraw.h>
8 #include <petscdmfield.h>
9 
10 /* Logging support */
11 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF;
12 
13 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
14 
15 /*@
16   DMPlexRefineSimplexToTensor - Uniformly refines simplicial cells into tensor product cells.
17   3 quadrilaterals per triangle in 2D and 4 hexahedra per tetrahedron in 3D.
18 
19   Collective
20 
21   Input Parameters:
22 . dm - The DMPlex object
23 
24   Output Parameters:
25 . dmRefined - The refined DMPlex object
26 
27   Note: Returns NULL if the mesh is already a tensor product mesh.
28 
29   Level: intermediate
30 
31 .seealso: DMPlexCreate(), DMPlexSetRefinementUniform()
32 @*/
33 PetscErrorCode DMPlexRefineSimplexToTensor(DM dm, DM *dmRefined)
34 {
35   PetscInt         dim, cMax, fMax, cStart, cEnd, coneSize;
36   CellRefiner      cellRefiner;
37   PetscBool        lop, allnoop, localized;
38   PetscErrorCode   ierr;
39 
40   PetscFunctionBegin;
41   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42   PetscValidPointer(dmRefined, 1);
43   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
44   ierr = DMPlexGetHybridBounds(dm,&cMax,&fMax,NULL,NULL);CHKERRQ(ierr);
45   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
46   if (!(cEnd - cStart)) cellRefiner = REFINER_NOOP;
47   else {
48     ierr = DMPlexGetConeSize(dm,cStart,&coneSize);CHKERRQ(ierr);
49     switch (dim) {
50     case 1:
51       cellRefiner = REFINER_NOOP;
52     break;
53     case 2:
54       switch (coneSize) {
55       case 3:
56         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
57         else cellRefiner = REFINER_SIMPLEX_TO_HEX_2D;
58       break;
59       case 4:
60         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_2D;
61         else cellRefiner = REFINER_NOOP;
62       break;
63       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
64       }
65     break;
66     case 3:
67       switch (coneSize) {
68       case 4:
69         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
70         else cellRefiner = REFINER_SIMPLEX_TO_HEX_3D;
71       break;
72       case 5:
73         if (cMax >= 0) cellRefiner = REFINER_HYBRID_SIMPLEX_TO_HEX_3D;
74         else cellRefiner = REFINER_NOOP;
75       break;
76       case 6:
77         if (cMax >= 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Simplex2Tensor in 3D with Hybrid mesh not yet done");
78         cellRefiner = REFINER_NOOP;
79       break;
80       default: SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle coneSize %D with dimension %D",coneSize,dim);
81       }
82     break;
83     default: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot handle dimension %D",dim);
84     }
85   }
86   /* return if we don't need to refine */
87   lop = (cellRefiner == REFINER_NOOP) ? PETSC_TRUE : PETSC_FALSE;
88   ierr = MPIU_Allreduce(&lop,&allnoop,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
89   if (allnoop) {
90     *dmRefined = NULL;
91     PetscFunctionReturn(0);
92   }
93   ierr = DMPlexRefineUniform_Internal(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
94   ierr = DMCopyBoundary(dm, *dmRefined);CHKERRQ(ierr);
95   ierr = DMGetCoordinatesLocalized(dm, &localized);CHKERRQ(ierr);
96   if (localized) {
97     ierr = DMLocalizeCoordinates(*dmRefined);CHKERRQ(ierr);
98   }
99   PetscFunctionReturn(0);
100 }
101 
102 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
103 {
104   PetscInt       dim, pStart, pEnd, vStart, vEnd, cStart, cEnd, cEndInterior;
105   PetscInt       vcdof[2] = {0,0}, globalvcdof[2];
106   PetscErrorCode ierr;
107 
108   PetscFunctionBegin;
109   *ft  = PETSC_VTK_POINT_FIELD;
110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
111   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
112   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
113   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
114   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
115   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
116   if (field >= 0) {
117     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]);CHKERRQ(ierr);}
118     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]);CHKERRQ(ierr);}
119   } else {
120     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vcdof[0]);CHKERRQ(ierr);}
121     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &vcdof[1]);CHKERRQ(ierr);}
122   }
123   ierr = MPI_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
124   if (globalvcdof[0]) {
125     *sStart = vStart;
126     *sEnd   = vEnd;
127     if (globalvcdof[0] == dim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
128     else                       *ft = PETSC_VTK_POINT_FIELD;
129   } else if (globalvcdof[1]) {
130     *sStart = cStart;
131     *sEnd   = cEnd;
132     if (globalvcdof[1] == dim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
133     else                       *ft = PETSC_VTK_CELL_FIELD;
134   } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
135   PetscFunctionReturn(0);
136 }
137 
138 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
139 {
140   DM                 dm;
141   PetscSection       s;
142   PetscDraw          draw, popup;
143   DM                 cdm;
144   PetscSection       coordSection;
145   Vec                coordinates;
146   const PetscScalar *coords, *array;
147   PetscReal          bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
148   PetscReal          vbound[2], time;
149   PetscBool          isnull, flg;
150   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
151   const char        *name;
152   char               title[PETSC_MAX_PATH_LEN];
153   PetscErrorCode     ierr;
154 
155   PetscFunctionBegin;
156   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
157   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
158   if (isnull) PetscFunctionReturn(0);
159 
160   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
161   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
162   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D. Use PETSCVIEWERGLVIS", dim);
163   ierr = DMGetSection(dm, &s);CHKERRQ(ierr);
164   ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
165   ierr = DMGetCoarsenLevel(dm, &level);CHKERRQ(ierr);
166   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
167   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
168   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
169   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
170   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
171 
172   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
173   ierr = DMGetOutputSequenceNumber(dm, &step, &time);CHKERRQ(ierr);
174 
175   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
176   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
177   for (c = 0; c < N; c += dim) {
178     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
179     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
180   }
181   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
182   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
183 
184   /* Could implement something like DMDASelectFields() */
185   for (f = 0; f < Nf; ++f) {
186     DM   fdm = dm;
187     Vec  fv  = v;
188     IS   fis;
189     char prefix[PETSC_MAX_PATH_LEN];
190     const char *fname;
191 
192     ierr = PetscSectionGetFieldComponents(s, f, &Nc);CHKERRQ(ierr);
193     ierr = PetscSectionGetFieldName(s, f, &fname);CHKERRQ(ierr);
194 
195     if (v->hdr.prefix) {ierr = PetscStrncpy(prefix, v->hdr.prefix,sizeof(prefix));CHKERRQ(ierr);}
196     else               {prefix[0] = '\0';}
197     if (Nf > 1) {
198       ierr = DMCreateSubDM(dm, 1, &f, &fis, &fdm);CHKERRQ(ierr);
199       ierr = VecGetSubVector(v, fis, &fv);CHKERRQ(ierr);
200       ierr = PetscStrlcat(prefix, fname,sizeof(prefix));CHKERRQ(ierr);
201       ierr = PetscStrlcat(prefix, "_",sizeof(prefix));CHKERRQ(ierr);
202     }
203     for (comp = 0; comp < Nc; ++comp, ++w) {
204       PetscInt nmax = 2;
205 
206       ierr = PetscViewerDrawGetDraw(viewer, w, &draw);CHKERRQ(ierr);
207       if (Nc > 1) {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s_%D Step: %D Time: %.4g", name, fname, comp, step, time);CHKERRQ(ierr);}
208       else        {ierr = PetscSNPrintf(title, sizeof(title), "%s:%s Step: %D Time: %.4g", name, fname, step, time);CHKERRQ(ierr);}
209       ierr = PetscDrawSetTitle(draw, title);CHKERRQ(ierr);
210 
211       /* TODO Get max and min only for this component */
212       ierr = PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg);CHKERRQ(ierr);
213       if (!flg) {
214         ierr = VecMin(fv, NULL, &vbound[0]);CHKERRQ(ierr);
215         ierr = VecMax(fv, NULL, &vbound[1]);CHKERRQ(ierr);
216         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
217       }
218       ierr = PetscDrawGetPopup(draw, &popup);CHKERRQ(ierr);
219       ierr = PetscDrawScalePopup(popup, vbound[0], vbound[1]);CHKERRQ(ierr);
220       ierr = PetscDrawSetCoordinates(draw, bound[0], bound[1], bound[2], bound[3]);CHKERRQ(ierr);
221 
222       ierr = VecGetArrayRead(fv, &array);CHKERRQ(ierr);
223       for (c = cStart; c < cEnd; ++c) {
224         PetscScalar *coords = NULL, *a = NULL;
225         PetscInt     numCoords, color[4] = {-1,-1,-1,-1};
226 
227         ierr = DMPlexPointLocalRead(fdm, c, array, &a);CHKERRQ(ierr);
228         if (a) {
229           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
230           color[1] = color[2] = color[3] = color[0];
231         } else {
232           PetscScalar *vals = NULL;
233           PetscInt     numVals, va;
234 
235           ierr = DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
236           if (numVals % Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %D does not divide the number of values in the closure %D", Nc, numVals);
237           switch (numVals/Nc) {
238           case 3: /* P1 Triangle */
239           case 4: /* P1 Quadrangle */
240             for (va = 0; va < numVals/Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp]), vbound[0], vbound[1]);
241             break;
242           case 6: /* P2 Triangle */
243           case 8: /* P2 Quadrangle */
244             for (va = 0; va < numVals/(Nc*2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va*Nc+comp + numVals/(Nc*2)]), vbound[0], vbound[1]);
245             break;
246           default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %D cannot be handled", numVals/Nc);
247           }
248           ierr = DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals);CHKERRQ(ierr);
249         }
250         ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
251         switch (numCoords) {
252         case 6:
253           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
254           break;
255         case 8:
256           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]);CHKERRQ(ierr);
257           ierr = PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]);CHKERRQ(ierr);
258           break;
259         default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D coordinates", numCoords);
260         }
261         ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
262       }
263       ierr = VecRestoreArrayRead(fv, &array);CHKERRQ(ierr);
264       ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
265       ierr = PetscDrawPause(draw);CHKERRQ(ierr);
266       ierr = PetscDrawSave(draw);CHKERRQ(ierr);
267     }
268     if (Nf > 1) {
269       ierr = VecRestoreSubVector(v, fis, &fv);CHKERRQ(ierr);
270       ierr = ISDestroy(&fis);CHKERRQ(ierr);
271       ierr = DMDestroy(&fdm);CHKERRQ(ierr);
272     }
273   }
274   PetscFunctionReturn(0);
275 }
276 
277 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
278 {
279   DM                      dm;
280   Vec                     locv;
281   const char              *name;
282   PetscSection            section;
283   PetscInt                pStart, pEnd;
284   PetscViewerVTKFieldType ft;
285   PetscErrorCode          ierr;
286 
287   PetscFunctionBegin;
288   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
289   ierr = DMCreateLocalVector(dm, &locv);CHKERRQ(ierr); /* VTK viewer requires exclusive ownership of the vector */
290   ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
291   ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
292   ierr = VecCopy(v, locv);CHKERRQ(ierr);
293   ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
294   ierr = DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft);CHKERRQ(ierr);
295   ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, PETSC_TRUE,(PetscObject) locv);CHKERRQ(ierr);
296   PetscFunctionReturn(0);
297 }
298 
299 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
300 {
301   DM             dm;
302   PetscBool      isvtk, ishdf5, isdraw, isglvis;
303   PetscErrorCode ierr;
304 
305   PetscFunctionBegin;
306   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
307   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
308   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
309   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
310   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
311   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
312   if (isvtk || ishdf5 || isdraw || isglvis) {
313     PetscInt    i,numFields;
314     PetscObject fe;
315     PetscBool   fem = PETSC_FALSE;
316     Vec         locv = v;
317     const char  *name;
318     PetscInt    step;
319     PetscReal   time;
320 
321     ierr = DMGetNumFields(dm, &numFields);CHKERRQ(ierr);
322     for (i=0; i<numFields; i++) {
323       ierr = DMGetField(dm, i, NULL, &fe);CHKERRQ(ierr);
324       if (fe->classid == PETSCFE_CLASSID) { fem = PETSC_TRUE; break; }
325     }
326     if (fem) {
327       ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
328       ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
329       ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
330       ierr = VecCopy(v, locv);CHKERRQ(ierr);
331       ierr = DMGetOutputSequenceNumber(dm, NULL, &time);CHKERRQ(ierr);
332       ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL);CHKERRQ(ierr);
333     }
334     if (isvtk) {
335       ierr = VecView_Plex_Local_VTK(locv, viewer);CHKERRQ(ierr);
336     } else if (ishdf5) {
337 #if defined(PETSC_HAVE_HDF5)
338       ierr = VecView_Plex_Local_HDF5_Internal(locv, viewer);CHKERRQ(ierr);
339 #else
340       SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
341 #endif
342     } else if (isdraw) {
343       ierr = VecView_Plex_Local_Draw(locv, viewer);CHKERRQ(ierr);
344     } else if (isglvis) {
345       ierr = DMGetOutputSequenceNumber(dm, &step, NULL);CHKERRQ(ierr);
346       ierr = PetscViewerGLVisSetSnapId(viewer, step);CHKERRQ(ierr);
347       ierr = VecView_GLVis(locv, viewer);CHKERRQ(ierr);
348     }
349     if (fem) {ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);}
350   } else {
351     PetscBool isseq;
352 
353     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
354     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
355     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
356   }
357   PetscFunctionReturn(0);
358 }
359 
360 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
361 {
362   DM             dm;
363   PetscBool      isvtk, ishdf5, isdraw, isglvis;
364   PetscErrorCode ierr;
365 
366   PetscFunctionBegin;
367   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
368   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
369   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
370   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
371   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
372   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
373   if (isvtk || isdraw || isglvis) {
374     Vec         locv;
375     const char *name;
376 
377     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
378     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
379     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
380     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
381     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
382     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
383     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
384   } else if (ishdf5) {
385 #if defined(PETSC_HAVE_HDF5)
386     ierr = VecView_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
387 #else
388     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
389 #endif
390   } else {
391     PetscBool isseq;
392 
393     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
394     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
395     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
396   }
397   PetscFunctionReturn(0);
398 }
399 
400 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
401 {
402   DM                dm;
403   MPI_Comm          comm;
404   PetscViewerFormat format;
405   Vec               v;
406   PetscBool         isvtk, ishdf5;
407   PetscErrorCode    ierr;
408 
409   PetscFunctionBegin;
410   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
411   ierr = PetscObjectGetComm((PetscObject) originalv, &comm);CHKERRQ(ierr);
412   if (!dm) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
413   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
414   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
415   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,  &isvtk);CHKERRQ(ierr);
416   if (format == PETSC_VIEWER_NATIVE) {
417     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
418     /* this need a better fix */
419     if (dm->useNatural) {
420       if (dm->sfNatural) {
421         const char *vecname;
422         PetscInt    n, nroots;
423 
424         ierr = VecGetLocalSize(originalv, &n);CHKERRQ(ierr);
425         ierr = PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
426         if (n == nroots) {
427           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
428           ierr = DMPlexGlobalToNaturalBegin(dm, originalv, v);CHKERRQ(ierr);
429           ierr = DMPlexGlobalToNaturalEnd(dm, originalv, v);CHKERRQ(ierr);
430           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
431           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
432         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
433       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
434     } else v = originalv;
435   } else v = originalv;
436 
437   if (ishdf5) {
438 #if defined(PETSC_HAVE_HDF5)
439     ierr = VecView_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
440 #else
441     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
442 #endif
443   } else if (isvtk) {
444     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
445   } else {
446     PetscBool isseq;
447 
448     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
449     if (isseq) {ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);}
450     else       {ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);}
451   }
452   if (v != originalv) {ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);}
453   PetscFunctionReturn(0);
454 }
455 
456 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
457 {
458   DM             dm;
459   PetscBool      ishdf5;
460   PetscErrorCode ierr;
461 
462   PetscFunctionBegin;
463   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
464   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
465   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
466   if (ishdf5) {
467     DM          dmBC;
468     Vec         gv;
469     const char *name;
470 
471     ierr = DMGetOutputDM(dm, &dmBC);CHKERRQ(ierr);
472     ierr = DMGetGlobalVector(dmBC, &gv);CHKERRQ(ierr);
473     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
474     ierr = PetscObjectSetName((PetscObject) gv, name);CHKERRQ(ierr);
475     ierr = VecLoad_Default(gv, viewer);CHKERRQ(ierr);
476     ierr = DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
477     ierr = DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v);CHKERRQ(ierr);
478     ierr = DMRestoreGlobalVector(dmBC, &gv);CHKERRQ(ierr);
479   } else {
480     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
481   }
482   PetscFunctionReturn(0);
483 }
484 
485 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
486 {
487   DM             dm;
488   PetscBool      ishdf5;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
493   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
494   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
495   if (ishdf5) {
496 #if defined(PETSC_HAVE_HDF5)
497     ierr = VecLoad_Plex_HDF5_Internal(v, viewer);CHKERRQ(ierr);
498 #else
499     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
500 #endif
501   } else {
502     ierr = VecLoad_Default(v, viewer);CHKERRQ(ierr);
503   }
504   PetscFunctionReturn(0);
505 }
506 
507 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
508 {
509   DM                dm;
510   PetscViewerFormat format;
511   PetscBool         ishdf5;
512   PetscErrorCode    ierr;
513 
514   PetscFunctionBegin;
515   ierr = VecGetDM(originalv, &dm);CHKERRQ(ierr);
516   if (!dm) SETERRQ(PetscObjectComm((PetscObject) originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
517   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
518   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5, &ishdf5);CHKERRQ(ierr);
519   if (format == PETSC_VIEWER_NATIVE) {
520     if (dm->useNatural) {
521       if (dm->sfNatural) {
522         if (ishdf5) {
523 #if defined(PETSC_HAVE_HDF5)
524           Vec         v;
525           const char *vecname;
526 
527           ierr = DMGetGlobalVector(dm, &v);CHKERRQ(ierr);
528           ierr = PetscObjectGetName((PetscObject) originalv, &vecname);CHKERRQ(ierr);
529           ierr = PetscObjectSetName((PetscObject) v, vecname);CHKERRQ(ierr);
530           ierr = VecLoad_Plex_HDF5_Native_Internal(v, viewer);CHKERRQ(ierr);
531           ierr = DMPlexNaturalToGlobalBegin(dm, v, originalv);CHKERRQ(ierr);
532           ierr = DMPlexNaturalToGlobalEnd(dm, v, originalv);CHKERRQ(ierr);
533           ierr = DMRestoreGlobalVector(dm, &v);CHKERRQ(ierr);
534 #else
535           SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
536 #endif
537         } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
538       }
539     } else {
540       ierr = VecLoad_Default(originalv, viewer);CHKERRQ(ierr);
541     }
542   }
543   PetscFunctionReturn(0);
544 }
545 
546 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
547 {
548   PetscSection       coordSection;
549   Vec                coordinates;
550   DMLabel            depthLabel;
551   const char        *name[4];
552   const PetscScalar *a;
553   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
554   PetscErrorCode     ierr;
555 
556   PetscFunctionBegin;
557   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
558   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
559   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
560   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
561   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
562   ierr = PetscSectionGetChart(coordSection, &pStart, &pEnd);CHKERRQ(ierr);
563   ierr = VecGetArrayRead(coordinates, &a);CHKERRQ(ierr);
564   name[0]     = "vertex";
565   name[1]     = "edge";
566   name[dim-1] = "face";
567   name[dim]   = "cell";
568   for (c = cStart; c < cEnd; ++c) {
569     PetscInt *closure = NULL;
570     PetscInt  closureSize, cl;
571 
572     ierr = PetscViewerASCIIPrintf(viewer, "Geometry for cell %D:\n", c);CHKERRQ(ierr);
573     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
574     ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
575     for (cl = 0; cl < closureSize*2; cl += 2) {
576       PetscInt point = closure[cl], depth, dof, off, d, p;
577 
578       if ((point < pStart) || (point >= pEnd)) continue;
579       ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
580       if (!dof) continue;
581       ierr = DMLabelGetValue(depthLabel, point, &depth);CHKERRQ(ierr);
582       ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
583       ierr = PetscViewerASCIIPrintf(viewer, "%s %D coords:", name[depth], point);CHKERRQ(ierr);
584       for (p = 0; p < dof/dim; ++p) {
585         ierr = PetscViewerASCIIPrintf(viewer, " (");CHKERRQ(ierr);
586         for (d = 0; d < dim; ++d) {
587           if (d > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
588           ierr = PetscViewerASCIIPrintf(viewer, "%g", PetscRealPart(a[off+p*dim+d]));CHKERRQ(ierr);
589         }
590         ierr = PetscViewerASCIIPrintf(viewer, ")");CHKERRQ(ierr);
591       }
592       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
593     }
594     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
595     ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
596   }
597   ierr = VecRestoreArrayRead(coordinates, &a);CHKERRQ(ierr);
598   PetscFunctionReturn(0);
599 }
600 
601 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
602 {
603   DM_Plex          *mesh = (DM_Plex*) dm->data;
604   DM                cdm;
605   DMLabel           markers;
606   PetscSection      coordSection;
607   Vec               coordinates;
608   PetscViewerFormat format;
609   PetscErrorCode    ierr;
610 
611   PetscFunctionBegin;
612   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
613   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
614   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
615   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
616   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
617     const char *name;
618     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
619     PetscInt    pStart, pEnd, p;
620     PetscMPIInt rank, size;
621 
622     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
623     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
624     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
625     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
626     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
627     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
628     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
629     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
630     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
631     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
632     ierr = PetscViewerASCIIPrintf(viewer, "Supports:\n", name);CHKERRQ(ierr);
633     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
634     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %D\n", rank, maxSupportSize);CHKERRQ(ierr);
635     for (p = pStart; p < pEnd; ++p) {
636       PetscInt dof, off, s;
637 
638       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
639       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
640       for (s = off; s < off+dof; ++s) {
641         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
642       }
643     }
644     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
645     ierr = PetscViewerASCIIPrintf(viewer, "Cones:\n", name);CHKERRQ(ierr);
646     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %D\n", rank, maxConeSize);CHKERRQ(ierr);
647     for (p = pStart; p < pEnd; ++p) {
648       PetscInt dof, off, c;
649 
650       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
651       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
652       for (c = off; c < off+dof; ++c) {
653         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
654       }
655     }
656     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
657     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
658     if (coordSection && coordinates) {
659       ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);
660     }
661     ierr = DMGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
662     if (markers) {ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);}
663     if (size > 1) {
664       PetscSF sf;
665 
666       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
667       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
668     }
669     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
670   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
671     const char  *name, *color;
672     const char  *defcolors[3]  = {"gray", "orange", "green"};
673     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
674     PetscReal    scale         = 2.0;
675     PetscReal    tikzscale     = 1.0;
676     PetscBool    useNumbers    = PETSC_TRUE, useLabels, useColors;
677     double       tcoords[3];
678     PetscScalar *coords;
679     PetscInt     numLabels, l, numColors, numLColors, dim, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
680     PetscMPIInt  rank, size;
681     char         **names, **colors, **lcolors;
682     PetscBool    plotEdges, flg;
683 
684     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
685     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
686     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
687     numLabels  = PetscMax(numLabels, 10);
688     numColors  = 10;
689     numLColors = 10;
690     ierr = PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors);CHKERRQ(ierr);
691     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_scale", &scale, NULL);CHKERRQ(ierr);
692     ierr = PetscOptionsGetReal(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL);CHKERRQ(ierr);
693     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL);CHKERRQ(ierr);
694     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels);CHKERRQ(ierr);
695     if (!useLabels) numLabels = 0;
696     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors);CHKERRQ(ierr);
697     if (!useColors) {
698       numColors = 3;
699       for (c = 0; c < numColors; ++c) {ierr = PetscStrallocpy(defcolors[c], &colors[c]);CHKERRQ(ierr);}
700     }
701     ierr = PetscOptionsGetStringArray(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors);CHKERRQ(ierr);
702     if (!useColors) {
703       numLColors = 4;
704       for (c = 0; c < numLColors; ++c) {ierr = PetscStrallocpy(deflcolors[c], &lcolors[c]);CHKERRQ(ierr);}
705     }
706     plotEdges = (PetscBool)(depth > 1 && useNumbers && dim < 3);
707     ierr = PetscOptionsGetBool(((PetscObject) viewer)->options,((PetscObject) viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg);CHKERRQ(ierr);
708     if (flg && plotEdges && depth < dim) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Mesh must be interpolated");
709     if (depth < dim) plotEdges = PETSC_FALSE;
710     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
711     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
712     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
713     ierr = PetscViewerASCIIPrintf(viewer, "\
714 \\documentclass[tikz]{standalone}\n\n\
715 \\usepackage{pgflibraryshapes}\n\
716 \\usetikzlibrary{backgrounds}\n\
717 \\usetikzlibrary{arrows}\n\
718 \\begin{document}\n");CHKERRQ(ierr);
719     if (size > 1) {
720       ierr = PetscViewerASCIIPrintf(viewer, "%s for process ", name);CHKERRQ(ierr);
721       for (p = 0; p < size; ++p) {
722         if (p > 0 && p == size-1) {
723           ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
724         } else if (p > 0) {
725           ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
726         }
727         ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
728       }
729       ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n");CHKERRQ(ierr);
730     }
731     ierr = PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", tikzscale);CHKERRQ(ierr);
732     /* Plot vertices */
733     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
734     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
735     ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
736     for (v = vStart; v < vEnd; ++v) {
737       PetscInt  off, dof, d;
738       PetscBool isLabeled = PETSC_FALSE;
739 
740       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
741       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
742       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
743       if (PetscUnlikely(dof > 3)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"coordSection vertex %D has dof %D > 3",v,dof);
744       for (d = 0; d < dof; ++d) {
745         tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
746         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
747       }
748       /* Rotate coordinates since PGF makes z point out of the page instead of up */
749       if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
750       for (d = 0; d < dof; ++d) {
751         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
752         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", tcoords[d]);CHKERRQ(ierr);
753       }
754       color = colors[rank%numColors];
755       for (l = 0; l < numLabels; ++l) {
756         PetscInt val;
757         ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
758         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
759       }
760       if (useNumbers) {
761         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", v, rank, color, v);CHKERRQ(ierr);
762       } else {
763         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
764       }
765     }
766     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
767     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
768     /* Plot cells */
769     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
770     ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
771     if (dim == 3 || !useNumbers) {
772       for (e = eStart; e < eEnd; ++e) {
773         const PetscInt *cone;
774 
775         color = colors[rank%numColors];
776         for (l = 0; l < numLabels; ++l) {
777           PetscInt val;
778           ierr = DMGetLabelValue(dm, names[l], e, &val);CHKERRQ(ierr);
779           if (val >= 0) {color = lcolors[l%numLColors]; break;}
780         }
781         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
782         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%D_%d) -- (%D_%d);\n", color, cone[0], rank, cone[1], rank);CHKERRQ(ierr);
783       }
784     } else {
785       for (c = cStart; c < cEnd; ++c) {
786         PetscInt *closure = NULL;
787         PetscInt  closureSize, firstPoint = -1;
788 
789         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
790         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
791         for (p = 0; p < closureSize*2; p += 2) {
792           const PetscInt point = closure[p];
793 
794           if ((point < vStart) || (point >= vEnd)) continue;
795           if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
796           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%d)", point, rank);CHKERRQ(ierr);
797           if (firstPoint < 0) firstPoint = point;
798         }
799         /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
800         ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%d);\n", firstPoint, rank);CHKERRQ(ierr);
801         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
802       }
803     }
804     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
805     for (c = cStart; c < cEnd; ++c) {
806       double    ccoords[3] = {0.0, 0.0, 0.0};
807       PetscBool isLabeled  = PETSC_FALSE;
808       PetscInt *closure    = NULL;
809       PetscInt  closureSize, dof, d, n = 0;
810 
811       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
812       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\path (");CHKERRQ(ierr);
813       for (p = 0; p < closureSize*2; p += 2) {
814         const PetscInt point = closure[p];
815         PetscInt       off;
816 
817         if ((point < vStart) || (point >= vEnd)) continue;
818         ierr = PetscSectionGetDof(coordSection, point, &dof);CHKERRQ(ierr);
819         ierr = PetscSectionGetOffset(coordSection, point, &off);CHKERRQ(ierr);
820         for (d = 0; d < dof; ++d) {
821           tcoords[d] = (double) (scale*PetscRealPart(coords[off+d]));
822           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
823         }
824         /* Rotate coordinates since PGF makes z point out of the page instead of up */
825         if (dof == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
826         for (d = 0; d < dof; ++d) {ccoords[d] += tcoords[d];}
827         ++n;
828       }
829       for (d = 0; d < dof; ++d) {ccoords[d] /= n;}
830       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
831       for (d = 0; d < dof; ++d) {
832         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
833         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", ccoords[d]);CHKERRQ(ierr);
834       }
835       color = colors[rank%numColors];
836       for (l = 0; l < numLabels; ++l) {
837         PetscInt val;
838         ierr = DMGetLabelValue(dm, names[l], c, &val);CHKERRQ(ierr);
839         if (val >= 0) {color = lcolors[l%numLColors]; isLabeled = PETSC_TRUE; break;}
840       }
841       if (useNumbers) {
842         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D};\n", c, rank, color, c);CHKERRQ(ierr);
843       } else {
844         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color);CHKERRQ(ierr);
845       }
846     }
847     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
848     /* Plot edges */
849     if (plotEdges) {
850       ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
851       ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
852       for (e = eStart; e < eEnd; ++e) {
853         const PetscInt *cone;
854         PetscInt        coneSize, offA, offB, dof, d;
855 
856         ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
857         if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %D cone should have two vertices, not %D", e, coneSize);
858         ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
859         ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
860         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
861         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
862         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
863         for (d = 0; d < dof; ++d) {
864           tcoords[d] = (double) (0.5*scale*PetscRealPart(coords[offA+d]+coords[offB+d]));
865           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
866         }
867         /* Rotate coordinates since PGF makes z point out of the page instead of up */
868         if (dim == 3) {PetscReal tmp = tcoords[1]; tcoords[1] = tcoords[2]; tcoords[2] = -tmp;}
869         for (d = 0; d < dof; ++d) {
870           if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
871           ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]);CHKERRQ(ierr);
872         }
873         color = colors[rank%numColors];
874         for (l = 0; l < numLabels; ++l) {
875           PetscInt val;
876           ierr = DMGetLabelValue(dm, names[l], v, &val);CHKERRQ(ierr);
877           if (val >= 0) {color = lcolors[l%numLColors]; break;}
878         }
879         ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%d) [draw,shape=circle,color=%s] {%D} --\n", e, rank, color, e);CHKERRQ(ierr);
880       }
881       ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
882       ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
883       ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
884     }
885     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
886     ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);
887     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n");CHKERRQ(ierr);
888     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
889     for (l = 0; l < numLabels;  ++l) {ierr = PetscFree(names[l]);CHKERRQ(ierr);}
890     for (c = 0; c < numColors;  ++c) {ierr = PetscFree(colors[c]);CHKERRQ(ierr);}
891     for (c = 0; c < numLColors; ++c) {ierr = PetscFree(lcolors[c]);CHKERRQ(ierr);}
892     ierr = PetscFree3(names, colors, lcolors);CHKERRQ(ierr);
893   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
894     Vec                    cown,acown;
895     VecScatter             sct;
896     ISLocalToGlobalMapping g2l;
897     IS                     gid,acis;
898     MPI_Comm               comm,ncomm = MPI_COMM_NULL;
899     MPI_Group              ggroup,ngroup;
900     PetscScalar            *array,nid;
901     const PetscInt         *idxs;
902     PetscInt               *idxs2,*start,*adjacency,*work;
903     PetscInt64             lm[3],gm[3];
904     PetscInt               i,c,cStart,cEnd,cum,numVertices,ect,ectn,cellHeight;
905     PetscMPIInt            d1,d2,rank;
906 
907     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
908     ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
909 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
910     ierr = MPI_Comm_split_type(comm,MPI_COMM_TYPE_SHARED,rank,MPI_INFO_NULL,&ncomm);CHKERRQ(ierr);
911 #endif
912     if (ncomm != MPI_COMM_NULL) {
913       ierr = MPI_Comm_group(comm,&ggroup);CHKERRQ(ierr);
914       ierr = MPI_Comm_group(ncomm,&ngroup);CHKERRQ(ierr);
915       d1   = 0;
916       ierr = MPI_Group_translate_ranks(ngroup,1,&d1,ggroup,&d2);CHKERRQ(ierr);
917       nid  = d2;
918       ierr = MPI_Group_free(&ggroup);CHKERRQ(ierr);
919       ierr = MPI_Group_free(&ngroup);CHKERRQ(ierr);
920       ierr = MPI_Comm_free(&ncomm);CHKERRQ(ierr);
921     } else nid = 0.0;
922 
923     /* Get connectivity */
924     ierr = DMPlexGetVTKCellHeight(dm,&cellHeight);CHKERRQ(ierr);
925     ierr = DMPlexCreatePartitionerGraph(dm,cellHeight,&numVertices,&start,&adjacency,&gid);CHKERRQ(ierr);
926 
927     /* filter overlapped local cells */
928     ierr = DMPlexGetHeightStratum(dm,cellHeight,&cStart,&cEnd);CHKERRQ(ierr);
929     ierr = ISGetIndices(gid,&idxs);CHKERRQ(ierr);
930     ierr = ISGetLocalSize(gid,&cum);CHKERRQ(ierr);
931     ierr = PetscMalloc1(cum,&idxs2);CHKERRQ(ierr);
932     for (c = cStart, cum = 0; c < cEnd; c++) {
933       if (idxs[c-cStart] < 0) continue;
934       idxs2[cum++] = idxs[c-cStart];
935     }
936     ierr = ISRestoreIndices(gid,&idxs);CHKERRQ(ierr);
937     if (numVertices != cum) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected %D != %D",numVertices,cum);
938     ierr = ISDestroy(&gid);CHKERRQ(ierr);
939     ierr = ISCreateGeneral(comm,numVertices,idxs2,PETSC_OWN_POINTER,&gid);CHKERRQ(ierr);
940 
941     /* support for node-aware cell locality */
942     ierr = ISCreateGeneral(comm,start[numVertices],adjacency,PETSC_USE_POINTER,&acis);CHKERRQ(ierr);
943     ierr = VecCreateSeq(PETSC_COMM_SELF,start[numVertices],&acown);CHKERRQ(ierr);
944     ierr = VecCreateMPI(comm,numVertices,PETSC_DECIDE,&cown);CHKERRQ(ierr);
945     ierr = VecGetArray(cown,&array);CHKERRQ(ierr);
946     for (c = 0; c < numVertices; c++) array[c] = nid;
947     ierr = VecRestoreArray(cown,&array);CHKERRQ(ierr);
948     ierr = VecScatterCreate(cown,acis,acown,NULL,&sct);CHKERRQ(ierr);
949     ierr = VecScatterBegin(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
950     ierr = VecScatterEnd(sct,cown,acown,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
951     ierr = ISDestroy(&acis);CHKERRQ(ierr);
952     ierr = VecScatterDestroy(&sct);CHKERRQ(ierr);
953     ierr = VecDestroy(&cown);CHKERRQ(ierr);
954 
955     /* compute edgeCut */
956     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum,start[c+1]-start[c]);
957     ierr = PetscMalloc1(cum,&work);CHKERRQ(ierr);
958     ierr = ISLocalToGlobalMappingCreateIS(gid,&g2l);CHKERRQ(ierr);
959     ierr = ISLocalToGlobalMappingSetType(g2l,ISLOCALTOGLOBALMAPPINGHASH);CHKERRQ(ierr);
960     ierr = ISDestroy(&gid);CHKERRQ(ierr);
961     ierr = VecGetArray(acown,&array);CHKERRQ(ierr);
962     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
963       PetscInt totl;
964 
965       totl = start[c+1]-start[c];
966       ierr = ISGlobalToLocalMappingApply(g2l,IS_GTOLM_MASK,totl,adjacency+start[c],NULL,work);CHKERRQ(ierr);
967       for (i = 0; i < totl; i++) {
968         if (work[i] < 0) {
969           ect  += 1;
970           ectn += (array[i + start[c]] != nid) ? 0 : 1;
971         }
972       }
973     }
974     ierr  = PetscFree(work);CHKERRQ(ierr);
975     ierr  = VecRestoreArray(acown,&array);CHKERRQ(ierr);
976     lm[0] = numVertices > 0 ?  numVertices : PETSC_MAX_INT;
977     lm[1] = -numVertices;
978     ierr  = MPIU_Allreduce(lm,gm,2,MPIU_INT64,MPI_MIN,comm);CHKERRQ(ierr);
979     ierr  = PetscViewerASCIIPrintf(viewer,"  Cell balance: %.2f (max %D, min %D",-((double)gm[1])/((double)gm[0]),-(PetscInt)gm[1],(PetscInt)gm[0]);CHKERRQ(ierr);
980     lm[0] = ect; /* edgeCut */
981     lm[1] = ectn; /* node-aware edgeCut */
982     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
983     ierr  = MPIU_Allreduce(lm,gm,3,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr);
984     ierr  = PetscViewerASCIIPrintf(viewer,", empty %D)\n",(PetscInt)gm[2]);CHKERRQ(ierr);
985 #if defined(PETSC_HAVE_MPI_SHARED_COMM)
986     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),gm[0] ? ((double)(gm[1]))/((double)gm[0]) : 1.);CHKERRQ(ierr);
987 #else
988     ierr  = PetscViewerASCIIPrintf(viewer,"  Edge Cut: %D (on node %.3f)\n",(PetscInt)(gm[0]/2),0.0);CHKERRQ(ierr);
989 #endif
990     ierr  = ISLocalToGlobalMappingDestroy(&g2l);CHKERRQ(ierr);
991     ierr  = PetscFree(start);CHKERRQ(ierr);
992     ierr  = PetscFree(adjacency);CHKERRQ(ierr);
993     ierr  = VecDestroy(&acown);CHKERRQ(ierr);
994   } else {
995     MPI_Comm    comm;
996     PetscInt   *sizes, *hybsizes;
997     PetscInt    locDepth, depth, cellHeight, dim, d, pMax[4];
998     PetscInt    pStart, pEnd, p;
999     PetscInt    numLabels, l;
1000     const char *name;
1001     PetscMPIInt size;
1002 
1003     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1004     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
1005     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1006     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
1007     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
1008     if (name) {ierr = PetscViewerASCIIPrintf(viewer, "%s in %D dimension%s:\n", name, dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1009     else      {ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimension%s:\n", dim, dim == 1 ? "" : "s");CHKERRQ(ierr);}
1010     if (cellHeight) {ierr = PetscViewerASCIIPrintf(viewer, "  Cells are at height %D\n", cellHeight);CHKERRQ(ierr);}
1011     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
1012     ierr = MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
1013     ierr = DMPlexGetHybridBounds(dm, &pMax[depth], depth > 0 ? &pMax[depth-1] : NULL, depth > 1 ? &pMax[depth - 2] : NULL, &pMax[0]);CHKERRQ(ierr);
1014     ierr = PetscCalloc2(size,&sizes,size,&hybsizes);CHKERRQ(ierr);
1015     if (depth == 1) {
1016       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1017       pEnd = pEnd - pStart;
1018       pMax[0] -= pStart;
1019       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1020       ierr = MPI_Gather(&pMax[0], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1021       ierr = PetscViewerASCIIPrintf(viewer, "  %d-cells:", 0);CHKERRQ(ierr);
1022       for (p = 0; p < size; ++p) {
1023         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1024         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1025       }
1026       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1027       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
1028       pEnd = pEnd - pStart;
1029       pMax[depth] -= pStart;
1030       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1031       ierr = MPI_Gather(&pMax[depth], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1032       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
1033       for (p = 0; p < size; ++p) {
1034         if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1035         else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1036       }
1037       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1038     } else {
1039       PetscMPIInt rank;
1040       ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
1041       for (d = 0; d <= dim; d++) {
1042         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1043         pEnd    -= pStart;
1044         pMax[d] -= pStart;
1045         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1046         ierr = MPI_Gather(&pMax[d], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
1047         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
1048         for (p = 0; p < size; ++p) {
1049           if (!rank) {
1050             if (hybsizes[p] >= 0) {ierr = PetscViewerASCIIPrintf(viewer, " %D (%D)", sizes[p], sizes[p] - hybsizes[p]);CHKERRQ(ierr);}
1051             else                  {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
1052           }
1053         }
1054         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
1055       }
1056     }
1057     ierr = PetscFree2(sizes,hybsizes);CHKERRQ(ierr);
1058     ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
1059     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
1060     for (l = 0; l < numLabels; ++l) {
1061       DMLabel         label;
1062       const char     *name;
1063       IS              valueIS;
1064       const PetscInt *values;
1065       PetscInt        numValues, v;
1066 
1067       ierr = DMGetLabelName(dm, l, &name);CHKERRQ(ierr);
1068       ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1069       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
1070       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %D strata with value/size (", name, numValues);CHKERRQ(ierr);
1071       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1072       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1073       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_FALSE);CHKERRQ(ierr);
1074       for (v = 0; v < numValues; ++v) {
1075         PetscInt size;
1076 
1077         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
1078         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
1079         ierr = PetscViewerASCIIPrintf(viewer, "%D (%D)", values[v], size);CHKERRQ(ierr);
1080       }
1081       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
1082       ierr = PetscViewerASCIIUseTabs(viewer, PETSC_TRUE);CHKERRQ(ierr);
1083       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1084       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1085     }
1086     /* If no fields are specified, people do not want to see adjacency */
1087     if (dm->Nf) {
1088       PetscInt f;
1089 
1090       for (f = 0; f < dm->Nf; ++f) {
1091         const char *name;
1092 
1093         ierr = PetscObjectGetName(dm->fields[f].disc, &name);CHKERRQ(ierr);
1094         if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Field %s:\n", name);CHKERRQ(ierr);}
1095         ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1096         if (dm->fields[f].label) {ierr = DMLabelView(dm->fields[f].label, viewer);CHKERRQ(ierr);}
1097         if (dm->fields[f].adjacency[0]) {
1098           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n");CHKERRQ(ierr);}
1099           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FVM\n");CHKERRQ(ierr);}
1100         } else {
1101           if (dm->fields[f].adjacency[1]) {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FEM\n");CHKERRQ(ierr);}
1102           else                            {ierr = PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n");CHKERRQ(ierr);}
1103         }
1104         ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1105       }
1106     }
1107     ierr = DMGetCoarseDM(dm, &cdm);CHKERRQ(ierr);
1108     if (cdm) {
1109       ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
1110       ierr = DMPlexView_Ascii(cdm, viewer);CHKERRQ(ierr);
1111       ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
1112     }
1113   }
1114   PetscFunctionReturn(0);
1115 }
1116 
1117 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1118 {
1119   PetscDraw          draw;
1120   DM                 cdm;
1121   PetscSection       coordSection;
1122   Vec                coordinates;
1123   const PetscScalar *coords;
1124   PetscReal          xyl[2],xyr[2],bound[4] = {PETSC_MAX_REAL, PETSC_MAX_REAL, PETSC_MIN_REAL, PETSC_MIN_REAL};
1125   PetscBool          isnull;
1126   PetscInt           dim, vStart, vEnd, cStart, cEnd, c, N;
1127   PetscMPIInt        rank;
1128   PetscErrorCode     ierr;
1129 
1130   PetscFunctionBegin;
1131   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
1132   if (dim != 2) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %D", dim);
1133   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
1134   ierr = DMGetSection(cdm, &coordSection);CHKERRQ(ierr);
1135   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
1136   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1137   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1138 
1139   ierr = PetscViewerDrawGetDraw(viewer, 0, &draw);CHKERRQ(ierr);
1140   ierr = PetscDrawIsNull(draw, &isnull);CHKERRQ(ierr);
1141   if (isnull) PetscFunctionReturn(0);
1142   ierr = PetscDrawSetTitle(draw, "Mesh");CHKERRQ(ierr);
1143 
1144   ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
1145   ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
1146   for (c = 0; c < N; c += dim) {
1147     bound[0] = PetscMin(bound[0], PetscRealPart(coords[c]));   bound[2] = PetscMax(bound[2], PetscRealPart(coords[c]));
1148     bound[1] = PetscMin(bound[1], PetscRealPart(coords[c+1])); bound[3] = PetscMax(bound[3], PetscRealPart(coords[c+1]));
1149   }
1150   ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
1151   ierr = MPIU_Allreduce(&bound[0],xyl,2,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1152   ierr = MPIU_Allreduce(&bound[2],xyr,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
1153   ierr = PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]);CHKERRQ(ierr);
1154   ierr = PetscDrawClear(draw);CHKERRQ(ierr);
1155 
1156   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
1157   for (c = cStart; c < cEnd; ++c) {
1158     PetscScalar *coords = NULL;
1159     PetscInt     numCoords,coneSize;
1160 
1161     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1162     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1163     switch (coneSize) {
1164     case 3:
1165       ierr = PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1166                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1167                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1168                                PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1169       break;
1170     case 4:
1171       ierr = PetscDrawRectangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]),
1172                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1173                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1174                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2,
1175                                 PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS-2) + 2);CHKERRQ(ierr);
1176       break;
1177     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1178     }
1179     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1180   }
1181   for (c = cStart; c < cEnd; ++c) {
1182     PetscScalar *coords = NULL;
1183     PetscInt     numCoords,coneSize;
1184 
1185     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1186     ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1187     switch (coneSize) {
1188     case 3:
1189       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1190       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1191       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1192       break;
1193     case 4:
1194       ierr = PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1195       ierr = PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1196       ierr = PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1197       ierr = PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK);CHKERRQ(ierr);
1198       break;
1199     default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %D facets", coneSize);
1200     }
1201     ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, &numCoords, &coords);CHKERRQ(ierr);
1202   }
1203   ierr = PetscDrawFlush(draw);CHKERRQ(ierr);
1204   ierr = PetscDrawPause(draw);CHKERRQ(ierr);
1205   ierr = PetscDrawSave(draw);CHKERRQ(ierr);
1206   PetscFunctionReturn(0);
1207 }
1208 
1209 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1210 {
1211   PetscBool      iascii, ishdf5, isvtk, isdraw, flg, isglvis;
1212   char           name[PETSC_MAX_PATH_LEN];
1213   PetscErrorCode ierr;
1214 
1215   PetscFunctionBegin;
1216   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1217   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1218   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
1219   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);CHKERRQ(ierr);
1220   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);CHKERRQ(ierr);
1221   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERDRAW,  &isdraw);CHKERRQ(ierr);
1222   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERGLVIS, &isglvis);CHKERRQ(ierr);
1223   if (iascii) {
1224     PetscViewerFormat format;
1225     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1226     if (format == PETSC_VIEWER_ASCII_GLVIS) {
1227       ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1228     } else {
1229       ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
1230     }
1231   } else if (ishdf5) {
1232 #if defined(PETSC_HAVE_HDF5)
1233     ierr = DMPlexView_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1234 #else
1235     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1236 #endif
1237   } else if (isvtk) {
1238     ierr = DMPlexVTKWriteAll((PetscObject) dm,viewer);CHKERRQ(ierr);
1239   } else if (isdraw) {
1240     ierr = DMPlexView_Draw(dm, viewer);CHKERRQ(ierr);
1241   } else if (isglvis) {
1242     ierr = DMPlexView_GLVis(dm, viewer);CHKERRQ(ierr);
1243   } else {
1244     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1245   }
1246   /* Optionally view the partition */
1247   ierr = PetscOptionsHasName(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_partition_view", &flg);CHKERRQ(ierr);
1248   if (flg) {
1249     Vec ranks;
1250     ierr = DMPlexCreateRankField(dm, &ranks);CHKERRQ(ierr);
1251     ierr = VecView(ranks, viewer);CHKERRQ(ierr);
1252     ierr = VecDestroy(&ranks);CHKERRQ(ierr);
1253   }
1254   /* Optionally view a label */
1255   ierr = PetscOptionsGetString(((PetscObject) dm)->options, ((PetscObject) dm)->prefix, "-dm_label_view", name, PETSC_MAX_PATH_LEN, &flg);CHKERRQ(ierr);
1256   if (flg) {
1257     DMLabel label;
1258     Vec     val;
1259 
1260     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
1261     if (!label) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1262     ierr = DMPlexCreateLabelField(dm, label, &val);CHKERRQ(ierr);
1263     ierr = VecView(val, viewer);CHKERRQ(ierr);
1264     ierr = VecDestroy(&val);CHKERRQ(ierr);
1265   }
1266   PetscFunctionReturn(0);
1267 }
1268 
1269 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
1270 {
1271   PetscBool      ishdf5;
1272   PetscErrorCode ierr;
1273 
1274   PetscFunctionBegin;
1275   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1276   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1277   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,   &ishdf5);CHKERRQ(ierr);
1278   if (ishdf5) {
1279 #if defined(PETSC_HAVE_HDF5)
1280     PetscViewerFormat format;
1281     ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
1282     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
1283       ierr = DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer);CHKERRQ(ierr);
1284     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1285       ierr = DMPlexLoad_HDF5_Internal(dm, viewer);CHKERRQ(ierr);
1286     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
1287 #else
1288     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1289 #endif
1290   } else {
1291     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
1292   }
1293   PetscFunctionReturn(0);
1294 }
1295 
1296 PetscErrorCode DMDestroy_Plex(DM dm)
1297 {
1298   DM_Plex       *mesh = (DM_Plex*) dm->data;
1299   PetscErrorCode ierr;
1300 
1301   PetscFunctionBegin;
1302   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",NULL);CHKERRQ(ierr);
1303   ierr = PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C", NULL);CHKERRQ(ierr);
1304   if (--mesh->refct > 0) PetscFunctionReturn(0);
1305   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
1306   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
1307   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
1308   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
1309   ierr = PetscSectionDestroy(&mesh->subdomainSection);CHKERRQ(ierr);
1310   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
1311   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
1312   ierr = PetscFree(mesh->tetgenOpts);CHKERRQ(ierr);
1313   ierr = PetscFree(mesh->triangleOpts);CHKERRQ(ierr);
1314   ierr = PetscPartitionerDestroy(&mesh->partitioner);CHKERRQ(ierr);
1315   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
1316   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
1317   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
1318   ierr = PetscSectionDestroy(&mesh->anchorSection);CHKERRQ(ierr);
1319   ierr = ISDestroy(&mesh->anchorIS);CHKERRQ(ierr);
1320   ierr = PetscSectionDestroy(&mesh->parentSection);CHKERRQ(ierr);
1321   ierr = PetscFree(mesh->parents);CHKERRQ(ierr);
1322   ierr = PetscFree(mesh->childIDs);CHKERRQ(ierr);
1323   ierr = PetscSectionDestroy(&mesh->childSection);CHKERRQ(ierr);
1324   ierr = PetscFree(mesh->children);CHKERRQ(ierr);
1325   ierr = DMDestroy(&mesh->referenceTree);CHKERRQ(ierr);
1326   ierr = PetscGridHashDestroy(&mesh->lbox);CHKERRQ(ierr);
1327   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
1328   ierr = PetscFree(mesh);CHKERRQ(ierr);
1329   PetscFunctionReturn(0);
1330 }
1331 
1332 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
1333 {
1334   PetscSection           sectionGlobal;
1335   PetscInt               bs = -1, mbs;
1336   PetscInt               localSize;
1337   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
1338   PetscErrorCode         ierr;
1339   MatType                mtype;
1340   ISLocalToGlobalMapping ltog;
1341 
1342   PetscFunctionBegin;
1343   ierr = MatInitializePackage();CHKERRQ(ierr);
1344   mtype = dm->mattype;
1345   ierr = DMGetGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
1346   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
1347   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
1348   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
1349   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
1350   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
1351   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
1352   ierr = MatGetBlockSize(*J, &mbs);CHKERRQ(ierr);
1353   if (mbs > 1) bs = mbs;
1354   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
1355   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
1356   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
1357   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
1358   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
1359   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
1360   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
1361   ierr = PetscStrcmp(mtype, MATIS, &isMatIS);CHKERRQ(ierr);
1362   if (!isShell) {
1363     PetscSection subSection;
1364     PetscBool    fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
1365     PetscInt    *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *ltogidx, lsize;
1366     PetscInt     pStart, pEnd, p, dof, cdof;
1367 
1368     /* Set localtoglobalmapping on the matrix for MatSetValuesLocal() to work (it also creates the local matrices in case of MATIS) */
1369     if (isMatIS) { /* need a different l2g map than the one computed by DMGetLocalToGlobalMapping */
1370       PetscSection section;
1371       PetscInt     size;
1372 
1373       ierr = DMGetSection(dm, &section);CHKERRQ(ierr);
1374       ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
1375       ierr = PetscMalloc1(size,&ltogidx);CHKERRQ(ierr);
1376       ierr = DMPlexGetSubdomainSection(dm, &subSection);CHKERRQ(ierr);
1377     } else {
1378       ierr = DMGetLocalToGlobalMapping(dm,&ltog);CHKERRQ(ierr);
1379     }
1380     ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
1381     for (p = pStart, lsize = 0; p < pEnd; ++p) {
1382       PetscInt bdof;
1383 
1384       ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
1385       ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
1386       dof  = dof < 0 ? -(dof+1) : dof;
1387       bdof = cdof && (dof-cdof) ? 1 : dof;
1388       if (dof) {
1389         if (bs < 0)          {bs = bdof;}
1390         else if (bs != bdof) {bs = 1; if (!isMatIS) break;}
1391       }
1392       if (isMatIS) {
1393         PetscInt loff,c,off;
1394         ierr = PetscSectionGetOffset(subSection, p, &loff);CHKERRQ(ierr);
1395         ierr = PetscSectionGetOffset(sectionGlobal, p, &off);CHKERRQ(ierr);
1396         for (c = 0; c < dof-cdof; ++c, ++lsize) ltogidx[loff+c] = off > -1 ? off+c : -(off+1)+c;
1397       }
1398     }
1399     /* Must have same blocksize on all procs (some might have no points) */
1400     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
1401     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
1402     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
1403     else                            {bs = bsMinMax[0];}
1404     bs = bs < 0 ? 1 : bs;
1405     if (isMatIS) {
1406       PetscInt l;
1407       /* Must reduce indices by blocksize */
1408       if (bs > 1) for (l = 0; l < lsize; ++l) ltogidx[l] /= bs;
1409       ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)dm), bs, lsize, ltogidx, PETSC_OWN_POINTER, &ltog);CHKERRQ(ierr);
1410     }
1411     ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
1412     if (isMatIS) {
1413       ierr = ISLocalToGlobalMappingDestroy(&ltog);CHKERRQ(ierr);
1414     }
1415     ierr = PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);CHKERRQ(ierr);
1416     ierr = DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
1417     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
1418   }
1419   ierr = MatSetDM(*J, dm);CHKERRQ(ierr);
1420   PetscFunctionReturn(0);
1421 }
1422 
1423 /*@
1424   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
1425 
1426   Not collective
1427 
1428   Input Parameter:
1429 . mesh - The DMPlex
1430 
1431   Output Parameters:
1432 . subsection - The subdomain section
1433 
1434   Level: developer
1435 
1436 .seealso:
1437 @*/
1438 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
1439 {
1440   DM_Plex       *mesh = (DM_Plex*) dm->data;
1441   PetscErrorCode ierr;
1442 
1443   PetscFunctionBegin;
1444   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1445   if (!mesh->subdomainSection) {
1446     PetscSection section;
1447     PetscSF      sf;
1448 
1449     ierr = PetscSFCreate(PETSC_COMM_SELF,&sf);CHKERRQ(ierr);
1450     ierr = DMGetSection(dm,&section);CHKERRQ(ierr);
1451     ierr = PetscSectionCreateGlobalSection(section,sf,PETSC_FALSE,PETSC_TRUE,&mesh->subdomainSection);CHKERRQ(ierr);
1452     ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
1453   }
1454   *subsection = mesh->subdomainSection;
1455   PetscFunctionReturn(0);
1456 }
1457 
1458 /*@
1459   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
1460 
1461   Not collective
1462 
1463   Input Parameter:
1464 . mesh - The DMPlex
1465 
1466   Output Parameters:
1467 + pStart - The first mesh point
1468 - pEnd   - The upper bound for mesh points
1469 
1470   Level: beginner
1471 
1472 .seealso: DMPlexCreate(), DMPlexSetChart()
1473 @*/
1474 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
1475 {
1476   DM_Plex       *mesh = (DM_Plex*) dm->data;
1477   PetscErrorCode ierr;
1478 
1479   PetscFunctionBegin;
1480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1481   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1482   PetscFunctionReturn(0);
1483 }
1484 
1485 /*@
1486   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
1487 
1488   Not collective
1489 
1490   Input Parameters:
1491 + mesh - The DMPlex
1492 . pStart - The first mesh point
1493 - pEnd   - The upper bound for mesh points
1494 
1495   Output Parameters:
1496 
1497   Level: beginner
1498 
1499 .seealso: DMPlexCreate(), DMPlexGetChart()
1500 @*/
1501 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
1502 {
1503   DM_Plex       *mesh = (DM_Plex*) dm->data;
1504   PetscErrorCode ierr;
1505 
1506   PetscFunctionBegin;
1507   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1508   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
1509   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
1510   PetscFunctionReturn(0);
1511 }
1512 
1513 /*@
1514   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
1515 
1516   Not collective
1517 
1518   Input Parameters:
1519 + mesh - The DMPlex
1520 - p - The point, which must lie in the chart set with DMPlexSetChart()
1521 
1522   Output Parameter:
1523 . size - The cone size for point p
1524 
1525   Level: beginner
1526 
1527 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1528 @*/
1529 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
1530 {
1531   DM_Plex       *mesh = (DM_Plex*) dm->data;
1532   PetscErrorCode ierr;
1533 
1534   PetscFunctionBegin;
1535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1536   PetscValidPointer(size, 3);
1537   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1538   PetscFunctionReturn(0);
1539 }
1540 
1541 /*@
1542   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
1543 
1544   Not collective
1545 
1546   Input Parameters:
1547 + mesh - The DMPlex
1548 . p - The point, which must lie in the chart set with DMPlexSetChart()
1549 - size - The cone size for point p
1550 
1551   Output Parameter:
1552 
1553   Note:
1554   This should be called after DMPlexSetChart().
1555 
1556   Level: beginner
1557 
1558 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
1559 @*/
1560 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
1561 {
1562   DM_Plex       *mesh = (DM_Plex*) dm->data;
1563   PetscErrorCode ierr;
1564 
1565   PetscFunctionBegin;
1566   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1567   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1568 
1569   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
1570   PetscFunctionReturn(0);
1571 }
1572 
1573 /*@
1574   DMPlexAddConeSize - Add the given number of in-edges to this point in the DAG
1575 
1576   Not collective
1577 
1578   Input Parameters:
1579 + mesh - The DMPlex
1580 . p - The point, which must lie in the chart set with DMPlexSetChart()
1581 - size - The additional cone size for point p
1582 
1583   Output Parameter:
1584 
1585   Note:
1586   This should be called after DMPlexSetChart().
1587 
1588   Level: beginner
1589 
1590 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexGetConeSize(), DMPlexSetChart()
1591 @*/
1592 PetscErrorCode DMPlexAddConeSize(DM dm, PetscInt p, PetscInt size)
1593 {
1594   DM_Plex       *mesh = (DM_Plex*) dm->data;
1595   PetscInt       csize;
1596   PetscErrorCode ierr;
1597 
1598   PetscFunctionBegin;
1599   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1600   ierr = PetscSectionAddDof(mesh->coneSection, p, size);CHKERRQ(ierr);
1601   ierr = PetscSectionGetDof(mesh->coneSection, p, &csize);CHKERRQ(ierr);
1602 
1603   mesh->maxConeSize = PetscMax(mesh->maxConeSize, csize);
1604   PetscFunctionReturn(0);
1605 }
1606 
1607 /*@C
1608   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
1609 
1610   Not collective
1611 
1612   Input Parameters:
1613 + dm - The DMPlex
1614 - p - The point, which must lie in the chart set with DMPlexSetChart()
1615 
1616   Output Parameter:
1617 . cone - An array of points which are on the in-edges for point p
1618 
1619   Level: beginner
1620 
1621   Fortran Notes:
1622   Since it returns an array, this routine is only available in Fortran 90, and you must
1623   include petsc.h90 in your code.
1624   You must also call DMPlexRestoreCone() after you finish using the returned array.
1625   DMPlexRestoreCone() is not needed/available in C.
1626 
1627 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexGetConeTuple(), DMPlexSetChart()
1628 @*/
1629 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
1630 {
1631   DM_Plex       *mesh = (DM_Plex*) dm->data;
1632   PetscInt       off;
1633   PetscErrorCode ierr;
1634 
1635   PetscFunctionBegin;
1636   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1637   PetscValidPointer(cone, 3);
1638   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1639   *cone = &mesh->cones[off];
1640   PetscFunctionReturn(0);
1641 }
1642 
1643 /*@C
1644   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
1645 
1646   Not collective
1647 
1648   Input Parameters:
1649 + dm - The DMPlex
1650 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1651 
1652   Output Parameter:
1653 + pConesSection - PetscSection describing the layout of pCones
1654 - pCones - An array of points which are on the in-edges for the point set p
1655 
1656   Level: intermediate
1657 
1658 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeRecursive(), DMPlexSetChart()
1659 @*/
1660 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
1661 {
1662   PetscSection        cs, newcs;
1663   PetscInt            *cones;
1664   PetscInt            *newarr=NULL;
1665   PetscInt            n;
1666   PetscErrorCode      ierr;
1667 
1668   PetscFunctionBegin;
1669   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
1670   ierr = DMPlexGetConeSection(dm, &cs);CHKERRQ(ierr);
1671   ierr = PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void**)&newarr) : NULL);CHKERRQ(ierr);
1672   if (pConesSection) *pConesSection = newcs;
1673   if (pCones) {
1674     ierr = PetscSectionGetStorageSize(newcs, &n);CHKERRQ(ierr);
1675     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1676   }
1677   PetscFunctionReturn(0);
1678 }
1679 
1680 static PetscErrorCode DMPlexGetConeRecursive_Private(DM dm, PetscInt *n_inout, const PetscInt points[], PetscInt *offset_inout, PetscInt buf[])
1681 {
1682   PetscInt p, n, cn, i;
1683   const PetscInt *cone;
1684   PetscErrorCode ierr;
1685 
1686   PetscFunctionBegin;
1687   n = *n_inout;
1688   *n_inout = 0;
1689   for (i=0; i<n; i++) {
1690     p = points[i];
1691     ierr = DMPlexGetConeSize(dm, p, &cn);CHKERRQ(ierr);
1692     if (!cn) {
1693       cn = 1;
1694       if (buf) {
1695         buf[*offset_inout] = p;
1696         ++(*offset_inout);
1697       }
1698     } else {
1699       ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
1700       ierr = DMPlexGetConeRecursive_Private(dm, &cn, cone, offset_inout, buf);CHKERRQ(ierr);
1701     }
1702     *n_inout += cn;
1703   }
1704   PetscFunctionReturn(0);
1705 }
1706 
1707 /*@C
1708   DMPlexGetConeRecursive - Like DMPlexGetConeTuple() but recursive, i.e. each cone point is expanded into a set of its own cone points until a vertex (DAG point with no cone) is reached.
1709 
1710   Not collective
1711 
1712   Input Parameters:
1713 + dm - The DMPlex
1714 - p - The IS of points, which must lie in the chart set with DMPlexSetChart()
1715 
1716   Output Parameter:
1717 . pCones - An array of recursively expanded cones, i.e. containing only vertices, and each of them can be present multiple times
1718 
1719   Level: advanced
1720 
1721 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexGetConeTuple()
1722 @*/
1723 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS p, IS *pCones)
1724 {
1725   const PetscInt      *arr=NULL;
1726   PetscInt            *cpoints=NULL;
1727   PetscInt            n, cn;
1728   PetscInt            zero;
1729   PetscErrorCode      ierr;
1730 
1731   PetscFunctionBegin;
1732   ierr = ISGetLocalSize(p, &n);CHKERRQ(ierr);
1733   ierr = ISGetIndices(p, &arr);CHKERRQ(ierr);
1734   zero = 0;
1735   /* first figure out the total number of returned points */
1736   cn = n;
1737   ierr = DMPlexGetConeRecursive_Private(dm, &cn, arr, &zero, NULL);CHKERRQ(ierr);
1738   ierr = PetscMalloc1(cn, &cpoints);CHKERRQ(ierr);
1739   /* now get recursive cones themselves */
1740   ierr = DMPlexGetConeRecursive_Private(dm, &n, arr, &zero, cpoints);CHKERRQ(ierr);
1741   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)p), n, cpoints, PETSC_OWN_POINTER, pCones);CHKERRQ(ierr);
1742   ierr = ISRestoreIndices(p, &arr);CHKERRQ(ierr);
1743   PetscFunctionReturn(0);
1744 }
1745 
1746 /*@
1747   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
1748 
1749   Not collective
1750 
1751   Input Parameters:
1752 + mesh - The DMPlex
1753 . p - The point, which must lie in the chart set with DMPlexSetChart()
1754 - cone - An array of points which are on the in-edges for point p
1755 
1756   Output Parameter:
1757 
1758   Note:
1759   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1760 
1761   Developer Note: Why not call this DMPlexSetCover()
1762 
1763   Level: beginner
1764 
1765 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp(), DMPlexSetSupport(), DMPlexSetSupportSize()
1766 @*/
1767 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
1768 {
1769   DM_Plex       *mesh = (DM_Plex*) dm->data;
1770   PetscInt       pStart, pEnd;
1771   PetscInt       dof, off, c;
1772   PetscErrorCode ierr;
1773 
1774   PetscFunctionBegin;
1775   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1776   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1777   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1778   if (dof) PetscValidPointer(cone, 3);
1779   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1780   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1781   for (c = 0; c < dof; ++c) {
1782     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
1783     mesh->cones[off+c] = cone[c];
1784   }
1785   PetscFunctionReturn(0);
1786 }
1787 
1788 /*@C
1789   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
1790 
1791   Not collective
1792 
1793   Input Parameters:
1794 + mesh - The DMPlex
1795 - p - The point, which must lie in the chart set with DMPlexSetChart()
1796 
1797   Output Parameter:
1798 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1799                     integer giving the prescription for cone traversal. If it is negative, the cone is
1800                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1801                     the index of the cone point on which to start.
1802 
1803   Level: beginner
1804 
1805   Fortran Notes:
1806   Since it returns an array, this routine is only available in Fortran 90, and you must
1807   include petsc.h90 in your code.
1808   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
1809   DMPlexRestoreConeOrientation() is not needed/available in C.
1810 
1811 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
1812 @*/
1813 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
1814 {
1815   DM_Plex       *mesh = (DM_Plex*) dm->data;
1816   PetscInt       off;
1817   PetscErrorCode ierr;
1818 
1819   PetscFunctionBegin;
1820   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1821 #if defined(PETSC_USE_DEBUG)
1822   {
1823     PetscInt dof;
1824     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1825     if (dof) PetscValidPointer(coneOrientation, 3);
1826   }
1827 #endif
1828   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1829 
1830   *coneOrientation = &mesh->coneOrientations[off];
1831   PetscFunctionReturn(0);
1832 }
1833 
1834 /*@
1835   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
1836 
1837   Not collective
1838 
1839   Input Parameters:
1840 + mesh - The DMPlex
1841 . p - The point, which must lie in the chart set with DMPlexSetChart()
1842 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
1843                     integer giving the prescription for cone traversal. If it is negative, the cone is
1844                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
1845                     the index of the cone point on which to start.
1846 
1847   Output Parameter:
1848 
1849   Note:
1850   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
1851 
1852   Level: beginner
1853 
1854 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1855 @*/
1856 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
1857 {
1858   DM_Plex       *mesh = (DM_Plex*) dm->data;
1859   PetscInt       pStart, pEnd;
1860   PetscInt       dof, off, c;
1861   PetscErrorCode ierr;
1862 
1863   PetscFunctionBegin;
1864   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1865   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1866   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1867   if (dof) PetscValidPointer(coneOrientation, 3);
1868   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1869   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1870   for (c = 0; c < dof; ++c) {
1871     PetscInt cdof, o = coneOrientation[c];
1872 
1873     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
1874     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
1875     mesh->coneOrientations[off+c] = o;
1876   }
1877   PetscFunctionReturn(0);
1878 }
1879 
1880 /*@
1881   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
1882 
1883   Not collective
1884 
1885   Input Parameters:
1886 + mesh - The DMPlex
1887 . p - The point, which must lie in the chart set with DMPlexSetChart()
1888 . conePos - The local index in the cone where the point should be put
1889 - conePoint - The mesh point to insert
1890 
1891   Level: beginner
1892 
1893 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1894 @*/
1895 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
1896 {
1897   DM_Plex       *mesh = (DM_Plex*) dm->data;
1898   PetscInt       pStart, pEnd;
1899   PetscInt       dof, off;
1900   PetscErrorCode ierr;
1901 
1902   PetscFunctionBegin;
1903   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1904   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1905   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1906   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
1907   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1908   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1909   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1910   mesh->cones[off+conePos] = conePoint;
1911   PetscFunctionReturn(0);
1912 }
1913 
1914 /*@
1915   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
1916 
1917   Not collective
1918 
1919   Input Parameters:
1920 + mesh - The DMPlex
1921 . p - The point, which must lie in the chart set with DMPlexSetChart()
1922 . conePos - The local index in the cone where the point should be put
1923 - coneOrientation - The point orientation to insert
1924 
1925   Level: beginner
1926 
1927 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
1928 @*/
1929 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
1930 {
1931   DM_Plex       *mesh = (DM_Plex*) dm->data;
1932   PetscInt       pStart, pEnd;
1933   PetscInt       dof, off;
1934   PetscErrorCode ierr;
1935 
1936   PetscFunctionBegin;
1937   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1938   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
1939   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1940   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1941   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1942   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
1943   mesh->coneOrientations[off+conePos] = coneOrientation;
1944   PetscFunctionReturn(0);
1945 }
1946 
1947 /*@
1948   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
1949 
1950   Not collective
1951 
1952   Input Parameters:
1953 + mesh - The DMPlex
1954 - p - The point, which must lie in the chart set with DMPlexSetChart()
1955 
1956   Output Parameter:
1957 . size - The support size for point p
1958 
1959   Level: beginner
1960 
1961 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
1962 @*/
1963 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
1964 {
1965   DM_Plex       *mesh = (DM_Plex*) dm->data;
1966   PetscErrorCode ierr;
1967 
1968   PetscFunctionBegin;
1969   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1970   PetscValidPointer(size, 3);
1971   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
1972   PetscFunctionReturn(0);
1973 }
1974 
1975 /*@
1976   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
1977 
1978   Not collective
1979 
1980   Input Parameters:
1981 + mesh - The DMPlex
1982 . p - The point, which must lie in the chart set with DMPlexSetChart()
1983 - size - The support size for point p
1984 
1985   Output Parameter:
1986 
1987   Note:
1988   This should be called after DMPlexSetChart().
1989 
1990   Level: beginner
1991 
1992 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
1993 @*/
1994 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
1995 {
1996   DM_Plex       *mesh = (DM_Plex*) dm->data;
1997   PetscErrorCode ierr;
1998 
1999   PetscFunctionBegin;
2000   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2001   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
2002 
2003   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
2004   PetscFunctionReturn(0);
2005 }
2006 
2007 /*@C
2008   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
2009 
2010   Not collective
2011 
2012   Input Parameters:
2013 + mesh - The DMPlex
2014 - p - The point, which must lie in the chart set with DMPlexSetChart()
2015 
2016   Output Parameter:
2017 . support - An array of points which are on the out-edges for point p
2018 
2019   Level: beginner
2020 
2021   Fortran Notes:
2022   Since it returns an array, this routine is only available in Fortran 90, and you must
2023   include petsc.h90 in your code.
2024   You must also call DMPlexRestoreSupport() after you finish using the returned array.
2025   DMPlexRestoreSupport() is not needed/available in C.
2026 
2027 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2028 @*/
2029 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
2030 {
2031   DM_Plex       *mesh = (DM_Plex*) dm->data;
2032   PetscInt       off;
2033   PetscErrorCode ierr;
2034 
2035   PetscFunctionBegin;
2036   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2037   PetscValidPointer(support, 3);
2038   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2039   *support = &mesh->supports[off];
2040   PetscFunctionReturn(0);
2041 }
2042 
2043 /*@
2044   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
2045 
2046   Not collective
2047 
2048   Input Parameters:
2049 + mesh - The DMPlex
2050 . p - The point, which must lie in the chart set with DMPlexSetChart()
2051 - support - An array of points which are on the out-edges for point p
2052 
2053   Output Parameter:
2054 
2055   Note:
2056   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
2057 
2058   Level: beginner
2059 
2060 .seealso: DMPlexSetCone(), DMPlexSetConeSize(), DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
2061 @*/
2062 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
2063 {
2064   DM_Plex       *mesh = (DM_Plex*) dm->data;
2065   PetscInt       pStart, pEnd;
2066   PetscInt       dof, off, c;
2067   PetscErrorCode ierr;
2068 
2069   PetscFunctionBegin;
2070   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2071   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2072   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2073   if (dof) PetscValidPointer(support, 3);
2074   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2075   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2076   for (c = 0; c < dof; ++c) {
2077     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
2078     mesh->supports[off+c] = support[c];
2079   }
2080   PetscFunctionReturn(0);
2081 }
2082 
2083 /*@
2084   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
2085 
2086   Not collective
2087 
2088   Input Parameters:
2089 + mesh - The DMPlex
2090 . p - The point, which must lie in the chart set with DMPlexSetChart()
2091 . supportPos - The local index in the cone where the point should be put
2092 - supportPoint - The mesh point to insert
2093 
2094   Level: beginner
2095 
2096 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
2097 @*/
2098 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
2099 {
2100   DM_Plex       *mesh = (DM_Plex*) dm->data;
2101   PetscInt       pStart, pEnd;
2102   PetscInt       dof, off;
2103   PetscErrorCode ierr;
2104 
2105   PetscFunctionBegin;
2106   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2107   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
2108   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2109   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
2110   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
2111   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
2112   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
2113   mesh->supports[off+supportPos] = supportPoint;
2114   PetscFunctionReturn(0);
2115 }
2116 
2117 /*@C
2118   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
2119 
2120   Not collective
2121 
2122   Input Parameters:
2123 + mesh - The DMPlex
2124 . p - The point, which must lie in the chart set with DMPlexSetChart()
2125 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2126 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2127 
2128   Output Parameters:
2129 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2130 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2131 
2132   Note:
2133   If using internal storage (points is NULL on input), each call overwrites the last output.
2134 
2135   Fortran Notes:
2136   Since it returns an array, this routine is only available in Fortran 90, and you must
2137   include petsc.h90 in your code.
2138 
2139   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2140 
2141   Level: beginner
2142 
2143 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2144 @*/
2145 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2146 {
2147   DM_Plex        *mesh = (DM_Plex*) dm->data;
2148   PetscInt       *closure, *fifo;
2149   const PetscInt *tmp = NULL, *tmpO = NULL;
2150   PetscInt        tmpSize, t;
2151   PetscInt        depth       = 0, maxSize;
2152   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2153   PetscErrorCode  ierr;
2154 
2155   PetscFunctionBegin;
2156   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2157   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2158   /* This is only 1-level */
2159   if (useCone) {
2160     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2161     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2162     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2163   } else {
2164     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2165     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2166   }
2167   if (depth == 1) {
2168     if (*points) {
2169       closure = *points;
2170     } else {
2171       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2172       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2173     }
2174     closure[0] = p; closure[1] = 0;
2175     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2176       closure[closureSize]   = tmp[t];
2177       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
2178     }
2179     if (numPoints) *numPoints = closureSize/2;
2180     if (points)    *points    = closure;
2181     PetscFunctionReturn(0);
2182   }
2183   {
2184     PetscInt c, coneSeries, s,supportSeries;
2185 
2186     c = mesh->maxConeSize;
2187     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2188     s = mesh->maxSupportSize;
2189     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2190     maxSize = 2*PetscMax(coneSeries,supportSeries);
2191   }
2192   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2193   if (*points) {
2194     closure = *points;
2195   } else {
2196     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2197   }
2198   closure[0] = p; closure[1] = 0;
2199   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2200     const PetscInt cp = tmp[t];
2201     const PetscInt co = tmpO ? tmpO[t] : 0;
2202 
2203     closure[closureSize]   = cp;
2204     closure[closureSize+1] = co;
2205     fifo[fifoSize]         = cp;
2206     fifo[fifoSize+1]       = co;
2207   }
2208   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2209   while (fifoSize - fifoStart) {
2210     const PetscInt q   = fifo[fifoStart];
2211     const PetscInt o   = fifo[fifoStart+1];
2212     const PetscInt rev = o >= 0 ? 0 : 1;
2213     const PetscInt off = rev ? -(o+1) : o;
2214 
2215     if (useCone) {
2216       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2217       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2218       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2219     } else {
2220       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2221       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2222       tmpO = NULL;
2223     }
2224     for (t = 0; t < tmpSize; ++t) {
2225       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2226       const PetscInt cp = tmp[i];
2227       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2228       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2229        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2230       PetscInt       co = tmpO ? tmpO[i] : 0;
2231       PetscInt       c;
2232 
2233       if (rev) {
2234         PetscInt childSize, coff;
2235         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2236         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2237         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2238       }
2239       /* Check for duplicate */
2240       for (c = 0; c < closureSize; c += 2) {
2241         if (closure[c] == cp) break;
2242       }
2243       if (c == closureSize) {
2244         closure[closureSize]   = cp;
2245         closure[closureSize+1] = co;
2246         fifo[fifoSize]         = cp;
2247         fifo[fifoSize+1]       = co;
2248         closureSize           += 2;
2249         fifoSize              += 2;
2250       }
2251     }
2252     fifoStart += 2;
2253   }
2254   if (numPoints) *numPoints = closureSize/2;
2255   if (points)    *points    = closure;
2256   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2257   PetscFunctionReturn(0);
2258 }
2259 
2260 /*@C
2261   DMPlexGetTransitiveClosure_Internal - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG with a specified initial orientation
2262 
2263   Not collective
2264 
2265   Input Parameters:
2266 + mesh - The DMPlex
2267 . p - The point, which must lie in the chart set with DMPlexSetChart()
2268 . orientation - The orientation of the point
2269 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2270 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
2271 
2272   Output Parameters:
2273 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
2274 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
2275 
2276   Note:
2277   If using internal storage (points is NULL on input), each call overwrites the last output.
2278 
2279   Fortran Notes:
2280   Since it returns an array, this routine is only available in Fortran 90, and you must
2281   include petsc.h90 in your code.
2282 
2283   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2284 
2285   Level: beginner
2286 
2287 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2288 @*/
2289 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2290 {
2291   DM_Plex        *mesh = (DM_Plex*) dm->data;
2292   PetscInt       *closure, *fifo;
2293   const PetscInt *tmp = NULL, *tmpO = NULL;
2294   PetscInt        tmpSize, t;
2295   PetscInt        depth       = 0, maxSize;
2296   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
2297   PetscErrorCode  ierr;
2298 
2299   PetscFunctionBegin;
2300   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2301   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2302   /* This is only 1-level */
2303   if (useCone) {
2304     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
2305     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
2306     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
2307   } else {
2308     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
2309     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
2310   }
2311   if (depth == 1) {
2312     if (*points) {
2313       closure = *points;
2314     } else {
2315       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
2316       ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2317     }
2318     closure[0] = p; closure[1] = ornt;
2319     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
2320       const PetscInt i = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2321       closure[closureSize]   = tmp[i];
2322       closure[closureSize+1] = tmpO ? tmpO[i] : 0;
2323     }
2324     if (numPoints) *numPoints = closureSize/2;
2325     if (points)    *points    = closure;
2326     PetscFunctionReturn(0);
2327   }
2328   {
2329     PetscInt c, coneSeries, s,supportSeries;
2330 
2331     c = mesh->maxConeSize;
2332     coneSeries = (c > 1) ? ((PetscPowInt(c,depth+1)-1)/(c-1)) : depth+1;
2333     s = mesh->maxSupportSize;
2334     supportSeries = (s > 1) ? ((PetscPowInt(s,depth+1)-1)/(s-1)) : depth+1;
2335     maxSize = 2*PetscMax(coneSeries,supportSeries);
2336   }
2337   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2338   if (*points) {
2339     closure = *points;
2340   } else {
2341     ierr = DMGetWorkArray(dm, maxSize, MPIU_INT, &closure);CHKERRQ(ierr);
2342   }
2343   closure[0] = p; closure[1] = ornt;
2344   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
2345     const PetscInt i  = ornt >= 0 ? (t+ornt)%tmpSize : (-(ornt+1) + tmpSize-t)%tmpSize;
2346     const PetscInt cp = tmp[i];
2347     PetscInt       co = tmpO ? tmpO[i] : 0;
2348 
2349     if (ornt < 0) {
2350       PetscInt childSize, coff;
2351       ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2352       coff = co < 0 ? -(tmpO[i]+1) : tmpO[i];
2353       co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2354     }
2355     closure[closureSize]   = cp;
2356     closure[closureSize+1] = co;
2357     fifo[fifoSize]         = cp;
2358     fifo[fifoSize+1]       = co;
2359   }
2360   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
2361   while (fifoSize - fifoStart) {
2362     const PetscInt q   = fifo[fifoStart];
2363     const PetscInt o   = fifo[fifoStart+1];
2364     const PetscInt rev = o >= 0 ? 0 : 1;
2365     const PetscInt off = rev ? -(o+1) : o;
2366 
2367     if (useCone) {
2368       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
2369       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
2370       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
2371     } else {
2372       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
2373       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
2374       tmpO = NULL;
2375     }
2376     for (t = 0; t < tmpSize; ++t) {
2377       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
2378       const PetscInt cp = tmp[i];
2379       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
2380       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
2381        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
2382       PetscInt       co = tmpO ? tmpO[i] : 0;
2383       PetscInt       c;
2384 
2385       if (rev) {
2386         PetscInt childSize, coff;
2387         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
2388         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
2389         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
2390       }
2391       /* Check for duplicate */
2392       for (c = 0; c < closureSize; c += 2) {
2393         if (closure[c] == cp) break;
2394       }
2395       if (c == closureSize) {
2396         closure[closureSize]   = cp;
2397         closure[closureSize+1] = co;
2398         fifo[fifoSize]         = cp;
2399         fifo[fifoSize+1]       = co;
2400         closureSize           += 2;
2401         fifoSize              += 2;
2402       }
2403     }
2404     fifoStart += 2;
2405   }
2406   if (numPoints) *numPoints = closureSize/2;
2407   if (points)    *points    = closure;
2408   ierr = DMRestoreWorkArray(dm, maxSize, MPIU_INT, &fifo);CHKERRQ(ierr);
2409   PetscFunctionReturn(0);
2410 }
2411 
2412 /*@C
2413   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
2414 
2415   Not collective
2416 
2417   Input Parameters:
2418 + mesh - The DMPlex
2419 . p - The point, which must lie in the chart set with DMPlexSetChart()
2420 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
2421 . numPoints - The number of points in the closure, so points[] is of size 2*numPoints, zeroed on exit
2422 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...], zeroed on exit
2423 
2424   Note:
2425   If not using internal storage (points is not NULL on input), this call is unnecessary
2426 
2427   Fortran Notes:
2428   Since it returns an array, this routine is only available in Fortran 90, and you must
2429   include petsc.h90 in your code.
2430 
2431   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2432 
2433   Level: beginner
2434 
2435 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
2436 @*/
2437 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
2438 {
2439   PetscErrorCode ierr;
2440 
2441   PetscFunctionBegin;
2442   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2443   if (numPoints) PetscValidIntPointer(numPoints,4);
2444   if (points) PetscValidPointer(points,5);
2445   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, points);CHKERRQ(ierr);
2446   if (numPoints) *numPoints = 0;
2447   PetscFunctionReturn(0);
2448 }
2449 
2450 /*@
2451   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
2452 
2453   Not collective
2454 
2455   Input Parameter:
2456 . mesh - The DMPlex
2457 
2458   Output Parameters:
2459 + maxConeSize - The maximum number of in-edges
2460 - maxSupportSize - The maximum number of out-edges
2461 
2462   Level: beginner
2463 
2464 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
2465 @*/
2466 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
2467 {
2468   DM_Plex *mesh = (DM_Plex*) dm->data;
2469 
2470   PetscFunctionBegin;
2471   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2472   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
2473   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
2474   PetscFunctionReturn(0);
2475 }
2476 
2477 PetscErrorCode DMSetUp_Plex(DM dm)
2478 {
2479   DM_Plex       *mesh = (DM_Plex*) dm->data;
2480   PetscInt       size;
2481   PetscErrorCode ierr;
2482 
2483   PetscFunctionBegin;
2484   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2485   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
2486   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
2487   ierr = PetscMalloc1(size, &mesh->cones);CHKERRQ(ierr);
2488   ierr = PetscCalloc1(size, &mesh->coneOrientations);CHKERRQ(ierr);
2489   if (mesh->maxSupportSize) {
2490     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2491     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
2492     ierr = PetscMalloc1(size, &mesh->supports);CHKERRQ(ierr);
2493   }
2494   PetscFunctionReturn(0);
2495 }
2496 
2497 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
2498 {
2499   PetscErrorCode ierr;
2500 
2501   PetscFunctionBegin;
2502   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
2503   ierr = DMCreateSectionSubDM(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
2504   if (subdm) {(*subdm)->useNatural = dm->useNatural;}
2505   if (dm->useNatural && dm->sfMigration) {
2506     PetscSF        sfMigrationInv,sfNatural;
2507     PetscSection   section, sectionSeq;
2508 
2509     (*subdm)->sfMigration = dm->sfMigration;
2510     ierr = PetscObjectReference((PetscObject) dm->sfMigration);CHKERRQ(ierr);
2511     ierr = DMGetSection((*subdm), &section);CHKERRQ(ierr);CHKERRQ(ierr);
2512     ierr = PetscSFCreateInverseSF((*subdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2513     ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*subdm)), &sectionSeq);CHKERRQ(ierr);
2514     ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2515 
2516     ierr = DMPlexCreateGlobalToNaturalSF(*subdm, sectionSeq, (*subdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2517     (*subdm)->sfNatural = sfNatural;
2518     ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2519     ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2520   }
2521   PetscFunctionReturn(0);
2522 }
2523 
2524 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
2525 {
2526   PetscErrorCode ierr;
2527   PetscInt       i = 0;
2528 
2529   PetscFunctionBegin;
2530   ierr = DMClone(dms[0], superdm);CHKERRQ(ierr);
2531   ierr = DMCreateSectionSuperDM(dms, len, is, superdm);CHKERRQ(ierr);
2532   (*superdm)->useNatural = PETSC_FALSE;
2533   for (i = 0; i < len; i++){
2534     if (dms[i]->useNatural && dms[i]->sfMigration) {
2535       PetscSF        sfMigrationInv,sfNatural;
2536       PetscSection   section, sectionSeq;
2537 
2538       (*superdm)->sfMigration = dms[i]->sfMigration;
2539       ierr = PetscObjectReference((PetscObject) dms[i]->sfMigration);CHKERRQ(ierr);
2540       (*superdm)->useNatural = PETSC_TRUE;
2541       ierr = DMGetSection((*superdm), &section);CHKERRQ(ierr);
2542       ierr = PetscSFCreateInverseSF((*superdm)->sfMigration, &sfMigrationInv);CHKERRQ(ierr);
2543       ierr = PetscSectionCreate(PetscObjectComm((PetscObject) (*superdm)), &sectionSeq);CHKERRQ(ierr);
2544       ierr = PetscSFDistributeSection(sfMigrationInv, section, NULL, sectionSeq);CHKERRQ(ierr);
2545 
2546       ierr = DMPlexCreateGlobalToNaturalSF(*superdm, sectionSeq, (*superdm)->sfMigration, &sfNatural);CHKERRQ(ierr);
2547       (*superdm)->sfNatural = sfNatural;
2548       ierr = PetscSectionDestroy(&sectionSeq);CHKERRQ(ierr);
2549       ierr = PetscSFDestroy(&sfMigrationInv);CHKERRQ(ierr);
2550       break;
2551     }
2552   }
2553   PetscFunctionReturn(0);
2554 }
2555 
2556 /*@
2557   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
2558 
2559   Not collective
2560 
2561   Input Parameter:
2562 . mesh - The DMPlex
2563 
2564   Output Parameter:
2565 
2566   Note:
2567   This should be called after all calls to DMPlexSetCone()
2568 
2569   Level: beginner
2570 
2571 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
2572 @*/
2573 PetscErrorCode DMPlexSymmetrize(DM dm)
2574 {
2575   DM_Plex       *mesh = (DM_Plex*) dm->data;
2576   PetscInt      *offsets;
2577   PetscInt       supportSize;
2578   PetscInt       pStart, pEnd, p;
2579   PetscErrorCode ierr;
2580 
2581   PetscFunctionBegin;
2582   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2583   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
2584   ierr = PetscLogEventBegin(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2585   /* Calculate support sizes */
2586   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2587   for (p = pStart; p < pEnd; ++p) {
2588     PetscInt dof, off, c;
2589 
2590     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2591     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2592     for (c = off; c < off+dof; ++c) {
2593       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
2594     }
2595   }
2596   for (p = pStart; p < pEnd; ++p) {
2597     PetscInt dof;
2598 
2599     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
2600 
2601     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
2602   }
2603   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
2604   /* Calculate supports */
2605   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
2606   ierr = PetscMalloc1(supportSize, &mesh->supports);CHKERRQ(ierr);
2607   ierr = PetscCalloc1(pEnd - pStart, &offsets);CHKERRQ(ierr);
2608   for (p = pStart; p < pEnd; ++p) {
2609     PetscInt dof, off, c;
2610 
2611     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
2612     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
2613     for (c = off; c < off+dof; ++c) {
2614       const PetscInt q = mesh->cones[c];
2615       PetscInt       offS;
2616 
2617       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
2618 
2619       mesh->supports[offS+offsets[q]] = p;
2620       ++offsets[q];
2621     }
2622   }
2623   ierr = PetscFree(offsets);CHKERRQ(ierr);
2624   ierr = PetscLogEventEnd(DMPLEX_Symmetrize,dm,0,0,0);CHKERRQ(ierr);
2625   PetscFunctionReturn(0);
2626 }
2627 
2628 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
2629 {
2630   IS             stratumIS;
2631   PetscErrorCode ierr;
2632 
2633   PetscFunctionBegin;
2634   if (pStart >= pEnd) PetscFunctionReturn(0);
2635 #if defined(PETSC_USE_DEBUG)
2636   {
2637     PetscInt  qStart, qEnd, numLevels, level;
2638     PetscBool overlap = PETSC_FALSE;
2639     ierr = DMLabelGetNumValues(label, &numLevels);CHKERRQ(ierr);
2640     for (level = 0; level < numLevels; level++) {
2641       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2642       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {overlap = PETSC_TRUE; break;}
2643     }
2644     if (overlap) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %D range [%D,%D) overlaps with depth %D range [%D,%D)", depth, pStart, pEnd, level, qStart, qEnd);
2645   }
2646 #endif
2647   ierr = ISCreateStride(PETSC_COMM_SELF, pEnd-pStart, pStart, 1, &stratumIS);CHKERRQ(ierr);
2648   ierr = DMLabelSetStratumIS(label, depth, stratumIS);CHKERRQ(ierr);
2649   ierr = ISDestroy(&stratumIS);CHKERRQ(ierr);
2650   PetscFunctionReturn(0);
2651 }
2652 
2653 static PetscErrorCode DMPlexCreateDimStratum(DM,DMLabel,DMLabel,PetscInt,PetscInt);
2654 
2655 /*@
2656   DMPlexStratify - The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
2657   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
2658   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
2659   the DAG.
2660 
2661   Collective on dm
2662 
2663   Input Parameter:
2664 . mesh - The DMPlex
2665 
2666   Output Parameter:
2667 
2668   Notes:
2669   Concretely, DMPlexStratify() creates a new label named "depth" containing the dimension of each element: 0 for vertices,
2670   1 for edges, and so on.  The depth label can be accessed through DMPlexGetDepthLabel() or DMPlexGetDepthStratum(), or
2671   manually via DMGetLabel().  The height is defined implicitly by height = maxDimension - depth, and can be accessed
2672   via DMPlexGetHeightStratum().  For example, cells have height 0 and faces have height 1.
2673 
2674   DMPlexStratify() should be called after all calls to DMPlexSymmetrize()
2675 
2676   Level: beginner
2677 
2678 .seealso: DMPlexCreate(), DMPlexSymmetrize()
2679 @*/
2680 PetscErrorCode DMPlexStratify(DM dm)
2681 {
2682   DM_Plex       *mesh = (DM_Plex*) dm->data;
2683   DMLabel        label;
2684   PetscInt       pStart, pEnd, p;
2685   PetscInt       numRoots = 0, numLeaves = 0;
2686   PetscInt       cMax, fMax, eMax, vMax;
2687   PetscErrorCode ierr;
2688 
2689   PetscFunctionBegin;
2690   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2691   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2692 
2693   /* Create depth label */
2694   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2695   ierr = DMCreateLabel(dm, "depth");CHKERRQ(ierr);
2696   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
2697 
2698   {
2699     /* Initialize roots and count leaves */
2700     PetscInt sMin = PETSC_MAX_INT;
2701     PetscInt sMax = PETSC_MIN_INT;
2702     PetscInt coneSize, supportSize;
2703 
2704     for (p = pStart; p < pEnd; ++p) {
2705       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2706       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2707       if (!coneSize && supportSize) {
2708         sMin = PetscMin(p, sMin);
2709         sMax = PetscMax(p, sMax);
2710         ++numRoots;
2711       } else if (!supportSize && coneSize) {
2712         ++numLeaves;
2713       } else if (!supportSize && !coneSize) {
2714         /* Isolated points */
2715         sMin = PetscMin(p, sMin);
2716         sMax = PetscMax(p, sMax);
2717       }
2718     }
2719     ierr = DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax+1);CHKERRQ(ierr);
2720   }
2721 
2722   if (numRoots + numLeaves == (pEnd - pStart)) {
2723     PetscInt sMin = PETSC_MAX_INT;
2724     PetscInt sMax = PETSC_MIN_INT;
2725     PetscInt coneSize, supportSize;
2726 
2727     for (p = pStart; p < pEnd; ++p) {
2728       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
2729       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2730       if (!supportSize && coneSize) {
2731         sMin = PetscMin(p, sMin);
2732         sMax = PetscMax(p, sMax);
2733       }
2734     }
2735     ierr = DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax+1);CHKERRQ(ierr);
2736   } else {
2737     PetscInt level = 0;
2738     PetscInt qStart, qEnd, q;
2739 
2740     ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2741     while (qEnd > qStart) {
2742       PetscInt sMin = PETSC_MAX_INT;
2743       PetscInt sMax = PETSC_MIN_INT;
2744 
2745       for (q = qStart; q < qEnd; ++q) {
2746         const PetscInt *support;
2747         PetscInt        supportSize, s;
2748 
2749         ierr = DMPlexGetSupportSize(dm, q, &supportSize);CHKERRQ(ierr);
2750         ierr = DMPlexGetSupport(dm, q, &support);CHKERRQ(ierr);
2751         for (s = 0; s < supportSize; ++s) {
2752           sMin = PetscMin(support[s], sMin);
2753           sMax = PetscMax(support[s], sMax);
2754         }
2755       }
2756       ierr = DMLabelGetNumValues(label, &level);CHKERRQ(ierr);
2757       ierr = DMPlexCreateDepthStratum(dm, label, level, sMin, sMax+1);CHKERRQ(ierr);
2758       ierr = DMLabelGetStratumBounds(label, level, &qStart, &qEnd);CHKERRQ(ierr);
2759     }
2760   }
2761   { /* just in case there is an empty process */
2762     PetscInt numValues, maxValues = 0, v;
2763 
2764     ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
2765     ierr = MPI_Allreduce(&numValues,&maxValues,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
2766     for (v = numValues; v < maxValues; v++) {
2767       ierr = DMLabelAddStratum(label, v);CHKERRQ(ierr);
2768     }
2769   }
2770   ierr = PetscObjectStateGet((PetscObject) label, &mesh->depthState);CHKERRQ(ierr);
2771 
2772   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
2773   if (cMax >= 0 || fMax >= 0 || eMax >= 0 || vMax >= 0) {
2774     PetscInt dim;
2775     DMLabel  dimLabel;
2776 
2777     ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2778     ierr = DMCreateLabel(dm, "dim");CHKERRQ(ierr);
2779     ierr = DMGetLabel(dm, "dim", &dimLabel);CHKERRQ(ierr);
2780     if (cMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim, cMax);CHKERRQ(ierr);}
2781     if (fMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, dim - 1, fMax);CHKERRQ(ierr);}
2782     if (eMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 1, eMax);CHKERRQ(ierr);}
2783     if (vMax >= 0) {ierr = DMPlexCreateDimStratum(dm, label, dimLabel, 0, vMax);CHKERRQ(ierr);}
2784   }
2785   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
2786   PetscFunctionReturn(0);
2787 }
2788 
2789 /*@C
2790   DMPlexGetJoin - Get an array for the join of the set of points
2791 
2792   Not Collective
2793 
2794   Input Parameters:
2795 + dm - The DMPlex object
2796 . numPoints - The number of input points for the join
2797 - points - The input points
2798 
2799   Output Parameters:
2800 + numCoveredPoints - The number of points in the join
2801 - coveredPoints - The points in the join
2802 
2803   Level: intermediate
2804 
2805   Note: Currently, this is restricted to a single level join
2806 
2807   Fortran Notes:
2808   Since it returns an array, this routine is only available in Fortran 90, and you must
2809   include petsc.h90 in your code.
2810 
2811   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2812 
2813 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
2814 @*/
2815 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2816 {
2817   DM_Plex       *mesh = (DM_Plex*) dm->data;
2818   PetscInt      *join[2];
2819   PetscInt       joinSize, i = 0;
2820   PetscInt       dof, off, p, c, m;
2821   PetscErrorCode ierr;
2822 
2823   PetscFunctionBegin;
2824   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2825   PetscValidIntPointer(points, 3);
2826   PetscValidIntPointer(numCoveredPoints, 4);
2827   PetscValidPointer(coveredPoints, 5);
2828   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2829   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2830   /* Copy in support of first point */
2831   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
2832   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
2833   for (joinSize = 0; joinSize < dof; ++joinSize) {
2834     join[i][joinSize] = mesh->supports[off+joinSize];
2835   }
2836   /* Check each successive support */
2837   for (p = 1; p < numPoints; ++p) {
2838     PetscInt newJoinSize = 0;
2839 
2840     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
2841     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
2842     for (c = 0; c < dof; ++c) {
2843       const PetscInt point = mesh->supports[off+c];
2844 
2845       for (m = 0; m < joinSize; ++m) {
2846         if (point == join[i][m]) {
2847           join[1-i][newJoinSize++] = point;
2848           break;
2849         }
2850       }
2851     }
2852     joinSize = newJoinSize;
2853     i        = 1-i;
2854   }
2855   *numCoveredPoints = joinSize;
2856   *coveredPoints    = join[i];
2857   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
2858   PetscFunctionReturn(0);
2859 }
2860 
2861 /*@C
2862   DMPlexRestoreJoin - Restore an array for the join of the set of points
2863 
2864   Not Collective
2865 
2866   Input Parameters:
2867 + dm - The DMPlex object
2868 . numPoints - The number of input points for the join
2869 - points - The input points
2870 
2871   Output Parameters:
2872 + numCoveredPoints - The number of points in the join
2873 - coveredPoints - The points in the join
2874 
2875   Fortran Notes:
2876   Since it returns an array, this routine is only available in Fortran 90, and you must
2877   include petsc.h90 in your code.
2878 
2879   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2880 
2881   Level: intermediate
2882 
2883 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
2884 @*/
2885 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2886 {
2887   PetscErrorCode ierr;
2888 
2889   PetscFunctionBegin;
2890   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2891   if (points) PetscValidIntPointer(points,3);
2892   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
2893   PetscValidPointer(coveredPoints, 5);
2894   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
2895   if (numCoveredPoints) *numCoveredPoints = 0;
2896   PetscFunctionReturn(0);
2897 }
2898 
2899 /*@C
2900   DMPlexGetFullJoin - Get an array for the join of the set of points
2901 
2902   Not Collective
2903 
2904   Input Parameters:
2905 + dm - The DMPlex object
2906 . numPoints - The number of input points for the join
2907 - points - The input points
2908 
2909   Output Parameters:
2910 + numCoveredPoints - The number of points in the join
2911 - coveredPoints - The points in the join
2912 
2913   Fortran Notes:
2914   Since it returns an array, this routine is only available in Fortran 90, and you must
2915   include petsc.h90 in your code.
2916 
2917   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
2918 
2919   Level: intermediate
2920 
2921 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
2922 @*/
2923 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
2924 {
2925   DM_Plex       *mesh = (DM_Plex*) dm->data;
2926   PetscInt      *offsets, **closures;
2927   PetscInt      *join[2];
2928   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
2929   PetscInt       p, d, c, m, ms;
2930   PetscErrorCode ierr;
2931 
2932   PetscFunctionBegin;
2933   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2934   PetscValidIntPointer(points, 3);
2935   PetscValidIntPointer(numCoveredPoints, 4);
2936   PetscValidPointer(coveredPoints, 5);
2937 
2938   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2939   ierr    = PetscCalloc1(numPoints, &closures);CHKERRQ(ierr);
2940   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
2941   ms      = mesh->maxSupportSize;
2942   maxSize = (ms > 1) ? ((PetscPowInt(ms,depth+1)-1)/(ms-1)) : depth + 1;
2943   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]);CHKERRQ(ierr);
2944   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]);CHKERRQ(ierr);
2945 
2946   for (p = 0; p < numPoints; ++p) {
2947     PetscInt closureSize;
2948 
2949     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
2950 
2951     offsets[p*(depth+2)+0] = 0;
2952     for (d = 0; d < depth+1; ++d) {
2953       PetscInt pStart, pEnd, i;
2954 
2955       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
2956       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
2957         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
2958           offsets[p*(depth+2)+d+1] = i;
2959           break;
2960         }
2961       }
2962       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
2963     }
2964     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
2965   }
2966   for (d = 0; d < depth+1; ++d) {
2967     PetscInt dof;
2968 
2969     /* Copy in support of first point */
2970     dof = offsets[d+1] - offsets[d];
2971     for (joinSize = 0; joinSize < dof; ++joinSize) {
2972       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
2973     }
2974     /* Check each successive cone */
2975     for (p = 1; p < numPoints && joinSize; ++p) {
2976       PetscInt newJoinSize = 0;
2977 
2978       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
2979       for (c = 0; c < dof; ++c) {
2980         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
2981 
2982         for (m = 0; m < joinSize; ++m) {
2983           if (point == join[i][m]) {
2984             join[1-i][newJoinSize++] = point;
2985             break;
2986           }
2987         }
2988       }
2989       joinSize = newJoinSize;
2990       i        = 1-i;
2991     }
2992     if (joinSize) break;
2993   }
2994   *numCoveredPoints = joinSize;
2995   *coveredPoints    = join[i];
2996   for (p = 0; p < numPoints; ++p) {
2997     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
2998   }
2999   ierr = PetscFree(closures);CHKERRQ(ierr);
3000   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3001   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, MPIU_INT, &join[1-i]);CHKERRQ(ierr);
3002   PetscFunctionReturn(0);
3003 }
3004 
3005 /*@C
3006   DMPlexGetMeet - Get an array for the meet of the set of points
3007 
3008   Not Collective
3009 
3010   Input Parameters:
3011 + dm - The DMPlex object
3012 . numPoints - The number of input points for the meet
3013 - points - The input points
3014 
3015   Output Parameters:
3016 + numCoveredPoints - The number of points in the meet
3017 - coveredPoints - The points in the meet
3018 
3019   Level: intermediate
3020 
3021   Note: Currently, this is restricted to a single level meet
3022 
3023   Fortran Notes:
3024   Since it returns an array, this routine is only available in Fortran 90, and you must
3025   include petsc.h90 in your code.
3026 
3027   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3028 
3029 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
3030 @*/
3031 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
3032 {
3033   DM_Plex       *mesh = (DM_Plex*) dm->data;
3034   PetscInt      *meet[2];
3035   PetscInt       meetSize, i = 0;
3036   PetscInt       dof, off, p, c, m;
3037   PetscErrorCode ierr;
3038 
3039   PetscFunctionBegin;
3040   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3041   PetscValidPointer(points, 2);
3042   PetscValidPointer(numCoveringPoints, 3);
3043   PetscValidPointer(coveringPoints, 4);
3044   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3045   ierr = DMGetWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3046   /* Copy in cone of first point */
3047   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
3048   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
3049   for (meetSize = 0; meetSize < dof; ++meetSize) {
3050     meet[i][meetSize] = mesh->cones[off+meetSize];
3051   }
3052   /* Check each successive cone */
3053   for (p = 1; p < numPoints; ++p) {
3054     PetscInt newMeetSize = 0;
3055 
3056     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
3057     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
3058     for (c = 0; c < dof; ++c) {
3059       const PetscInt point = mesh->cones[off+c];
3060 
3061       for (m = 0; m < meetSize; ++m) {
3062         if (point == meet[i][m]) {
3063           meet[1-i][newMeetSize++] = point;
3064           break;
3065         }
3066       }
3067     }
3068     meetSize = newMeetSize;
3069     i        = 1-i;
3070   }
3071   *numCoveringPoints = meetSize;
3072   *coveringPoints    = meet[i];
3073   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3074   PetscFunctionReturn(0);
3075 }
3076 
3077 /*@C
3078   DMPlexRestoreMeet - Restore an array for the meet of the set of points
3079 
3080   Not Collective
3081 
3082   Input Parameters:
3083 + dm - The DMPlex object
3084 . numPoints - The number of input points for the meet
3085 - points - The input points
3086 
3087   Output Parameters:
3088 + numCoveredPoints - The number of points in the meet
3089 - coveredPoints - The points in the meet
3090 
3091   Level: intermediate
3092 
3093   Fortran Notes:
3094   Since it returns an array, this routine is only available in Fortran 90, and you must
3095   include petsc.h90 in your code.
3096 
3097   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3098 
3099 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
3100 @*/
3101 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3102 {
3103   PetscErrorCode ierr;
3104 
3105   PetscFunctionBegin;
3106   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3107   if (points) PetscValidIntPointer(points,3);
3108   if (numCoveredPoints) PetscValidIntPointer(numCoveredPoints,4);
3109   PetscValidPointer(coveredPoints,5);
3110   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, (void*) coveredPoints);CHKERRQ(ierr);
3111   if (numCoveredPoints) *numCoveredPoints = 0;
3112   PetscFunctionReturn(0);
3113 }
3114 
3115 /*@C
3116   DMPlexGetFullMeet - Get an array for the meet of the set of points
3117 
3118   Not Collective
3119 
3120   Input Parameters:
3121 + dm - The DMPlex object
3122 . numPoints - The number of input points for the meet
3123 - points - The input points
3124 
3125   Output Parameters:
3126 + numCoveredPoints - The number of points in the meet
3127 - coveredPoints - The points in the meet
3128 
3129   Level: intermediate
3130 
3131   Fortran Notes:
3132   Since it returns an array, this routine is only available in Fortran 90, and you must
3133   include petsc.h90 in your code.
3134 
3135   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
3136 
3137 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
3138 @*/
3139 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
3140 {
3141   DM_Plex       *mesh = (DM_Plex*) dm->data;
3142   PetscInt      *offsets, **closures;
3143   PetscInt      *meet[2];
3144   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
3145   PetscInt       p, h, c, m, mc;
3146   PetscErrorCode ierr;
3147 
3148   PetscFunctionBegin;
3149   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3150   PetscValidPointer(points, 2);
3151   PetscValidPointer(numCoveredPoints, 3);
3152   PetscValidPointer(coveredPoints, 4);
3153 
3154   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
3155   ierr    = PetscMalloc1(numPoints, &closures);CHKERRQ(ierr);
3156   ierr    = DMGetWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3157   mc      = mesh->maxConeSize;
3158   maxSize = (mc > 1) ? ((PetscPowInt(mc,height+1)-1)/(mc-1)) : height + 1;
3159   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]);CHKERRQ(ierr);
3160   ierr    = DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]);CHKERRQ(ierr);
3161 
3162   for (p = 0; p < numPoints; ++p) {
3163     PetscInt closureSize;
3164 
3165     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
3166 
3167     offsets[p*(height+2)+0] = 0;
3168     for (h = 0; h < height+1; ++h) {
3169       PetscInt pStart, pEnd, i;
3170 
3171       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
3172       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
3173         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
3174           offsets[p*(height+2)+h+1] = i;
3175           break;
3176         }
3177       }
3178       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
3179     }
3180     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
3181   }
3182   for (h = 0; h < height+1; ++h) {
3183     PetscInt dof;
3184 
3185     /* Copy in cone of first point */
3186     dof = offsets[h+1] - offsets[h];
3187     for (meetSize = 0; meetSize < dof; ++meetSize) {
3188       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
3189     }
3190     /* Check each successive cone */
3191     for (p = 1; p < numPoints && meetSize; ++p) {
3192       PetscInt newMeetSize = 0;
3193 
3194       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
3195       for (c = 0; c < dof; ++c) {
3196         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
3197 
3198         for (m = 0; m < meetSize; ++m) {
3199           if (point == meet[i][m]) {
3200             meet[1-i][newMeetSize++] = point;
3201             break;
3202           }
3203         }
3204       }
3205       meetSize = newMeetSize;
3206       i        = 1-i;
3207     }
3208     if (meetSize) break;
3209   }
3210   *numCoveredPoints = meetSize;
3211   *coveredPoints    = meet[i];
3212   for (p = 0; p < numPoints; ++p) {
3213     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
3214   }
3215   ierr = PetscFree(closures);CHKERRQ(ierr);
3216   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), MPIU_INT, &offsets);CHKERRQ(ierr);
3217   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, MPIU_INT, &meet[1-i]);CHKERRQ(ierr);
3218   PetscFunctionReturn(0);
3219 }
3220 
3221 /*@C
3222   DMPlexEqual - Determine if two DMs have the same topology
3223 
3224   Not Collective
3225 
3226   Input Parameters:
3227 + dmA - A DMPlex object
3228 - dmB - A DMPlex object
3229 
3230   Output Parameters:
3231 . equal - PETSC_TRUE if the topologies are identical
3232 
3233   Level: intermediate
3234 
3235   Notes:
3236   We are not solving graph isomorphism, so we do not permutation.
3237 
3238 .seealso: DMPlexGetCone()
3239 @*/
3240 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
3241 {
3242   PetscInt       depth, depthB, pStart, pEnd, pStartB, pEndB, p;
3243   PetscErrorCode ierr;
3244 
3245   PetscFunctionBegin;
3246   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3247   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3248   PetscValidPointer(equal, 3);
3249 
3250   *equal = PETSC_FALSE;
3251   ierr = DMPlexGetDepth(dmA, &depth);CHKERRQ(ierr);
3252   ierr = DMPlexGetDepth(dmB, &depthB);CHKERRQ(ierr);
3253   if (depth != depthB) PetscFunctionReturn(0);
3254   ierr = DMPlexGetChart(dmA, &pStart,  &pEnd);CHKERRQ(ierr);
3255   ierr = DMPlexGetChart(dmB, &pStartB, &pEndB);CHKERRQ(ierr);
3256   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(0);
3257   for (p = pStart; p < pEnd; ++p) {
3258     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
3259     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
3260 
3261     ierr = DMPlexGetConeSize(dmA, p, &coneSize);CHKERRQ(ierr);
3262     ierr = DMPlexGetCone(dmA, p, &cone);CHKERRQ(ierr);
3263     ierr = DMPlexGetConeOrientation(dmA, p, &ornt);CHKERRQ(ierr);
3264     ierr = DMPlexGetConeSize(dmB, p, &coneSizeB);CHKERRQ(ierr);
3265     ierr = DMPlexGetCone(dmB, p, &coneB);CHKERRQ(ierr);
3266     ierr = DMPlexGetConeOrientation(dmB, p, &orntB);CHKERRQ(ierr);
3267     if (coneSize != coneSizeB) PetscFunctionReturn(0);
3268     for (c = 0; c < coneSize; ++c) {
3269       if (cone[c] != coneB[c]) PetscFunctionReturn(0);
3270       if (ornt[c] != orntB[c]) PetscFunctionReturn(0);
3271     }
3272     ierr = DMPlexGetSupportSize(dmA, p, &supportSize);CHKERRQ(ierr);
3273     ierr = DMPlexGetSupport(dmA, p, &support);CHKERRQ(ierr);
3274     ierr = DMPlexGetSupportSize(dmB, p, &supportSizeB);CHKERRQ(ierr);
3275     ierr = DMPlexGetSupport(dmB, p, &supportB);CHKERRQ(ierr);
3276     if (supportSize != supportSizeB) PetscFunctionReturn(0);
3277     for (s = 0; s < supportSize; ++s) {
3278       if (support[s] != supportB[s]) PetscFunctionReturn(0);
3279     }
3280   }
3281   *equal = PETSC_TRUE;
3282   PetscFunctionReturn(0);
3283 }
3284 
3285 /*@C
3286   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
3287 
3288   Not Collective
3289 
3290   Input Parameters:
3291 + dm         - The DMPlex
3292 . cellDim    - The cell dimension
3293 - numCorners - The number of vertices on a cell
3294 
3295   Output Parameters:
3296 . numFaceVertices - The number of vertices on a face
3297 
3298   Level: developer
3299 
3300   Notes:
3301   Of course this can only work for a restricted set of symmetric shapes
3302 
3303 .seealso: DMPlexGetCone()
3304 @*/
3305 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
3306 {
3307   MPI_Comm       comm;
3308   PetscErrorCode ierr;
3309 
3310   PetscFunctionBegin;
3311   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3312   PetscValidPointer(numFaceVertices,3);
3313   switch (cellDim) {
3314   case 0:
3315     *numFaceVertices = 0;
3316     break;
3317   case 1:
3318     *numFaceVertices = 1;
3319     break;
3320   case 2:
3321     switch (numCorners) {
3322     case 3: /* triangle */
3323       *numFaceVertices = 2; /* Edge has 2 vertices */
3324       break;
3325     case 4: /* quadrilateral */
3326       *numFaceVertices = 2; /* Edge has 2 vertices */
3327       break;
3328     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
3329       *numFaceVertices = 3; /* Edge has 3 vertices */
3330       break;
3331     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
3332       *numFaceVertices = 3; /* Edge has 3 vertices */
3333       break;
3334     default:
3335       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3336     }
3337     break;
3338   case 3:
3339     switch (numCorners) {
3340     case 4: /* tetradehdron */
3341       *numFaceVertices = 3; /* Face has 3 vertices */
3342       break;
3343     case 6: /* tet cohesive cells */
3344       *numFaceVertices = 4; /* Face has 4 vertices */
3345       break;
3346     case 8: /* hexahedron */
3347       *numFaceVertices = 4; /* Face has 4 vertices */
3348       break;
3349     case 9: /* tet cohesive Lagrange cells */
3350       *numFaceVertices = 6; /* Face has 6 vertices */
3351       break;
3352     case 10: /* quadratic tetrahedron */
3353       *numFaceVertices = 6; /* Face has 6 vertices */
3354       break;
3355     case 12: /* hex cohesive Lagrange cells */
3356       *numFaceVertices = 6; /* Face has 6 vertices */
3357       break;
3358     case 18: /* quadratic tet cohesive Lagrange cells */
3359       *numFaceVertices = 6; /* Face has 6 vertices */
3360       break;
3361     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
3362       *numFaceVertices = 9; /* Face has 9 vertices */
3363       break;
3364     default:
3365       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %D for dimension %D", numCorners, cellDim);
3366     }
3367     break;
3368   default:
3369     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %D", cellDim);
3370   }
3371   PetscFunctionReturn(0);
3372 }
3373 
3374 /*@
3375   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
3376 
3377   Not Collective
3378 
3379   Input Parameter:
3380 . dm    - The DMPlex object
3381 
3382   Output Parameter:
3383 . depthLabel - The DMLabel recording point depth
3384 
3385   Level: developer
3386 
3387 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3388 @*/
3389 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
3390 {
3391   PetscErrorCode ierr;
3392 
3393   PetscFunctionBegin;
3394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3395   PetscValidPointer(depthLabel, 2);
3396   if (!dm->depthLabel) {ierr = DMGetLabel(dm, "depth", &dm->depthLabel);CHKERRQ(ierr);}
3397   *depthLabel = dm->depthLabel;
3398   PetscFunctionReturn(0);
3399 }
3400 
3401 /*@
3402   DMPlexGetDepth - Get the depth of the DAG representing this mesh
3403 
3404   Not Collective
3405 
3406   Input Parameter:
3407 . dm    - The DMPlex object
3408 
3409   Output Parameter:
3410 . depth - The number of strata (breadth first levels) in the DAG
3411 
3412   Level: developer
3413 
3414 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
3415 @*/
3416 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
3417 {
3418   DMLabel        label;
3419   PetscInt       d = 0;
3420   PetscErrorCode ierr;
3421 
3422   PetscFunctionBegin;
3423   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3424   PetscValidPointer(depth, 2);
3425   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3426   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
3427   *depth = d-1;
3428   PetscFunctionReturn(0);
3429 }
3430 
3431 /*@
3432   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
3433 
3434   Not Collective
3435 
3436   Input Parameters:
3437 + dm           - The DMPlex object
3438 - stratumValue - The requested depth
3439 
3440   Output Parameters:
3441 + start - The first point at this depth
3442 - end   - One beyond the last point at this depth
3443 
3444   Notes:
3445   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
3446   often "vertices".  If the mesh is "interpolated" (see DMPlexInterpolate()), then depth stratum 1 contains the next
3447   higher dimension, e.g., "edges".
3448 
3449   Level: developer
3450 
3451 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
3452 @*/
3453 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3454 {
3455   DMLabel        label;
3456   PetscInt       pStart, pEnd;
3457   PetscErrorCode ierr;
3458 
3459   PetscFunctionBegin;
3460   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3461   if (start) {PetscValidPointer(start, 3); *start = 0;}
3462   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3463   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3464   if (pStart == pEnd) PetscFunctionReturn(0);
3465   if (stratumValue < 0) {
3466     if (start) *start = pStart;
3467     if (end)   *end   = pEnd;
3468     PetscFunctionReturn(0);
3469   }
3470   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3471   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3472   ierr = DMLabelGetStratumBounds(label, stratumValue, start, end);CHKERRQ(ierr);
3473   PetscFunctionReturn(0);
3474 }
3475 
3476 /*@
3477   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
3478 
3479   Not Collective
3480 
3481   Input Parameters:
3482 + dm           - The DMPlex object
3483 - stratumValue - The requested height
3484 
3485   Output Parameters:
3486 + start - The first point at this height
3487 - end   - One beyond the last point at this height
3488 
3489   Notes:
3490   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
3491   points, often called "cells" or "elements".  If the mesh is "interpolated" (see DMPlexInterpolate()), then height
3492   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
3493 
3494   Level: developer
3495 
3496 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
3497 @*/
3498 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
3499 {
3500   DMLabel        label;
3501   PetscInt       depth, pStart, pEnd;
3502   PetscErrorCode ierr;
3503 
3504   PetscFunctionBegin;
3505   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3506   if (start) {PetscValidPointer(start, 3); *start = 0;}
3507   if (end)   {PetscValidPointer(end,   4); *end   = 0;}
3508   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3509   if (pStart == pEnd) PetscFunctionReturn(0);
3510   if (stratumValue < 0) {
3511     if (start) *start = pStart;
3512     if (end)   *end   = pEnd;
3513     PetscFunctionReturn(0);
3514   }
3515   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3516   if (!label) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
3517   ierr = DMLabelGetNumValues(label, &depth);CHKERRQ(ierr);
3518   ierr = DMLabelGetStratumBounds(label, depth-1-stratumValue, start, end);CHKERRQ(ierr);
3519   PetscFunctionReturn(0);
3520 }
3521 
3522 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
3523 {
3524   PetscSection   section, s;
3525   Mat            m;
3526   PetscInt       maxHeight;
3527   PetscErrorCode ierr;
3528 
3529   PetscFunctionBegin;
3530   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
3531   ierr = DMPlexGetMaxProjectionHeight(dm, &maxHeight);CHKERRQ(ierr);
3532   ierr = DMPlexSetMaxProjectionHeight(*cdm, maxHeight);CHKERRQ(ierr);
3533   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
3534   ierr = DMSetSection(*cdm, section);CHKERRQ(ierr);
3535   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
3536   ierr = PetscSectionCreate(PETSC_COMM_SELF, &s);CHKERRQ(ierr);
3537   ierr = MatCreate(PETSC_COMM_SELF, &m);CHKERRQ(ierr);
3538   ierr = DMSetDefaultConstraints(*cdm, s, m);CHKERRQ(ierr);
3539   ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
3540   ierr = MatDestroy(&m);CHKERRQ(ierr);
3541 
3542   ierr = DMSetNumFields(*cdm, 1);CHKERRQ(ierr);
3543   ierr = DMCreateDS(*cdm);CHKERRQ(ierr);
3544   PetscFunctionReturn(0);
3545 }
3546 
3547 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
3548 {
3549   Vec            coordsLocal;
3550   DM             coordsDM;
3551   PetscErrorCode ierr;
3552 
3553   PetscFunctionBegin;
3554   *field = NULL;
3555   ierr = DMGetCoordinatesLocal(dm,&coordsLocal);CHKERRQ(ierr);
3556   ierr = DMGetCoordinateDM(dm,&coordsDM);CHKERRQ(ierr);
3557   if (coordsLocal && coordsDM) {
3558     ierr = DMFieldCreateDS(coordsDM, 0, coordsLocal, field);CHKERRQ(ierr);
3559   }
3560   PetscFunctionReturn(0);
3561 }
3562 
3563 /*@C
3564   DMPlexGetConeSection - Return a section which describes the layout of cone data
3565 
3566   Not Collective
3567 
3568   Input Parameters:
3569 . dm        - The DMPlex object
3570 
3571   Output Parameter:
3572 . section - The PetscSection object
3573 
3574   Level: developer
3575 
3576 .seealso: DMPlexGetSupportSection(), DMPlexGetCones(), DMPlexGetConeOrientations()
3577 @*/
3578 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
3579 {
3580   DM_Plex *mesh = (DM_Plex*) dm->data;
3581 
3582   PetscFunctionBegin;
3583   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3584   if (section) *section = mesh->coneSection;
3585   PetscFunctionReturn(0);
3586 }
3587 
3588 /*@C
3589   DMPlexGetSupportSection - Return a section which describes the layout of support data
3590 
3591   Not Collective
3592 
3593   Input Parameters:
3594 . dm        - The DMPlex object
3595 
3596   Output Parameter:
3597 . section - The PetscSection object
3598 
3599   Level: developer
3600 
3601 .seealso: DMPlexGetConeSection()
3602 @*/
3603 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
3604 {
3605   DM_Plex *mesh = (DM_Plex*) dm->data;
3606 
3607   PetscFunctionBegin;
3608   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3609   if (section) *section = mesh->supportSection;
3610   PetscFunctionReturn(0);
3611 }
3612 
3613 /*@C
3614   DMPlexGetCones - Return cone data
3615 
3616   Not Collective
3617 
3618   Input Parameters:
3619 . dm        - The DMPlex object
3620 
3621   Output Parameter:
3622 . cones - The cone for each point
3623 
3624   Level: developer
3625 
3626 .seealso: DMPlexGetConeSection()
3627 @*/
3628 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
3629 {
3630   DM_Plex *mesh = (DM_Plex*) dm->data;
3631 
3632   PetscFunctionBegin;
3633   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3634   if (cones) *cones = mesh->cones;
3635   PetscFunctionReturn(0);
3636 }
3637 
3638 /*@C
3639   DMPlexGetConeOrientations - Return cone orientation data
3640 
3641   Not Collective
3642 
3643   Input Parameters:
3644 . dm        - The DMPlex object
3645 
3646   Output Parameter:
3647 . coneOrientations - The cone orientation for each point
3648 
3649   Level: developer
3650 
3651 .seealso: DMPlexGetConeSection()
3652 @*/
3653 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
3654 {
3655   DM_Plex *mesh = (DM_Plex*) dm->data;
3656 
3657   PetscFunctionBegin;
3658   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3659   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
3660   PetscFunctionReturn(0);
3661 }
3662 
3663 /******************************** FEM Support **********************************/
3664 
3665 /*
3666  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
3667  representing a line in the section.
3668 */
3669 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(PetscSection section,PetscInt field,PetscInt line,PetscBool vertexchart,PetscInt *Nc,PetscInt *k)
3670 {
3671   PetscErrorCode ierr;
3672 
3673   PetscFunctionBeginHot;
3674   ierr = PetscSectionGetFieldComponents(section, field, Nc);CHKERRQ(ierr);
3675   if (vertexchart) {            /* If we only have a vertex chart, we must have degree k=1 */
3676     *k = 1;
3677   } else {                      /* Assume the full interpolated mesh is in the chart; lines in particular */
3678     /* An order k SEM disc has k-1 dofs on an edge */
3679     ierr = PetscSectionGetFieldDof(section, line, field, k);CHKERRQ(ierr);
3680     *k = *k / *Nc + 1;
3681   }
3682   PetscFunctionReturn(0);
3683 }
3684 
3685 /*@
3686 
3687   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
3688   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
3689   section provided (or default section of the DM).
3690 
3691   Input Parameters:
3692 + dm      - The DM
3693 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or PETSC_DETERMINE
3694 - section - The PetscSection to reorder, or NULL for the default section
3695 
3696   Note: The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
3697   degree of the basis.
3698 
3699   Example:
3700   A typical interpolated single-quad mesh might order points as
3701 .vb
3702   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
3703 
3704   v4 -- e6 -- v3
3705   |           |
3706   e7    c0    e8
3707   |           |
3708   v1 -- e5 -- v2
3709 .ve
3710 
3711   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
3712   dofs in the order of points, e.g.,
3713 .vb
3714     c0 -> [0,1,2,3]
3715     v1 -> [4]
3716     ...
3717     e5 -> [8, 9]
3718 .ve
3719 
3720   which corresponds to the dofs
3721 .vb
3722     6   10  11  7
3723     13  2   3   15
3724     12  0   1   14
3725     4   8   9   5
3726 .ve
3727 
3728   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
3729 .vb
3730   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
3731 .ve
3732 
3733   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
3734 .vb
3735    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
3736 .ve
3737 
3738   Level: developer
3739 
3740 .seealso: DMGetSection(), PetscSectionSetClosurePermutation()
3741 @*/
3742 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
3743 {
3744   DMLabel        label;
3745   PetscInt      *perm;
3746   PetscInt       dim, depth, eStart, k, Nf, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
3747   PetscBool      vertexchart;
3748   PetscErrorCode ierr;
3749 
3750   PetscFunctionBegin;
3751   if (point < 0) {ierr = DMPlexGetDepthStratum(dm, 1, &point, NULL);CHKERRQ(ierr);}
3752   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3753   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
3754   ierr = DMLabelGetValue(label, point, &depth);CHKERRQ(ierr);
3755   if (depth == 1) {eStart = point;}
3756   else if  (depth == dim) {
3757     const PetscInt *cone;
3758 
3759     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3760     if (dim == 2) eStart = cone[0];
3761     else if (dim == 3) {
3762       const PetscInt *cone2;
3763       ierr = DMPlexGetCone(dm, cone[0], &cone2);CHKERRQ(ierr);
3764       eStart = cone2[0];
3765     } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3766   } else SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %D of depth %D cannot be used to bootstrap spectral ordering for dim %D", point, depth, dim);
3767   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
3768   {                             /* Determine whether the chart covers all points or just vertices. */
3769     PetscInt pStart,pEnd,cStart,cEnd;
3770     ierr = DMPlexGetDepthStratum(dm,0,&pStart,&pEnd);CHKERRQ(ierr);
3771     ierr = PetscSectionGetChart(section,&cStart,&cEnd);CHKERRQ(ierr);
3772     if (pStart == cStart && pEnd == cEnd) vertexchart = PETSC_TRUE; /* Just vertices */
3773     else vertexchart = PETSC_FALSE;                                 /* Assume all interpolated points are in chart */
3774   }
3775   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
3776   if (dim < 1) PetscFunctionReturn(0);
3777   for (f = 0; f < Nf; ++f) {
3778     ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3779     size += PetscPowInt(k+1, dim)*Nc;
3780   }
3781   ierr = PetscMalloc1(size, &perm);CHKERRQ(ierr);
3782   for (f = 0; f < Nf; ++f) {
3783     switch (dim) {
3784     case 1:
3785       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3786       /*
3787         Original ordering is [ edge of length k-1; vtx0; vtx1 ]
3788         We want              [ vtx0; edge of length k-1; vtx1 ]
3789       */
3790       for (c=0; c<Nc; c++,offset++) perm[offset] = (k-1)*Nc + c + foffset;
3791       for (i=0; i<k-1; i++) for (c=0; c<Nc; c++,offset++) perm[offset] = i*Nc + c + foffset;
3792       for (c=0; c<Nc; c++,offset++) perm[offset] = k*Nc + c + foffset;
3793       foffset = offset;
3794       break;
3795     case 2:
3796       /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
3797       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3798       /* The SEM order is
3799 
3800          v_lb, {e_b}, v_rb,
3801          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
3802          v_lt, reverse {e_t}, v_rt
3803       */
3804       {
3805         const PetscInt of   = 0;
3806         const PetscInt oeb  = of   + PetscSqr(k-1);
3807         const PetscInt oer  = oeb  + (k-1);
3808         const PetscInt oet  = oer  + (k-1);
3809         const PetscInt oel  = oet  + (k-1);
3810         const PetscInt ovlb = oel  + (k-1);
3811         const PetscInt ovrb = ovlb + 1;
3812         const PetscInt ovrt = ovrb + 1;
3813         const PetscInt ovlt = ovrt + 1;
3814         PetscInt       o;
3815 
3816         /* bottom */
3817         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb*Nc + c + foffset;
3818         for (o = oeb; o < oer; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3819         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb*Nc + c + foffset;
3820         /* middle */
3821         for (i = 0; i < k-1; ++i) {
3822           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel+(k-2)-i)*Nc + c + foffset;
3823           for (o = of+(k-1)*i; o < of+(k-1)*(i+1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3824           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer+i)*Nc + c + foffset;
3825         }
3826         /* top */
3827         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt*Nc + c + foffset;
3828         for (o = oel-1; o >= oet; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3829         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt*Nc + c + foffset;
3830         foffset = offset;
3831       }
3832       break;
3833     case 3:
3834       /* The original hex closure is
3835 
3836          {c,
3837           f_b, f_t, f_f, f_b, f_r, f_l,
3838           e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
3839           v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
3840       */
3841       ierr = PetscSectionFieldGetTensorDegree_Private(section,f,eStart,vertexchart,&Nc,&k);CHKERRQ(ierr);
3842       /* The SEM order is
3843          Bottom Slice
3844          v_blf, {e^{(k-1)-n}_bf}, v_brf,
3845          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
3846          v_blb, {e_bb}, v_brb,
3847 
3848          Middle Slice (j)
3849          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
3850          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
3851          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
3852 
3853          Top Slice
3854          v_tlf, {e_tf}, v_trf,
3855          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
3856          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
3857       */
3858       {
3859         const PetscInt oc    = 0;
3860         const PetscInt ofb   = oc    + PetscSqr(k-1)*(k-1);
3861         const PetscInt oft   = ofb   + PetscSqr(k-1);
3862         const PetscInt off   = oft   + PetscSqr(k-1);
3863         const PetscInt ofk   = off   + PetscSqr(k-1);
3864         const PetscInt ofr   = ofk   + PetscSqr(k-1);
3865         const PetscInt ofl   = ofr   + PetscSqr(k-1);
3866         const PetscInt oebl  = ofl   + PetscSqr(k-1);
3867         const PetscInt oebb  = oebl  + (k-1);
3868         const PetscInt oebr  = oebb  + (k-1);
3869         const PetscInt oebf  = oebr  + (k-1);
3870         const PetscInt oetf  = oebf  + (k-1);
3871         const PetscInt oetr  = oetf  + (k-1);
3872         const PetscInt oetb  = oetr  + (k-1);
3873         const PetscInt oetl  = oetb  + (k-1);
3874         const PetscInt oerf  = oetl  + (k-1);
3875         const PetscInt oelf  = oerf  + (k-1);
3876         const PetscInt oelb  = oelf  + (k-1);
3877         const PetscInt oerb  = oelb  + (k-1);
3878         const PetscInt ovblf = oerb  + (k-1);
3879         const PetscInt ovblb = ovblf + 1;
3880         const PetscInt ovbrb = ovblb + 1;
3881         const PetscInt ovbrf = ovbrb + 1;
3882         const PetscInt ovtlf = ovbrf + 1;
3883         const PetscInt ovtrf = ovtlf + 1;
3884         const PetscInt ovtrb = ovtrf + 1;
3885         const PetscInt ovtlb = ovtrb + 1;
3886         PetscInt       o, n;
3887 
3888         /* Bottom Slice */
3889         /*   bottom */
3890         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf*Nc + c + foffset;
3891         for (o = oetf-1; o >= oebf; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3892         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf*Nc + c + foffset;
3893         /*   middle */
3894         for (i = 0; i < k-1; ++i) {
3895           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl+i)*Nc + c + foffset;
3896           for (n = 0; n < k-1; ++n) {o = ofb+n*(k-1)+i; for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;}
3897           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr+(k-2)-i)*Nc + c + foffset;
3898         }
3899         /*   top */
3900         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb*Nc + c + foffset;
3901         for (o = oebb; o < oebr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3902         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb*Nc + c + foffset;
3903 
3904         /* Middle Slice */
3905         for (j = 0; j < k-1; ++j) {
3906           /*   bottom */
3907           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf+(k-2)-j)*Nc + c + foffset;
3908           for (o = off+j*(k-1); o < off+(j+1)*(k-1); ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3909           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf+j)*Nc + c + foffset;
3910           /*   middle */
3911           for (i = 0; i < k-1; ++i) {
3912             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl+i*(k-1)+j)*Nc + c + foffset;
3913             for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc+(j*(k-1)+i)*(k-1)+n)*Nc + c + foffset;
3914             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr+j*(k-1)+i)*Nc + c + foffset;
3915           }
3916           /*   top */
3917           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb+j)*Nc + c + foffset;
3918           for (o = ofk+j*(k-1)+(k-2); o >= ofk+j*(k-1); --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3919           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb+(k-2)-j)*Nc + c + foffset;
3920         }
3921 
3922         /* Top Slice */
3923         /*   bottom */
3924         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf*Nc + c + foffset;
3925         for (o = oetf; o < oetr; ++o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3926         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf*Nc + c + foffset;
3927         /*   middle */
3928         for (i = 0; i < k-1; ++i) {
3929           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl+(k-2)-i)*Nc + c + foffset;
3930           for (n = 0; n < k-1; ++n) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft+i*(k-1)+n)*Nc + c + foffset;
3931           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr+i)*Nc + c + foffset;
3932         }
3933         /*   top */
3934         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb*Nc + c + foffset;
3935         for (o = oetl-1; o >= oetb; --o) for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o*Nc + c + foffset;
3936         for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb*Nc + c + foffset;
3937 
3938         foffset = offset;
3939       }
3940       break;
3941     default: SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %D", dim);
3942     }
3943   }
3944   if (offset != size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Number of permutation entries %D != %D", offset, size);
3945   /* Check permutation */
3946   {
3947     PetscInt *check;
3948 
3949     ierr = PetscMalloc1(size, &check);CHKERRQ(ierr);
3950     for (i = 0; i < size; ++i) {check[i] = -1; if (perm[i] < 0 || perm[i] >= size) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid permutation index p[%D] = %D", i, perm[i]);}
3951     for (i = 0; i < size; ++i) check[perm[i]] = i;
3952     for (i = 0; i < size; ++i) {if (check[i] < 0) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Missing permutation index %D", i);}
3953     ierr = PetscFree(check);CHKERRQ(ierr);
3954   }
3955   ierr = PetscSectionSetClosurePermutation_Internal(section, (PetscObject) dm, size, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
3956   PetscFunctionReturn(0);
3957 }
3958 
3959 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
3960 {
3961   PetscDS        prob;
3962   PetscInt       depth, Nf, h;
3963   DMLabel        label;
3964   PetscErrorCode ierr;
3965 
3966   PetscFunctionBeginHot;
3967   ierr = DMGetDS(dm, &prob);CHKERRQ(ierr);
3968   Nf      = prob->Nf;
3969   label   = dm->depthLabel;
3970   *dspace = NULL;
3971   if (field < Nf) {
3972     PetscObject disc = prob->disc[field];
3973 
3974     if (disc->classid == PETSCFE_CLASSID) {
3975       PetscDualSpace dsp;
3976 
3977       ierr = PetscFEGetDualSpace((PetscFE)disc,&dsp);CHKERRQ(ierr);
3978       ierr = DMLabelGetNumValues(label,&depth);CHKERRQ(ierr);
3979       ierr = DMLabelGetValue(label,point,&h);CHKERRQ(ierr);
3980       h    = depth - 1 - h;
3981       if (h) {
3982         ierr = PetscDualSpaceGetHeightSubspace(dsp,h,dspace);CHKERRQ(ierr);
3983       } else {
3984         *dspace = dsp;
3985       }
3986     }
3987   }
3988   PetscFunctionReturn(0);
3989 }
3990 
3991 
3992 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
3993 {
3994   PetscScalar    *array, *vArray;
3995   const PetscInt *cone, *coneO;
3996   PetscInt        pStart, pEnd, p, numPoints, size = 0, offset = 0;
3997   PetscErrorCode  ierr;
3998 
3999   PetscFunctionBeginHot;
4000   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4001   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4002   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4003   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4004   if (!values || !*values) {
4005     if ((point >= pStart) && (point < pEnd)) {
4006       PetscInt dof;
4007 
4008       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4009       size += dof;
4010     }
4011     for (p = 0; p < numPoints; ++p) {
4012       const PetscInt cp = cone[p];
4013       PetscInt       dof;
4014 
4015       if ((cp < pStart) || (cp >= pEnd)) continue;
4016       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4017       size += dof;
4018     }
4019     if (!values) {
4020       if (csize) *csize = size;
4021       PetscFunctionReturn(0);
4022     }
4023     ierr = DMGetWorkArray(dm, size, MPIU_SCALAR, &array);CHKERRQ(ierr);
4024   } else {
4025     array = *values;
4026   }
4027   size = 0;
4028   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
4029   if ((point >= pStart) && (point < pEnd)) {
4030     PetscInt     dof, off, d;
4031     PetscScalar *varr;
4032 
4033     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4034     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4035     varr = &vArray[off];
4036     for (d = 0; d < dof; ++d, ++offset) {
4037       array[offset] = varr[d];
4038     }
4039     size += dof;
4040   }
4041   for (p = 0; p < numPoints; ++p) {
4042     const PetscInt cp = cone[p];
4043     PetscInt       o  = coneO[p];
4044     PetscInt       dof, off, d;
4045     PetscScalar   *varr;
4046 
4047     if ((cp < pStart) || (cp >= pEnd)) continue;
4048     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4049     ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
4050     varr = &vArray[off];
4051     if (o >= 0) {
4052       for (d = 0; d < dof; ++d, ++offset) {
4053         array[offset] = varr[d];
4054       }
4055     } else {
4056       for (d = dof-1; d >= 0; --d, ++offset) {
4057         array[offset] = varr[d];
4058       }
4059     }
4060     size += dof;
4061   }
4062   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
4063   if (!*values) {
4064     if (csize) *csize = size;
4065     *values = array;
4066   } else {
4067     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4068     *csize = size;
4069   }
4070   PetscFunctionReturn(0);
4071 }
4072 
4073 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4074 {
4075   const PetscInt *cla;
4076   PetscInt       np, *pts = NULL;
4077   PetscErrorCode ierr;
4078 
4079   PetscFunctionBeginHot;
4080   ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, clSec, clPoints);CHKERRQ(ierr);
4081   if (!*clPoints) {
4082     PetscInt pStart, pEnd, p, q;
4083 
4084     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4085     ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &np, &pts);CHKERRQ(ierr);
4086     /* Compress out points not in the section */
4087     for (p = 0, q = 0; p < np; p++) {
4088       PetscInt r = pts[2*p];
4089       if ((r >= pStart) && (r < pEnd)) {
4090         pts[q*2]   = r;
4091         pts[q*2+1] = pts[2*p+1];
4092         ++q;
4093       }
4094     }
4095     np = q;
4096     cla = NULL;
4097   } else {
4098     PetscInt dof, off;
4099 
4100     ierr = PetscSectionGetDof(*clSec, point, &dof);CHKERRQ(ierr);
4101     ierr = PetscSectionGetOffset(*clSec, point, &off);CHKERRQ(ierr);
4102     ierr = ISGetIndices(*clPoints, &cla);CHKERRQ(ierr);
4103     np   = dof/2;
4104     pts  = (PetscInt *) &cla[off];
4105   }
4106   *numPoints = np;
4107   *points    = pts;
4108   *clp       = cla;
4109 
4110   PetscFunctionReturn(0);
4111 }
4112 
4113 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
4114 {
4115   PetscErrorCode ierr;
4116 
4117   PetscFunctionBeginHot;
4118   if (!*clPoints) {
4119     ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points);CHKERRQ(ierr);
4120   } else {
4121     ierr = ISRestoreIndices(*clPoints, clp);CHKERRQ(ierr);
4122   }
4123   *numPoints = 0;
4124   *points    = NULL;
4125   *clSec     = NULL;
4126   *clPoints  = NULL;
4127   *clp       = NULL;
4128   PetscFunctionReturn(0);
4129 }
4130 
4131 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4132 {
4133   PetscInt          offset = 0, p;
4134   const PetscInt    **perms = NULL;
4135   const PetscScalar **flips = NULL;
4136   PetscErrorCode    ierr;
4137 
4138   PetscFunctionBeginHot;
4139   *size = 0;
4140   ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4141   for (p = 0; p < numPoints; p++) {
4142     const PetscInt    point = points[2*p];
4143     const PetscInt    *perm = perms ? perms[p] : NULL;
4144     const PetscScalar *flip = flips ? flips[p] : NULL;
4145     PetscInt          dof, off, d;
4146     const PetscScalar *varr;
4147 
4148     ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4149     ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4150     varr = &vArray[off];
4151     if (clperm) {
4152       if (perm) {
4153         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]]  = varr[d];
4154       } else {
4155         for (d = 0; d < dof; d++) array[clperm[offset +      d ]]  = varr[d];
4156       }
4157       if (flip) {
4158         for (d = 0; d < dof; d++) array[clperm[offset +      d ]] *= flip[d];
4159       }
4160     } else {
4161       if (perm) {
4162         for (d = 0; d < dof; d++) array[offset + perm[d]]  = varr[d];
4163       } else {
4164         for (d = 0; d < dof; d++) array[offset +      d ]  = varr[d];
4165       }
4166       if (flip) {
4167         for (d = 0; d < dof; d++) array[offset +      d ] *= flip[d];
4168       }
4169     }
4170     offset += dof;
4171   }
4172   ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4173   *size = offset;
4174   PetscFunctionReturn(0);
4175 }
4176 
4177 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
4178 {
4179   PetscInt          offset = 0, f;
4180   PetscErrorCode    ierr;
4181 
4182   PetscFunctionBeginHot;
4183   *size = 0;
4184   for (f = 0; f < numFields; ++f) {
4185     PetscInt          p;
4186     const PetscInt    **perms = NULL;
4187     const PetscScalar **flips = NULL;
4188 
4189     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4190     for (p = 0; p < numPoints; p++) {
4191       const PetscInt    point = points[2*p];
4192       PetscInt          fdof, foff, b;
4193       const PetscScalar *varr;
4194       const PetscInt    *perm = perms ? perms[p] : NULL;
4195       const PetscScalar *flip = flips ? flips[p] : NULL;
4196 
4197       ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4198       ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4199       varr = &vArray[foff];
4200       if (clperm) {
4201         if (perm) {for (b = 0; b < fdof; b++) {array[clperm[offset + perm[b]]]  = varr[b];}}
4202         else      {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]]  = varr[b];}}
4203         if (flip) {for (b = 0; b < fdof; b++) {array[clperm[offset +      b ]] *= flip[b];}}
4204       } else {
4205         if (perm) {for (b = 0; b < fdof; b++) {array[offset + perm[b]]  = varr[b];}}
4206         else      {for (b = 0; b < fdof; b++) {array[offset +      b ]  = varr[b];}}
4207         if (flip) {for (b = 0; b < fdof; b++) {array[offset +      b ] *= flip[b];}}
4208       }
4209       offset += fdof;
4210     }
4211     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4212   }
4213   *size = offset;
4214   PetscFunctionReturn(0);
4215 }
4216 
4217 /*@C
4218   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
4219 
4220   Not collective
4221 
4222   Input Parameters:
4223 + dm - The DM
4224 . section - The section describing the layout in v, or NULL to use the default section
4225 . v - The local vector
4226 . point - The point in the DM
4227 . csize - The size of the input values array, or NULL
4228 - values - An array to use for the values, or NULL to have it allocated automatically
4229 
4230   Output Parameters:
4231 + csize - The number of values in the closure
4232 - values - The array of values. If the user provided NULL, it is a borrowed array and should not be freed
4233 
4234 $ Note that DMPlexVecGetClosure/DMPlexVecRestoreClosure only allocates the values array if it set to NULL in the
4235 $ calling function. This is because DMPlexVecGetClosure() is typically called in the inner loop of a Vec or Mat
4236 $ assembly function, and a user may already have allocated storage for this operation.
4237 $
4238 $ A typical use could be
4239 $
4240 $  values = NULL;
4241 $  ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4242 $  for (cl = 0; cl < clSize; ++cl) {
4243 $    <Compute on closure>
4244 $  }
4245 $  ierr = DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4246 $
4247 $ or
4248 $
4249 $  PetscMalloc1(clMaxSize, &values);
4250 $  for (p = pStart; p < pEnd; ++p) {
4251 $    clSize = clMaxSize;
4252 $    ierr = DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values);CHKERRQ(ierr);
4253 $    for (cl = 0; cl < clSize; ++cl) {
4254 $      <Compute on closure>
4255 $    }
4256 $  }
4257 $  PetscFree(values);
4258 
4259   Fortran Notes:
4260   Since it returns an array, this routine is only available in Fortran 90, and you must
4261   include petsc.h90 in your code.
4262 
4263   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4264 
4265   Level: intermediate
4266 
4267 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4268 @*/
4269 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4270 {
4271   PetscSection       clSection;
4272   IS                 clPoints;
4273   PetscScalar       *array;
4274   const PetscScalar *vArray;
4275   PetscInt          *points = NULL;
4276   const PetscInt    *clp, *perm;
4277   PetscInt           depth, numFields, numPoints, size;
4278   PetscErrorCode     ierr;
4279 
4280   PetscFunctionBeginHot;
4281   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4282   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4283   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4284   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4285   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4286   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4287   if (depth == 1 && numFields < 2) {
4288     ierr = DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values);CHKERRQ(ierr);
4289     PetscFunctionReturn(0);
4290   }
4291   /* Get points */
4292   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4293   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &perm);CHKERRQ(ierr);
4294   /* Get array */
4295   if (!values || !*values) {
4296     PetscInt asize = 0, dof, p;
4297 
4298     for (p = 0; p < numPoints*2; p += 2) {
4299       ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
4300       asize += dof;
4301     }
4302     if (!values) {
4303       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4304       if (csize) *csize = asize;
4305       PetscFunctionReturn(0);
4306     }
4307     ierr = DMGetWorkArray(dm, asize, MPIU_SCALAR, &array);CHKERRQ(ierr);
4308   } else {
4309     array = *values;
4310   }
4311   ierr = VecGetArrayRead(v, &vArray);CHKERRQ(ierr);
4312   /* Get values */
4313   if (numFields > 0) {ierr = DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, array);CHKERRQ(ierr);}
4314   else               {ierr = DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, array);CHKERRQ(ierr);}
4315   /* Cleanup points */
4316   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4317   /* Cleanup array */
4318   ierr = VecRestoreArrayRead(v, &vArray);CHKERRQ(ierr);
4319   if (!*values) {
4320     if (csize) *csize = size;
4321     *values = array;
4322   } else {
4323     if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %D < actual size %D", *csize, size);
4324     *csize = size;
4325   }
4326   PetscFunctionReturn(0);
4327 }
4328 
4329 /*@C
4330   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
4331 
4332   Not collective
4333 
4334   Input Parameters:
4335 + dm - The DM
4336 . section - The section describing the layout in v, or NULL to use the default section
4337 . v - The local vector
4338 . point - The point in the DM
4339 . csize - The number of values in the closure, or NULL
4340 - values - The array of values, which is a borrowed array and should not be freed
4341 
4342   Note that the array values are discarded and not copied back into v. In order to copy values back to v, use DMPlexVecSetClosure()
4343 
4344   Fortran Notes:
4345   Since it returns an array, this routine is only available in Fortran 90, and you must
4346   include petsc.h90 in your code.
4347 
4348   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
4349 
4350   Level: intermediate
4351 
4352 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
4353 @*/
4354 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
4355 {
4356   PetscInt       size = 0;
4357   PetscErrorCode ierr;
4358 
4359   PetscFunctionBegin;
4360   /* Should work without recalculating size */
4361   ierr = DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void*) values);CHKERRQ(ierr);
4362   *values = NULL;
4363   PetscFunctionReturn(0);
4364 }
4365 
4366 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
4367 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
4368 
4369 PETSC_STATIC_INLINE PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4370 {
4371   PetscInt        cdof;   /* The number of constraints on this point */
4372   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4373   PetscScalar    *a;
4374   PetscInt        off, cind = 0, k;
4375   PetscErrorCode  ierr;
4376 
4377   PetscFunctionBegin;
4378   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4379   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4380   a    = &array[off];
4381   if (!cdof || setBC) {
4382     if (clperm) {
4383       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));}}
4384       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));}}
4385     } else {
4386       if (perm) {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));}}
4387       else      {for (k = 0; k < dof; ++k) {fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));}}
4388     }
4389   } else {
4390     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4391     if (clperm) {
4392       if (perm) {for (k = 0; k < dof; ++k) {
4393           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4394           fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4395         }
4396       } else {
4397         for (k = 0; k < dof; ++k) {
4398           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4399           fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4400         }
4401       }
4402     } else {
4403       if (perm) {
4404         for (k = 0; k < dof; ++k) {
4405           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4406           fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4407         }
4408       } else {
4409         for (k = 0; k < dof; ++k) {
4410           if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4411           fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4412         }
4413       }
4414     }
4415   }
4416   PetscFunctionReturn(0);
4417 }
4418 
4419 PETSC_STATIC_INLINE PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
4420 {
4421   PetscInt        cdof;   /* The number of constraints on this point */
4422   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4423   PetscScalar    *a;
4424   PetscInt        off, cind = 0, k;
4425   PetscErrorCode  ierr;
4426 
4427   PetscFunctionBegin;
4428   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4429   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
4430   a    = &array[off];
4431   if (cdof) {
4432     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4433     if (clperm) {
4434       if (perm) {
4435         for (k = 0; k < dof; ++k) {
4436           if ((cind < cdof) && (k == cdofs[cind])) {
4437             fuse(&a[k], values[clperm[offset+perm[k]]] * (flip ? flip[perm[k]] : 1.));
4438             cind++;
4439           }
4440         }
4441       } else {
4442         for (k = 0; k < dof; ++k) {
4443           if ((cind < cdof) && (k == cdofs[cind])) {
4444             fuse(&a[k], values[clperm[offset+     k ]] * (flip ? flip[     k ] : 1.));
4445             cind++;
4446           }
4447         }
4448       }
4449     } else {
4450       if (perm) {
4451         for (k = 0; k < dof; ++k) {
4452           if ((cind < cdof) && (k == cdofs[cind])) {
4453             fuse(&a[k], values[offset+perm[k]] * (flip ? flip[perm[k]] : 1.));
4454             cind++;
4455           }
4456         }
4457       } else {
4458         for (k = 0; k < dof; ++k) {
4459           if ((cind < cdof) && (k == cdofs[cind])) {
4460             fuse(&a[k], values[offset+     k ] * (flip ? flip[     k ] : 1.));
4461             cind++;
4462           }
4463         }
4464       }
4465     }
4466   }
4467   PetscFunctionReturn(0);
4468 }
4469 
4470 PETSC_STATIC_INLINE PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4471 {
4472   PetscScalar    *a;
4473   PetscInt        fdof, foff, fcdof, foffset = *offset;
4474   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4475   PetscInt        cind = 0, b;
4476   PetscErrorCode  ierr;
4477 
4478   PetscFunctionBegin;
4479   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4480   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4481   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4482   a    = &array[foff];
4483   if (!fcdof || setBC) {
4484     if (clperm) {
4485       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}}
4486       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}}
4487     } else {
4488       if (perm) {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}}
4489       else      {for (b = 0; b < fdof; b++) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}}
4490     }
4491   } else {
4492     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4493     if (clperm) {
4494       if (perm) {
4495         for (b = 0; b < fdof; b++) {
4496           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4497           fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4498         }
4499       } else {
4500         for (b = 0; b < fdof; b++) {
4501           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4502           fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4503         }
4504       }
4505     } else {
4506       if (perm) {
4507         for (b = 0; b < fdof; b++) {
4508           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4509           fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4510         }
4511       } else {
4512         for (b = 0; b < fdof; b++) {
4513           if ((cind < fcdof) && (b == fcdofs[cind])) {++cind; continue;}
4514           fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4515         }
4516       }
4517     }
4518   }
4519   *offset += fdof;
4520   PetscFunctionReturn(0);
4521 }
4522 
4523 PETSC_STATIC_INLINE PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar*, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
4524 {
4525   PetscScalar    *a;
4526   PetscInt        fdof, foff, fcdof, foffset = *offset;
4527   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
4528   PetscInt        cind = 0, ncind = 0, b;
4529   PetscBool       ncSet, fcSet;
4530   PetscErrorCode  ierr;
4531 
4532   PetscFunctionBegin;
4533   ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
4534   ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
4535   ierr = PetscSectionGetFieldOffset(section, point, f, &foff);CHKERRQ(ierr);
4536   a    = &array[foff];
4537   if (fcdof) {
4538     /* We just override fcdof and fcdofs with Ncc and comps */
4539     ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
4540     if (clperm) {
4541       if (perm) {
4542         if (comps) {
4543           for (b = 0; b < fdof; b++) {
4544             ncSet = fcSet = PETSC_FALSE;
4545             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4546             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4547             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));}
4548           }
4549         } else {
4550           for (b = 0; b < fdof; b++) {
4551             if ((cind < fcdof) && (b == fcdofs[cind])) {
4552               fuse(&a[b], values[clperm[foffset+perm[b]]] * (flip ? flip[perm[b]] : 1.));
4553               ++cind;
4554             }
4555           }
4556         }
4557       } else {
4558         if (comps) {
4559           for (b = 0; b < fdof; b++) {
4560             ncSet = fcSet = PETSC_FALSE;
4561             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4562             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4563             if (ncSet && fcSet) {fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));}
4564           }
4565         } else {
4566           for (b = 0; b < fdof; b++) {
4567             if ((cind < fcdof) && (b == fcdofs[cind])) {
4568               fuse(&a[b], values[clperm[foffset+     b ]] * (flip ? flip[     b ] : 1.));
4569               ++cind;
4570             }
4571           }
4572         }
4573       }
4574     } else {
4575       if (perm) {
4576         if (comps) {
4577           for (b = 0; b < fdof; b++) {
4578             ncSet = fcSet = PETSC_FALSE;
4579             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4580             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4581             if (ncSet && fcSet) {fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));}
4582           }
4583         } else {
4584           for (b = 0; b < fdof; b++) {
4585             if ((cind < fcdof) && (b == fcdofs[cind])) {
4586               fuse(&a[b], values[foffset+perm[b]] * (flip ? flip[perm[b]] : 1.));
4587               ++cind;
4588             }
4589           }
4590         }
4591       } else {
4592         if (comps) {
4593           for (b = 0; b < fdof; b++) {
4594             ncSet = fcSet = PETSC_FALSE;
4595             if ((ncind < Ncc)  && (b == comps[ncind])) {++ncind; ncSet = PETSC_TRUE;}
4596             if ((cind < fcdof) && (b == fcdofs[cind])) {++cind;  fcSet = PETSC_TRUE;}
4597             if (ncSet && fcSet) {fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));}
4598           }
4599         } else {
4600           for (b = 0; b < fdof; b++) {
4601             if ((cind < fcdof) && (b == fcdofs[cind])) {
4602               fuse(&a[b], values[foffset+     b ] * (flip ? flip[     b ] : 1.));
4603               ++cind;
4604             }
4605           }
4606         }
4607       }
4608     }
4609   }
4610   *offset += fdof;
4611   PetscFunctionReturn(0);
4612 }
4613 
4614 PETSC_STATIC_INLINE PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4615 {
4616   PetscScalar    *array;
4617   const PetscInt *cone, *coneO;
4618   PetscInt        pStart, pEnd, p, numPoints, off, dof;
4619   PetscErrorCode  ierr;
4620 
4621   PetscFunctionBeginHot;
4622   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
4623   ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
4624   ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
4625   ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
4626   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4627   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
4628     const PetscInt cp = !p ? point : cone[p-1];
4629     const PetscInt o  = !p ? 0     : coneO[p-1];
4630 
4631     if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
4632     ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
4633     /* ADD_VALUES */
4634     {
4635       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4636       PetscScalar    *a;
4637       PetscInt        cdof, coff, cind = 0, k;
4638 
4639       ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
4640       ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
4641       a    = &array[coff];
4642       if (!cdof) {
4643         if (o >= 0) {
4644           for (k = 0; k < dof; ++k) {
4645             a[k] += values[off+k];
4646           }
4647         } else {
4648           for (k = 0; k < dof; ++k) {
4649             a[k] += values[off+dof-k-1];
4650           }
4651         }
4652       } else {
4653         ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
4654         if (o >= 0) {
4655           for (k = 0; k < dof; ++k) {
4656             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4657             a[k] += values[off+k];
4658           }
4659         } else {
4660           for (k = 0; k < dof; ++k) {
4661             if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
4662             a[k] += values[off+dof-k-1];
4663           }
4664         }
4665       }
4666     }
4667   }
4668   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4669   PetscFunctionReturn(0);
4670 }
4671 
4672 /*@C
4673   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
4674 
4675   Not collective
4676 
4677   Input Parameters:
4678 + dm - The DM
4679 . section - The section describing the layout in v, or NULL to use the default section
4680 . v - The local vector
4681 . point - The point in the DM
4682 . values - The array of values
4683 - mode - The insert mode. One of INSERT_ALL_VALUES, ADD_ALL_VALUES, INSERT_VALUES, ADD_VALUES, INSERT_BC_VALUES, and ADD_BC_VALUES,
4684          where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions.
4685 
4686   Fortran Notes:
4687   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
4688 
4689   Level: intermediate
4690 
4691 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
4692 @*/
4693 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
4694 {
4695   PetscSection    clSection;
4696   IS              clPoints;
4697   PetscScalar    *array;
4698   PetscInt       *points = NULL;
4699   const PetscInt *clp, *clperm;
4700   PetscInt        depth, numFields, numPoints, p;
4701   PetscErrorCode  ierr;
4702 
4703   PetscFunctionBeginHot;
4704   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4705   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4706   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4707   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4708   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4709   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4710   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
4711     ierr = DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode);CHKERRQ(ierr);
4712     PetscFunctionReturn(0);
4713   }
4714   /* Get points */
4715   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4716   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4717   /* Get array */
4718   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4719   /* Get values */
4720   if (numFields > 0) {
4721     PetscInt offset = 0, f;
4722     for (f = 0; f < numFields; ++f) {
4723       const PetscInt    **perms = NULL;
4724       const PetscScalar **flips = NULL;
4725 
4726       ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4727       switch (mode) {
4728       case INSERT_VALUES:
4729         for (p = 0; p < numPoints; p++) {
4730           const PetscInt    point = points[2*p];
4731           const PetscInt    *perm = perms ? perms[p] : NULL;
4732           const PetscScalar *flip = flips ? flips[p] : NULL;
4733           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4734         } break;
4735       case INSERT_ALL_VALUES:
4736         for (p = 0; p < numPoints; p++) {
4737           const PetscInt    point = points[2*p];
4738           const PetscInt    *perm = perms ? perms[p] : NULL;
4739           const PetscScalar *flip = flips ? flips[p] : NULL;
4740           updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4741         } break;
4742       case INSERT_BC_VALUES:
4743         for (p = 0; p < numPoints; p++) {
4744           const PetscInt    point = points[2*p];
4745           const PetscInt    *perm = perms ? perms[p] : NULL;
4746           const PetscScalar *flip = flips ? flips[p] : NULL;
4747           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array);
4748         } break;
4749       case ADD_VALUES:
4750         for (p = 0; p < numPoints; p++) {
4751           const PetscInt    point = points[2*p];
4752           const PetscInt    *perm = perms ? perms[p] : NULL;
4753           const PetscScalar *flip = flips ? flips[p] : NULL;
4754           updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4755         } break;
4756       case ADD_ALL_VALUES:
4757         for (p = 0; p < numPoints; p++) {
4758           const PetscInt    point = points[2*p];
4759           const PetscInt    *perm = perms ? perms[p] : NULL;
4760           const PetscScalar *flip = flips ? flips[p] : NULL;
4761           updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4762         } break;
4763       case ADD_BC_VALUES:
4764         for (p = 0; p < numPoints; p++) {
4765           const PetscInt    point = points[2*p];
4766           const PetscInt    *perm = perms ? perms[p] : NULL;
4767           const PetscScalar *flip = flips ? flips[p] : NULL;
4768           updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array);
4769         } break;
4770       default:
4771         SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4772       }
4773       ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4774     }
4775   } else {
4776     PetscInt dof, off;
4777     const PetscInt    **perms = NULL;
4778     const PetscScalar **flips = NULL;
4779 
4780     ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4781     switch (mode) {
4782     case INSERT_VALUES:
4783       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4784         const PetscInt    point = points[2*p];
4785         const PetscInt    *perm = perms ? perms[p] : NULL;
4786         const PetscScalar *flip = flips ? flips[p] : NULL;
4787         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4788         updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array);
4789       } break;
4790     case INSERT_ALL_VALUES:
4791       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4792         const PetscInt    point = points[2*p];
4793         const PetscInt    *perm = perms ? perms[p] : NULL;
4794         const PetscScalar *flip = flips ? flips[p] : NULL;
4795         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4796         updatePoint_private(section, point, dof, insert, PETSC_TRUE,  perm, flip, clperm, values, off, array);
4797       } break;
4798     case INSERT_BC_VALUES:
4799       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4800         const PetscInt    point = points[2*p];
4801         const PetscInt    *perm = perms ? perms[p] : NULL;
4802         const PetscScalar *flip = flips ? flips[p] : NULL;
4803         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4804         updatePointBC_private(section, point, dof, insert,  perm, flip, clperm, values, off, array);
4805       } break;
4806     case ADD_VALUES:
4807       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4808         const PetscInt    point = points[2*p];
4809         const PetscInt    *perm = perms ? perms[p] : NULL;
4810         const PetscScalar *flip = flips ? flips[p] : NULL;
4811         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4812         updatePoint_private(section, point, dof, add,    PETSC_FALSE, perm, flip, clperm, values, off, array);
4813       } break;
4814     case ADD_ALL_VALUES:
4815       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4816         const PetscInt    point = points[2*p];
4817         const PetscInt    *perm = perms ? perms[p] : NULL;
4818         const PetscScalar *flip = flips ? flips[p] : NULL;
4819         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4820         updatePoint_private(section, point, dof, add,    PETSC_TRUE,  perm, flip, clperm, values, off, array);
4821       } break;
4822     case ADD_BC_VALUES:
4823       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
4824         const PetscInt    point = points[2*p];
4825         const PetscInt    *perm = perms ? perms[p] : NULL;
4826         const PetscScalar *flip = flips ? flips[p] : NULL;
4827         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4828         updatePointBC_private(section, point, dof, add,  perm, flip, clperm, values, off, array);
4829       } break;
4830     default:
4831       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4832     }
4833     ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4834   }
4835   /* Cleanup points */
4836   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4837   /* Cleanup array */
4838   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4839   PetscFunctionReturn(0);
4840 }
4841 
4842 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], const PetscScalar values[], InsertMode mode)
4843 {
4844   PetscSection      clSection;
4845   IS                clPoints;
4846   PetscScalar       *array;
4847   PetscInt          *points = NULL;
4848   const PetscInt    *clp, *clperm;
4849   PetscInt          numFields, numPoints, p;
4850   PetscInt          offset = 0, f;
4851   PetscErrorCode    ierr;
4852 
4853   PetscFunctionBeginHot;
4854   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4855   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
4856   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
4857   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
4858   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
4859   /* Get points */
4860   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
4861   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4862   /* Get array */
4863   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
4864   /* Get values */
4865   for (f = 0; f < numFields; ++f) {
4866     const PetscInt    **perms = NULL;
4867     const PetscScalar **flips = NULL;
4868 
4869     if (!fieldActive[f]) {
4870       for (p = 0; p < numPoints*2; p += 2) {
4871         PetscInt fdof;
4872         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
4873         offset += fdof;
4874       }
4875       continue;
4876     }
4877     ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4878     switch (mode) {
4879     case INSERT_VALUES:
4880       for (p = 0; p < numPoints; p++) {
4881         const PetscInt    point = points[2*p];
4882         const PetscInt    *perm = perms ? perms[p] : NULL;
4883         const PetscScalar *flip = flips ? flips[p] : NULL;
4884         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array);
4885       } break;
4886     case INSERT_ALL_VALUES:
4887       for (p = 0; p < numPoints; p++) {
4888         const PetscInt    point = points[2*p];
4889         const PetscInt    *perm = perms ? perms[p] : NULL;
4890         const PetscScalar *flip = flips ? flips[p] : NULL;
4891         updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array);
4892         } break;
4893     case INSERT_BC_VALUES:
4894       for (p = 0; p < numPoints; p++) {
4895         const PetscInt    point = points[2*p];
4896         const PetscInt    *perm = perms ? perms[p] : NULL;
4897         const PetscScalar *flip = flips ? flips[p] : NULL;
4898         updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, clperm, values, &offset, array);
4899       } break;
4900     case ADD_VALUES:
4901       for (p = 0; p < numPoints; p++) {
4902         const PetscInt    point = points[2*p];
4903         const PetscInt    *perm = perms ? perms[p] : NULL;
4904         const PetscScalar *flip = flips ? flips[p] : NULL;
4905         updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array);
4906       } break;
4907     case ADD_ALL_VALUES:
4908       for (p = 0; p < numPoints; p++) {
4909         const PetscInt    point = points[2*p];
4910         const PetscInt    *perm = perms ? perms[p] : NULL;
4911         const PetscScalar *flip = flips ? flips[p] : NULL;
4912         updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array);
4913       } break;
4914     default:
4915       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
4916     }
4917     ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms,&flips);CHKERRQ(ierr);
4918   }
4919   /* Cleanup points */
4920   ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
4921   /* Cleanup array */
4922   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
4923   PetscFunctionReturn(0);
4924 }
4925 
4926 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
4927 {
4928   PetscMPIInt    rank;
4929   PetscInt       i, j;
4930   PetscErrorCode ierr;
4931 
4932   PetscFunctionBegin;
4933   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
4934   ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat for point %D\n", rank, point);CHKERRQ(ierr);
4935   for (i = 0; i < numRIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%D] = %D\n", rank, i, rindices[i]);CHKERRQ(ierr);}
4936   for (i = 0; i < numCIndices; i++) {ierr = PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%D] = %D\n", rank, i, cindices[i]);CHKERRQ(ierr);}
4937   numCIndices = numCIndices ? numCIndices : numRIndices;
4938   for (i = 0; i < numRIndices; i++) {
4939     ierr = PetscViewerASCIIPrintf(viewer, "[%d]", rank);CHKERRQ(ierr);
4940     for (j = 0; j < numCIndices; j++) {
4941 #if defined(PETSC_USE_COMPLEX)
4942       ierr = PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i*numCIndices+j]), (double)PetscImaginaryPart(values[i*numCIndices+j]));CHKERRQ(ierr);
4943 #else
4944       ierr = PetscViewerASCIIPrintf(viewer, " %g", (double)values[i*numCIndices+j]);CHKERRQ(ierr);
4945 #endif
4946     }
4947     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
4948   }
4949   PetscFunctionReturn(0);
4950 }
4951 
4952 /*
4953   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
4954 
4955   Input Parameters:
4956 + section - The section for this data layout
4957 . point   - The point contributing dofs with these indices
4958 . off     - The global offset of this point
4959 . loff    - The local offset of each field
4960 . setBC   - The flag determining whether to include indices of bounsary values
4961 . perm    - A permutation of the dofs on this point, or NULL
4962 - indperm - A permutation of the entire indices array, or NULL
4963 
4964   Output Parameter:
4965 . indices - Indices for dofs on this point
4966 
4967   Level: developer
4968 
4969   Note: The indices could be local or global, depending on the value of 'off'.
4970 */
4971 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
4972 {
4973   PetscInt        dof;   /* The number of unknowns on this point */
4974   PetscInt        cdof;  /* The number of constraints on this point */
4975   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
4976   PetscInt        cind = 0, k;
4977   PetscErrorCode  ierr;
4978 
4979   PetscFunctionBegin;
4980   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
4981   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
4982   if (!cdof || setBC) {
4983     for (k = 0; k < dof; ++k) {
4984       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
4985       const PetscInt ind    = indperm ? indperm[preind] : preind;
4986 
4987       indices[ind] = off + k;
4988     }
4989   } else {
4990     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
4991     for (k = 0; k < dof; ++k) {
4992       const PetscInt preind = perm ? *loff+perm[k] : *loff+k;
4993       const PetscInt ind    = indperm ? indperm[preind] : preind;
4994 
4995       if ((cind < cdof) && (k == cdofs[cind])) {
4996         /* Insert check for returning constrained indices */
4997         indices[ind] = -(off+k+1);
4998         ++cind;
4999       } else {
5000         indices[ind] = off+k-cind;
5001       }
5002     }
5003   }
5004   *loff += dof;
5005   PetscFunctionReturn(0);
5006 }
5007 
5008 /*
5009   This version only believes the point offset from the globalSection
5010 
5011  . off - The global offset of this point
5012 */
5013 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5014 {
5015   PetscInt       numFields, foff, f;
5016   PetscErrorCode ierr;
5017 
5018   PetscFunctionBegin;
5019   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5020   for (f = 0, foff = 0; f < numFields; ++f) {
5021     PetscInt        fdof, cfdof;
5022     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5023     PetscInt        cind = 0, b;
5024     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5025 
5026     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5027     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5028     if (!cfdof || setBC) {
5029       for (b = 0; b < fdof; ++b) {
5030         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5031         const PetscInt ind    = indperm ? indperm[preind] : preind;
5032 
5033         indices[ind] = off+foff+b;
5034       }
5035     } else {
5036       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5037       for (b = 0; b < fdof; ++b) {
5038         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5039         const PetscInt ind    = indperm ? indperm[preind] : preind;
5040 
5041         if ((cind < cfdof) && (b == fcdofs[cind])) {
5042           indices[ind] = -(off+foff+b+1);
5043           ++cind;
5044         } else {
5045           indices[ind] = off+foff+b-cind;
5046         }
5047       }
5048     }
5049     foff     += (setBC ? fdof : (fdof - cfdof));
5050     foffs[f] += fdof;
5051   }
5052   PetscFunctionReturn(0);
5053 }
5054 
5055 /*
5056   This version believes the globalSection offsets for each field, rather than just the point offset
5057 
5058  . foffs - The offset into 'indices' for each field, since it is segregated by field
5059 */
5060 PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
5061 {
5062   PetscInt       numFields, foff, f;
5063   PetscErrorCode ierr;
5064 
5065   PetscFunctionBegin;
5066   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5067   for (f = 0; f < numFields; ++f) {
5068     PetscInt        fdof, cfdof;
5069     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
5070     PetscInt        cind = 0, b;
5071     const PetscInt  *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
5072 
5073     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
5074     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
5075     ierr = PetscSectionGetFieldOffset(globalSection, point, f, &foff);CHKERRQ(ierr);
5076     if (!cfdof || setBC) {
5077       for (b = 0; b < fdof; ++b) {
5078         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5079         const PetscInt ind    = indperm ? indperm[preind] : preind;
5080 
5081         indices[ind] = foff+b;
5082       }
5083     } else {
5084       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
5085       for (b = 0; b < fdof; ++b) {
5086         const PetscInt preind = perm ? foffs[f]+perm[b] : foffs[f]+b;
5087         const PetscInt ind    = indperm ? indperm[preind] : preind;
5088 
5089         if ((cind < cfdof) && (b == fcdofs[cind])) {
5090           indices[ind] = -(foff+b+1);
5091           ++cind;
5092         } else {
5093           indices[ind] = foff+b-cind;
5094         }
5095       }
5096     }
5097     foffs[f] += fdof;
5098   }
5099   PetscFunctionReturn(0);
5100 }
5101 
5102 PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
5103 {
5104   Mat             cMat;
5105   PetscSection    aSec, cSec;
5106   IS              aIS;
5107   PetscInt        aStart = -1, aEnd = -1;
5108   const PetscInt  *anchors;
5109   PetscInt        numFields, f, p, q, newP = 0;
5110   PetscInt        newNumPoints = 0, newNumIndices = 0;
5111   PetscInt        *newPoints, *indices, *newIndices;
5112   PetscInt        maxAnchor, maxDof;
5113   PetscInt        newOffsets[32];
5114   PetscInt        *pointMatOffsets[32];
5115   PetscInt        *newPointOffsets[32];
5116   PetscScalar     *pointMat[32];
5117   PetscScalar     *newValues=NULL,*tmpValues;
5118   PetscBool       anyConstrained = PETSC_FALSE;
5119   PetscErrorCode  ierr;
5120 
5121   PetscFunctionBegin;
5122   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5123   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5124   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5125 
5126   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
5127   /* if there are point-to-point constraints */
5128   if (aSec) {
5129     ierr = PetscArrayzero(newOffsets, 32);CHKERRQ(ierr);
5130     ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
5131     ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
5132     /* figure out how many points are going to be in the new element matrix
5133      * (we allow double counting, because it's all just going to be summed
5134      * into the global matrix anyway) */
5135     for (p = 0; p < 2*numPoints; p+=2) {
5136       PetscInt b    = points[p];
5137       PetscInt bDof = 0, bSecDof;
5138 
5139       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5140       if (!bSecDof) {
5141         continue;
5142       }
5143       if (b >= aStart && b < aEnd) {
5144         ierr = PetscSectionGetDof(aSec,b,&bDof);CHKERRQ(ierr);
5145       }
5146       if (bDof) {
5147         /* this point is constrained */
5148         /* it is going to be replaced by its anchors */
5149         PetscInt bOff, q;
5150 
5151         anyConstrained = PETSC_TRUE;
5152         newNumPoints  += bDof;
5153         ierr = PetscSectionGetOffset(aSec,b,&bOff);CHKERRQ(ierr);
5154         for (q = 0; q < bDof; q++) {
5155           PetscInt a = anchors[bOff + q];
5156           PetscInt aDof;
5157 
5158           ierr           = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
5159           newNumIndices += aDof;
5160           for (f = 0; f < numFields; ++f) {
5161             PetscInt fDof;
5162 
5163             ierr             = PetscSectionGetFieldDof(section, a, f, &fDof);CHKERRQ(ierr);
5164             newOffsets[f+1] += fDof;
5165           }
5166         }
5167       }
5168       else {
5169         /* this point is not constrained */
5170         newNumPoints++;
5171         newNumIndices += bSecDof;
5172         for (f = 0; f < numFields; ++f) {
5173           PetscInt fDof;
5174 
5175           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5176           newOffsets[f+1] += fDof;
5177         }
5178       }
5179     }
5180   }
5181   if (!anyConstrained) {
5182     if (outNumPoints)  *outNumPoints  = 0;
5183     if (outNumIndices) *outNumIndices = 0;
5184     if (outPoints)     *outPoints     = NULL;
5185     if (outValues)     *outValues     = NULL;
5186     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5187     PetscFunctionReturn(0);
5188   }
5189 
5190   if (outNumPoints)  *outNumPoints  = newNumPoints;
5191   if (outNumIndices) *outNumIndices = newNumIndices;
5192 
5193   for (f = 0; f < numFields; ++f) newOffsets[f+1] += newOffsets[f];
5194 
5195   if (!outPoints && !outValues) {
5196     if (offsets) {
5197       for (f = 0; f <= numFields; f++) {
5198         offsets[f] = newOffsets[f];
5199       }
5200     }
5201     if (aSec) {ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);}
5202     PetscFunctionReturn(0);
5203   }
5204 
5205   if (numFields && newOffsets[numFields] != newNumIndices) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", newOffsets[numFields], newNumIndices);
5206 
5207   ierr = DMGetDefaultConstraints(dm, &cSec, &cMat);CHKERRQ(ierr);
5208 
5209   /* workspaces */
5210   if (numFields) {
5211     for (f = 0; f < numFields; f++) {
5212       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5213       ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5214     }
5215   }
5216   else {
5217     ierr = DMGetWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5218     ierr = DMGetWorkArray(dm,numPoints,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5219   }
5220 
5221   /* get workspaces for the point-to-point matrices */
5222   if (numFields) {
5223     PetscInt totalOffset, totalMatOffset;
5224 
5225     for (p = 0; p < numPoints; p++) {
5226       PetscInt b    = points[2*p];
5227       PetscInt bDof = 0, bSecDof;
5228 
5229       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5230       if (!bSecDof) {
5231         for (f = 0; f < numFields; f++) {
5232           newPointOffsets[f][p + 1] = 0;
5233           pointMatOffsets[f][p + 1] = 0;
5234         }
5235         continue;
5236       }
5237       if (b >= aStart && b < aEnd) {
5238         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5239       }
5240       if (bDof) {
5241         for (f = 0; f < numFields; f++) {
5242           PetscInt fDof, q, bOff, allFDof = 0;
5243 
5244           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5245           ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5246           for (q = 0; q < bDof; q++) {
5247             PetscInt a = anchors[bOff + q];
5248             PetscInt aFDof;
5249 
5250             ierr     = PetscSectionGetFieldDof(section, a, f, &aFDof);CHKERRQ(ierr);
5251             allFDof += aFDof;
5252           }
5253           newPointOffsets[f][p+1] = allFDof;
5254           pointMatOffsets[f][p+1] = fDof * allFDof;
5255         }
5256       }
5257       else {
5258         for (f = 0; f < numFields; f++) {
5259           PetscInt fDof;
5260 
5261           ierr = PetscSectionGetFieldDof(section, b, f, &fDof);CHKERRQ(ierr);
5262           newPointOffsets[f][p+1] = fDof;
5263           pointMatOffsets[f][p+1] = 0;
5264         }
5265       }
5266     }
5267     for (f = 0, totalOffset = 0, totalMatOffset = 0; f < numFields; f++) {
5268       newPointOffsets[f][0] = totalOffset;
5269       pointMatOffsets[f][0] = totalMatOffset;
5270       for (p = 0; p < numPoints; p++) {
5271         newPointOffsets[f][p+1] += newPointOffsets[f][p];
5272         pointMatOffsets[f][p+1] += pointMatOffsets[f][p];
5273       }
5274       totalOffset    = newPointOffsets[f][numPoints];
5275       totalMatOffset = pointMatOffsets[f][numPoints];
5276       ierr = DMGetWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5277     }
5278   }
5279   else {
5280     for (p = 0; p < numPoints; p++) {
5281       PetscInt b    = points[2*p];
5282       PetscInt bDof = 0, bSecDof;
5283 
5284       ierr = PetscSectionGetDof(section,b,&bSecDof);CHKERRQ(ierr);
5285       if (!bSecDof) {
5286         newPointOffsets[0][p + 1] = 0;
5287         pointMatOffsets[0][p + 1] = 0;
5288         continue;
5289       }
5290       if (b >= aStart && b < aEnd) {
5291         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5292       }
5293       if (bDof) {
5294         PetscInt bOff, q, allDof = 0;
5295 
5296         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5297         for (q = 0; q < bDof; q++) {
5298           PetscInt a = anchors[bOff + q], aDof;
5299 
5300           ierr    = PetscSectionGetDof(section, a, &aDof);CHKERRQ(ierr);
5301           allDof += aDof;
5302         }
5303         newPointOffsets[0][p+1] = allDof;
5304         pointMatOffsets[0][p+1] = bSecDof * allDof;
5305       }
5306       else {
5307         newPointOffsets[0][p+1] = bSecDof;
5308         pointMatOffsets[0][p+1] = 0;
5309       }
5310     }
5311     newPointOffsets[0][0] = 0;
5312     pointMatOffsets[0][0] = 0;
5313     for (p = 0; p < numPoints; p++) {
5314       newPointOffsets[0][p+1] += newPointOffsets[0][p];
5315       pointMatOffsets[0][p+1] += pointMatOffsets[0][p];
5316     }
5317     ierr = DMGetWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5318   }
5319 
5320   /* output arrays */
5321   ierr = DMGetWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5322 
5323   /* get the point-to-point matrices; construct newPoints */
5324   ierr = PetscSectionGetMaxDof(aSec, &maxAnchor);CHKERRQ(ierr);
5325   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
5326   ierr = DMGetWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5327   ierr = DMGetWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5328   if (numFields) {
5329     for (p = 0, newP = 0; p < numPoints; p++) {
5330       PetscInt b    = points[2*p];
5331       PetscInt o    = points[2*p+1];
5332       PetscInt bDof = 0, bSecDof;
5333 
5334       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5335       if (!bSecDof) {
5336         continue;
5337       }
5338       if (b >= aStart && b < aEnd) {
5339         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5340       }
5341       if (bDof) {
5342         PetscInt fStart[32], fEnd[32], fAnchorStart[32], fAnchorEnd[32], bOff, q;
5343 
5344         fStart[0] = 0;
5345         fEnd[0]   = 0;
5346         for (f = 0; f < numFields; f++) {
5347           PetscInt fDof;
5348 
5349           ierr        = PetscSectionGetFieldDof(cSec, b, f, &fDof);CHKERRQ(ierr);
5350           fStart[f+1] = fStart[f] + fDof;
5351           fEnd[f+1]   = fStart[f+1];
5352         }
5353         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5354         ierr = DMPlexGetIndicesPointFields_Internal(cSec, b, bOff, fEnd, PETSC_TRUE, perms, p, NULL, indices);CHKERRQ(ierr);
5355 
5356         fAnchorStart[0] = 0;
5357         fAnchorEnd[0]   = 0;
5358         for (f = 0; f < numFields; f++) {
5359           PetscInt fDof = newPointOffsets[f][p + 1] - newPointOffsets[f][p];
5360 
5361           fAnchorStart[f+1] = fAnchorStart[f] + fDof;
5362           fAnchorEnd[f+1]   = fAnchorStart[f + 1];
5363         }
5364         ierr = PetscSectionGetOffset(aSec, b, &bOff);CHKERRQ(ierr);
5365         for (q = 0; q < bDof; q++) {
5366           PetscInt a = anchors[bOff + q], aOff;
5367 
5368           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5369           newPoints[2*(newP + q)]     = a;
5370           newPoints[2*(newP + q) + 1] = 0;
5371           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5372           ierr = DMPlexGetIndicesPointFields_Internal(section, a, aOff, fAnchorEnd, PETSC_TRUE, NULL, -1, NULL, newIndices);CHKERRQ(ierr);
5373         }
5374         newP += bDof;
5375 
5376         if (outValues) {
5377           /* get the point-to-point submatrix */
5378           for (f = 0; f < numFields; f++) {
5379             ierr = MatGetValues(cMat,fEnd[f]-fStart[f],indices + fStart[f],fAnchorEnd[f] - fAnchorStart[f],newIndices + fAnchorStart[f],pointMat[f] + pointMatOffsets[f][p]);CHKERRQ(ierr);
5380           }
5381         }
5382       }
5383       else {
5384         newPoints[2 * newP]     = b;
5385         newPoints[2 * newP + 1] = o;
5386         newP++;
5387       }
5388     }
5389   } else {
5390     for (p = 0; p < numPoints; p++) {
5391       PetscInt b    = points[2*p];
5392       PetscInt o    = points[2*p+1];
5393       PetscInt bDof = 0, bSecDof;
5394 
5395       ierr = PetscSectionGetDof(section, b, &bSecDof);CHKERRQ(ierr);
5396       if (!bSecDof) {
5397         continue;
5398       }
5399       if (b >= aStart && b < aEnd) {
5400         ierr = PetscSectionGetDof(aSec, b, &bDof);CHKERRQ(ierr);
5401       }
5402       if (bDof) {
5403         PetscInt bEnd = 0, bAnchorEnd = 0, bOff;
5404 
5405         ierr = PetscSectionGetOffset(cSec, b, &bOff);CHKERRQ(ierr);
5406         ierr = DMPlexGetIndicesPoint_Internal(cSec, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, indices);CHKERRQ(ierr);
5407 
5408         ierr = PetscSectionGetOffset (aSec, b, &bOff);CHKERRQ(ierr);
5409         for (q = 0; q < bDof; q++) {
5410           PetscInt a = anchors[bOff + q], aOff;
5411 
5412           /* we take the orientation of ap into account in the order that we constructed the indices above: the newly added points have no orientation */
5413 
5414           newPoints[2*(newP + q)]     = a;
5415           newPoints[2*(newP + q) + 1] = 0;
5416           ierr = PetscSectionGetOffset(section, a, &aOff);CHKERRQ(ierr);
5417           ierr = DMPlexGetIndicesPoint_Internal(section, a, aOff, &bAnchorEnd, PETSC_TRUE, NULL, NULL, newIndices);CHKERRQ(ierr);
5418         }
5419         newP += bDof;
5420 
5421         /* get the point-to-point submatrix */
5422         if (outValues) {
5423           ierr = MatGetValues(cMat,bEnd,indices,bAnchorEnd,newIndices,pointMat[0] + pointMatOffsets[0][p]);CHKERRQ(ierr);
5424         }
5425       }
5426       else {
5427         newPoints[2 * newP]     = b;
5428         newPoints[2 * newP + 1] = o;
5429         newP++;
5430       }
5431     }
5432   }
5433 
5434   if (outValues) {
5435     ierr = DMGetWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5436     ierr = PetscArrayzero(tmpValues,newNumIndices*numIndices);CHKERRQ(ierr);
5437     /* multiply constraints on the right */
5438     if (numFields) {
5439       for (f = 0; f < numFields; f++) {
5440         PetscInt oldOff = offsets[f];
5441 
5442         for (p = 0; p < numPoints; p++) {
5443           PetscInt cStart = newPointOffsets[f][p];
5444           PetscInt b      = points[2 * p];
5445           PetscInt c, r, k;
5446           PetscInt dof;
5447 
5448           ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5449           if (!dof) {
5450             continue;
5451           }
5452           if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5453             PetscInt nCols         = newPointOffsets[f][p+1]-cStart;
5454             const PetscScalar *mat = pointMat[f] + pointMatOffsets[f][p];
5455 
5456             for (r = 0; r < numIndices; r++) {
5457               for (c = 0; c < nCols; c++) {
5458                 for (k = 0; k < dof; k++) {
5459                   tmpValues[r * newNumIndices + cStart + c] += values[r * numIndices + oldOff + k] * mat[k * nCols + c];
5460                 }
5461               }
5462             }
5463           }
5464           else {
5465             /* copy this column as is */
5466             for (r = 0; r < numIndices; r++) {
5467               for (c = 0; c < dof; c++) {
5468                 tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5469               }
5470             }
5471           }
5472           oldOff += dof;
5473         }
5474       }
5475     }
5476     else {
5477       PetscInt oldOff = 0;
5478       for (p = 0; p < numPoints; p++) {
5479         PetscInt cStart = newPointOffsets[0][p];
5480         PetscInt b      = points[2 * p];
5481         PetscInt c, r, k;
5482         PetscInt dof;
5483 
5484         ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5485         if (!dof) {
5486           continue;
5487         }
5488         if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5489           PetscInt nCols         = newPointOffsets[0][p+1]-cStart;
5490           const PetscScalar *mat = pointMat[0] + pointMatOffsets[0][p];
5491 
5492           for (r = 0; r < numIndices; r++) {
5493             for (c = 0; c < nCols; c++) {
5494               for (k = 0; k < dof; k++) {
5495                 tmpValues[r * newNumIndices + cStart + c] += mat[k * nCols + c] * values[r * numIndices + oldOff + k];
5496               }
5497             }
5498           }
5499         }
5500         else {
5501           /* copy this column as is */
5502           for (r = 0; r < numIndices; r++) {
5503             for (c = 0; c < dof; c++) {
5504               tmpValues[r * newNumIndices + cStart + c] = values[r * numIndices + oldOff + c];
5505             }
5506           }
5507         }
5508         oldOff += dof;
5509       }
5510     }
5511 
5512     if (multiplyLeft) {
5513       ierr = DMGetWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5514       ierr = PetscArrayzero(newValues,newNumIndices*newNumIndices);CHKERRQ(ierr);
5515       /* multiply constraints transpose on the left */
5516       if (numFields) {
5517         for (f = 0; f < numFields; f++) {
5518           PetscInt oldOff = offsets[f];
5519 
5520           for (p = 0; p < numPoints; p++) {
5521             PetscInt rStart = newPointOffsets[f][p];
5522             PetscInt b      = points[2 * p];
5523             PetscInt c, r, k;
5524             PetscInt dof;
5525 
5526             ierr = PetscSectionGetFieldDof(section,b,f,&dof);CHKERRQ(ierr);
5527             if (pointMatOffsets[f][p] < pointMatOffsets[f][p + 1]) {
5528               PetscInt nRows                        = newPointOffsets[f][p+1]-rStart;
5529               const PetscScalar *PETSC_RESTRICT mat = pointMat[f] + pointMatOffsets[f][p];
5530 
5531               for (r = 0; r < nRows; r++) {
5532                 for (c = 0; c < newNumIndices; c++) {
5533                   for (k = 0; k < dof; k++) {
5534                     newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5535                   }
5536                 }
5537               }
5538             }
5539             else {
5540               /* copy this row as is */
5541               for (r = 0; r < dof; r++) {
5542                 for (c = 0; c < newNumIndices; c++) {
5543                   newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5544                 }
5545               }
5546             }
5547             oldOff += dof;
5548           }
5549         }
5550       }
5551       else {
5552         PetscInt oldOff = 0;
5553 
5554         for (p = 0; p < numPoints; p++) {
5555           PetscInt rStart = newPointOffsets[0][p];
5556           PetscInt b      = points[2 * p];
5557           PetscInt c, r, k;
5558           PetscInt dof;
5559 
5560           ierr = PetscSectionGetDof(section,b,&dof);CHKERRQ(ierr);
5561           if (pointMatOffsets[0][p] < pointMatOffsets[0][p + 1]) {
5562             PetscInt nRows                        = newPointOffsets[0][p+1]-rStart;
5563             const PetscScalar *PETSC_RESTRICT mat = pointMat[0] + pointMatOffsets[0][p];
5564 
5565             for (r = 0; r < nRows; r++) {
5566               for (c = 0; c < newNumIndices; c++) {
5567                 for (k = 0; k < dof; k++) {
5568                   newValues[(rStart + r) * newNumIndices + c] += mat[k * nRows + r] * tmpValues[(oldOff + k) * newNumIndices + c];
5569                 }
5570               }
5571             }
5572           }
5573           else {
5574             /* copy this row as is */
5575             for (r = 0; r < dof; r++) {
5576               for (c = 0; c < newNumIndices; c++) {
5577                 newValues[(rStart + r) * newNumIndices + c] = tmpValues[(oldOff + r) * newNumIndices + c];
5578               }
5579             }
5580           }
5581           oldOff += dof;
5582         }
5583       }
5584 
5585       ierr = DMRestoreWorkArray(dm,newNumIndices*numIndices,MPIU_SCALAR,&tmpValues);CHKERRQ(ierr);
5586     }
5587     else {
5588       newValues = tmpValues;
5589     }
5590   }
5591 
5592   /* clean up */
5593   ierr = DMRestoreWorkArray(dm,maxDof,MPIU_INT,&indices);CHKERRQ(ierr);
5594   ierr = DMRestoreWorkArray(dm,maxAnchor*maxDof,MPIU_INT,&newIndices);CHKERRQ(ierr);
5595 
5596   if (numFields) {
5597     for (f = 0; f < numFields; f++) {
5598       ierr = DMRestoreWorkArray(dm,pointMatOffsets[f][numPoints],MPIU_SCALAR,&pointMat[f]);CHKERRQ(ierr);
5599       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[f]);CHKERRQ(ierr);
5600       ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[f]);CHKERRQ(ierr);
5601     }
5602   }
5603   else {
5604     ierr = DMRestoreWorkArray(dm,pointMatOffsets[0][numPoints],MPIU_SCALAR,&pointMat[0]);CHKERRQ(ierr);
5605     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&pointMatOffsets[0]);CHKERRQ(ierr);
5606     ierr = DMRestoreWorkArray(dm,numPoints+1,MPIU_INT,&newPointOffsets[0]);CHKERRQ(ierr);
5607   }
5608   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
5609 
5610   /* output */
5611   if (outPoints) {
5612     *outPoints = newPoints;
5613   }
5614   else {
5615     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5616   }
5617   if (outValues) {
5618     *outValues = newValues;
5619   }
5620   for (f = 0; f <= numFields; f++) {
5621     offsets[f] = newOffsets[f];
5622   }
5623   PetscFunctionReturn(0);
5624 }
5625 
5626 /*@C
5627   DMPlexGetClosureIndices - Get the global indices in a vector v for all points in the closure of the given point
5628 
5629   Not collective
5630 
5631   Input Parameters:
5632 + dm - The DM
5633 . section - The section describing the layout in v, or NULL to use the default section
5634 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5635 - point - The mesh point
5636 
5637   Output parameters:
5638 + numIndices - The number of indices
5639 . indices - The indices
5640 - outOffsets - Field offset if not NULL
5641 
5642   Note: Must call DMPlexRestoreClosureIndices() to free allocated memory
5643 
5644   Level: advanced
5645 
5646 .seealso DMPlexRestoreClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5647 @*/
5648 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices, PetscInt *outOffsets)
5649 {
5650   PetscSection    clSection;
5651   IS              clPoints;
5652   const PetscInt *clp, *clperm;
5653   const PetscInt  **perms[32] = {NULL};
5654   PetscInt       *points = NULL, *pointsNew;
5655   PetscInt        numPoints, numPointsNew;
5656   PetscInt        offsets[32];
5657   PetscInt        Nf, Nind, NindNew, off, globalOff, f, p;
5658   PetscErrorCode  ierr;
5659 
5660   PetscFunctionBegin;
5661   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5662   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5663   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5664   if (numIndices) PetscValidPointer(numIndices, 4);
5665   PetscValidPointer(indices, 5);
5666   ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
5667   if (Nf > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", Nf);
5668   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5669   /* Get points in closure */
5670   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5671   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5672   /* Get number of indices and indices per field */
5673   for (p = 0, Nind = 0; p < numPoints*2; p += 2) {
5674     PetscInt dof, fdof;
5675 
5676     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5677     for (f = 0; f < Nf; ++f) {
5678       ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5679       offsets[f+1] += fdof;
5680     }
5681     Nind += dof;
5682   }
5683   for (f = 1; f < Nf; ++f) offsets[f+1] += offsets[f];
5684   if (Nf && offsets[Nf] != Nind) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[Nf], Nind);
5685   if (!Nf) offsets[1] = Nind;
5686   /* Get dual space symmetries */
5687   for (f = 0; f < PetscMax(1,Nf); f++) {
5688     if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5689     else    {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5690   }
5691   /* Correct for hanging node constraints */
5692   {
5693     ierr = DMPlexAnchorsModifyMat(dm, section, numPoints, Nind, points, perms, NULL, &numPointsNew, &NindNew, &pointsNew, NULL, offsets, PETSC_TRUE);CHKERRQ(ierr);
5694     if (numPointsNew) {
5695       for (f = 0; f < PetscMax(1,Nf); f++) {
5696         if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5697         else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5698       }
5699       for (f = 0; f < PetscMax(1,Nf); f++) {
5700         if (Nf) {ierr = PetscSectionGetFieldPointSyms(section,f,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5701         else    {ierr = PetscSectionGetPointSyms(section,numPointsNew,pointsNew,&perms[f],NULL);CHKERRQ(ierr);}
5702       }
5703       ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5704       numPoints = numPointsNew;
5705       Nind      = NindNew;
5706       points    = pointsNew;
5707     }
5708   }
5709   /* Calculate indices */
5710   ierr = DMGetWorkArray(dm, Nind, MPIU_INT, indices);CHKERRQ(ierr);
5711   if (Nf) {
5712     if (outOffsets) {
5713       PetscInt f;
5714 
5715       for (f = 0; f <= Nf; f++) {
5716         outOffsets[f] = offsets[f];
5717       }
5718     }
5719     for (p = 0; p < numPoints; p++) {
5720       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5721       DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, *indices);
5722     }
5723   } else {
5724     for (p = 0, off = 0; p < numPoints; p++) {
5725       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5726 
5727       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5728       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, *indices);
5729     }
5730   }
5731   /* Cleanup points */
5732   for (f = 0; f < PetscMax(1,Nf); f++) {
5733     if (Nf) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5734     else    {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],NULL);CHKERRQ(ierr);}
5735   }
5736   if (numPointsNew) {
5737     ierr = DMRestoreWorkArray(dm, 2*numPointsNew, MPIU_INT, &pointsNew);CHKERRQ(ierr);
5738   } else {
5739     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5740   }
5741   if (numIndices) *numIndices = Nind;
5742   PetscFunctionReturn(0);
5743 }
5744 
5745 /*@C
5746   DMPlexRestoreClosureIndices - Restore the indices in a vector v for all points in the closure of the given point
5747 
5748   Not collective
5749 
5750   Input Parameters:
5751 + dm - The DM
5752 . section - The section describing the layout in v, or NULL to use the default section
5753 . globalSection - The section describing the parallel layout in v, or NULL to use the default section
5754 . point - The mesh point
5755 . numIndices - The number of indices
5756 . indices - The indices
5757 - outOffsets - Field offset if not NULL
5758 
5759   Level: advanced
5760 
5761 .seealso DMPlexGetClosureIndices(), DMPlexVecGetClosure(), DMPlexMatSetClosure()
5762 @*/
5763 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *numIndices, PetscInt **indices,PetscInt *outOffsets)
5764 {
5765   PetscErrorCode ierr;
5766 
5767   PetscFunctionBegin;
5768   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5769   PetscValidPointer(indices, 5);
5770   ierr = DMRestoreWorkArray(dm, 0, MPIU_INT, indices);CHKERRQ(ierr);
5771   PetscFunctionReturn(0);
5772 }
5773 
5774 /*@C
5775   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
5776 
5777   Not collective
5778 
5779   Input Parameters:
5780 + dm - The DM
5781 . section - The section describing the layout in v, or NULL to use the default section
5782 . globalSection - The section describing the layout in v, or NULL to use the default global section
5783 . A - The matrix
5784 . point - The point in the DM
5785 . values - The array of values
5786 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
5787 
5788   Fortran Notes:
5789   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
5790 
5791   Level: intermediate
5792 
5793 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
5794 @*/
5795 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5796 {
5797   DM_Plex            *mesh   = (DM_Plex*) dm->data;
5798   PetscSection        clSection;
5799   IS                  clPoints;
5800   PetscInt           *points = NULL, *newPoints;
5801   const PetscInt     *clp, *clperm;
5802   PetscInt           *indices;
5803   PetscInt            offsets[32];
5804   const PetscInt    **perms[32] = {NULL};
5805   const PetscScalar **flips[32] = {NULL};
5806   PetscInt            numFields, numPoints, newNumPoints, numIndices, newNumIndices, dof, off, globalOff, p, f;
5807   PetscScalar        *valCopy = NULL;
5808   PetscScalar        *newValues;
5809   PetscErrorCode      ierr;
5810 
5811   PetscFunctionBegin;
5812   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5813   if (!section) {ierr = DMGetSection(dm, &section);CHKERRQ(ierr);}
5814   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
5815   if (!globalSection) {ierr = DMGetGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
5816   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
5817   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
5818   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
5819   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5820   ierr = PetscArrayzero(offsets, 32);CHKERRQ(ierr);
5821   ierr = PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject) dm, NULL, &clperm);CHKERRQ(ierr);
5822   ierr = DMPlexGetCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5823   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
5824     PetscInt fdof;
5825 
5826     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
5827     for (f = 0; f < numFields; ++f) {
5828       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
5829       offsets[f+1] += fdof;
5830     }
5831     numIndices += dof;
5832   }
5833   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
5834 
5835   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", offsets[numFields], numIndices);
5836   /* Get symmetries */
5837   for (f = 0; f < PetscMax(1,numFields); f++) {
5838     if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5839     else           {ierr = PetscSectionGetPointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5840     if (values && flips[f]) { /* may need to apply sign changes to the element matrix */
5841       PetscInt foffset = offsets[f];
5842 
5843       for (p = 0; p < numPoints; p++) {
5844         PetscInt point          = points[2*p], fdof;
5845         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
5846 
5847         if (!numFields) {
5848           ierr = PetscSectionGetDof(section,point,&fdof);CHKERRQ(ierr);
5849         } else {
5850           ierr = PetscSectionGetFieldDof(section,point,f,&fdof);CHKERRQ(ierr);
5851         }
5852         if (flip) {
5853           PetscInt i, j, k;
5854 
5855           if (!valCopy) {
5856             ierr = DMGetWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5857             for (j = 0; j < numIndices * numIndices; j++) valCopy[j] = values[j];
5858             values = valCopy;
5859           }
5860           for (i = 0; i < fdof; i++) {
5861             PetscScalar fval = flip[i];
5862 
5863             for (k = 0; k < numIndices; k++) {
5864               valCopy[numIndices * (foffset + i) + k] *= fval;
5865               valCopy[numIndices * k + (foffset + i)] *= fval;
5866             }
5867           }
5868         }
5869         foffset += fdof;
5870       }
5871     }
5872   }
5873   ierr = DMPlexAnchorsModifyMat(dm,section,numPoints,numIndices,points,perms,values,&newNumPoints,&newNumIndices,&newPoints,&newValues,offsets,PETSC_TRUE);CHKERRQ(ierr);
5874   if (newNumPoints) {
5875     if (valCopy) {
5876       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5877     }
5878     for (f = 0; f < PetscMax(1,numFields); f++) {
5879       if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5880       else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5881     }
5882     for (f = 0; f < PetscMax(1,numFields); f++) {
5883       if (numFields) {ierr = PetscSectionGetFieldPointSyms(section,f,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5884       else           {ierr = PetscSectionGetPointSyms(section,newNumPoints,newPoints,&perms[f],&flips[f]);CHKERRQ(ierr);}
5885     }
5886     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5887     numPoints  = newNumPoints;
5888     numIndices = newNumIndices;
5889     points     = newPoints;
5890     values     = newValues;
5891   }
5892   ierr = DMGetWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5893   if (numFields) {
5894     PetscBool useFieldOffsets;
5895 
5896     ierr = PetscSectionGetUseFieldOffsets(globalSection, &useFieldOffsets);CHKERRQ(ierr);
5897     if (useFieldOffsets) {
5898       for (p = 0; p < numPoints; p++) {
5899         DMPlexGetIndicesPointFieldsSplit_Internal(section, globalSection, points[2*p], offsets, PETSC_FALSE, perms, p, clperm, indices);
5900       }
5901     } else {
5902       for (p = 0; p < numPoints; p++) {
5903         ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5904         DMPlexGetIndicesPointFields_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, perms, p, clperm, indices);
5905       }
5906     }
5907   } else {
5908     for (p = 0, off = 0; p < numPoints; p++) {
5909       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
5910       ierr = PetscSectionGetOffset(globalSection, points[2*p], &globalOff);CHKERRQ(ierr);
5911       DMPlexGetIndicesPoint_Internal(section, points[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, clperm, indices);
5912     }
5913   }
5914   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr);}
5915   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
5916   if (mesh->printFEM > 1) {
5917     PetscInt i;
5918     ierr = PetscPrintf(PETSC_COMM_SELF, "  Indices:");CHKERRQ(ierr);
5919     for (i = 0; i < numIndices; ++i) {ierr = PetscPrintf(PETSC_COMM_SELF, " %D", indices[i]);CHKERRQ(ierr);}
5920     ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
5921   }
5922   if (ierr) {
5923     PetscMPIInt    rank;
5924     PetscErrorCode ierr2;
5925 
5926     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
5927     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
5928     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values);CHKERRQ(ierr2);
5929     ierr2 = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr2);
5930     CHKERRQ(ierr);
5931   }
5932   for (f = 0; f < PetscMax(1,numFields); f++) {
5933     if (numFields) {ierr = PetscSectionRestoreFieldPointSyms(section,f,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5934     else           {ierr = PetscSectionRestorePointSyms(section,numPoints,points,&perms[f],&flips[f]);CHKERRQ(ierr);}
5935   }
5936   if (newNumPoints) {
5937     ierr = DMRestoreWorkArray(dm,newNumIndices*newNumIndices,MPIU_SCALAR,&newValues);CHKERRQ(ierr);
5938     ierr = DMRestoreWorkArray(dm,2*newNumPoints,MPIU_INT,&newPoints);CHKERRQ(ierr);
5939   }
5940   else {
5941     if (valCopy) {
5942       ierr = DMRestoreWorkArray(dm,numIndices*numIndices,MPIU_SCALAR,&valCopy);CHKERRQ(ierr);
5943     }
5944     ierr = DMPlexRestoreCompressedClosure(dm,section,point,&numPoints,&points,&clSection,&clPoints,&clp);CHKERRQ(ierr);
5945   }
5946   ierr = DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices);CHKERRQ(ierr);
5947   PetscFunctionReturn(0);
5948 }
5949 
5950 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
5951 {
5952   DM_Plex        *mesh   = (DM_Plex*) dmf->data;
5953   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
5954   PetscInt       *cpoints = NULL;
5955   PetscInt       *findices, *cindices;
5956   const PetscInt *fclperm, *cclperm;
5957   PetscInt        foffsets[32], coffsets[32];
5958   CellRefiner     cellRefiner;
5959   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
5960   PetscErrorCode  ierr;
5961 
5962   PetscFunctionBegin;
5963   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
5964   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
5965   if (!fsection) {ierr = DMGetSection(dmf, &fsection);CHKERRQ(ierr);}
5966   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
5967   if (!csection) {ierr = DMGetSection(dmc, &csection);CHKERRQ(ierr);}
5968   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
5969   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
5970   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
5971   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
5972   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
5973   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
5974   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
5975   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
5976   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
5977   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
5978   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
5979   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
5980   /* Column indices */
5981   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
5982   maxFPoints = numCPoints;
5983   /* Compress out points not in the section */
5984   /*   TODO: Squeeze out points with 0 dof as well */
5985   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
5986   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
5987     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
5988       cpoints[q*2]   = cpoints[p];
5989       cpoints[q*2+1] = cpoints[p+1];
5990       ++q;
5991     }
5992   }
5993   numCPoints = q;
5994   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
5995     PetscInt fdof;
5996 
5997     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
5998     if (!dof) continue;
5999     for (f = 0; f < numFields; ++f) {
6000       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6001       coffsets[f+1] += fdof;
6002     }
6003     numCIndices += dof;
6004   }
6005   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6006   /* Row indices */
6007   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6008   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6009   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6010   for (r = 0, q = 0; r < numSubcells; ++r) {
6011     /* TODO Map from coarse to fine cells */
6012     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6013     /* Compress out points not in the section */
6014     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6015     for (p = 0; p < numFPoints*2; p += 2) {
6016       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6017         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6018         if (!dof) continue;
6019         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6020         if (s < q) continue;
6021         ftotpoints[q*2]   = fpoints[p];
6022         ftotpoints[q*2+1] = fpoints[p+1];
6023         ++q;
6024       }
6025     }
6026     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6027   }
6028   numFPoints = q;
6029   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6030     PetscInt fdof;
6031 
6032     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6033     if (!dof) continue;
6034     for (f = 0; f < numFields; ++f) {
6035       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6036       foffsets[f+1] += fdof;
6037     }
6038     numFIndices += dof;
6039   }
6040   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6041 
6042   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6043   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6044   ierr = DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6045   ierr = DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6046   if (numFields) {
6047     const PetscInt **permsF[32] = {NULL};
6048     const PetscInt **permsC[32] = {NULL};
6049 
6050     for (f = 0; f < numFields; f++) {
6051       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6052       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6053     }
6054     for (p = 0; p < numFPoints; p++) {
6055       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6056       ierr = DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);CHKERRQ(ierr);
6057     }
6058     for (p = 0; p < numCPoints; p++) {
6059       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6060       ierr = DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);CHKERRQ(ierr);
6061     }
6062     for (f = 0; f < numFields; f++) {
6063       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6064       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6065     }
6066   } else {
6067     const PetscInt **permsF = NULL;
6068     const PetscInt **permsC = NULL;
6069 
6070     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6071     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6072     for (p = 0, off = 0; p < numFPoints; p++) {
6073       const PetscInt *perm = permsF ? permsF[p] : NULL;
6074 
6075       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6076       ierr = DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);CHKERRQ(ierr);
6077     }
6078     for (p = 0, off = 0; p < numCPoints; p++) {
6079       const PetscInt *perm = permsC ? permsC[p] : NULL;
6080 
6081       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6082       ierr = DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);CHKERRQ(ierr);
6083     }
6084     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6085     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6086   }
6087   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr);}
6088   /* TODO: flips */
6089   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
6090   if (ierr) {
6091     PetscMPIInt    rank;
6092     PetscErrorCode ierr2;
6093 
6094     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
6095     ierr2 = (*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
6096     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values);CHKERRQ(ierr2);
6097     ierr2 = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr2);
6098     ierr2 = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr2);
6099     CHKERRQ(ierr);
6100   }
6101   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6102   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6103   ierr = DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices);CHKERRQ(ierr);
6104   ierr = DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices);CHKERRQ(ierr);
6105   PetscFunctionReturn(0);
6106 }
6107 
6108 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
6109 {
6110   PetscInt      *fpoints = NULL, *ftotpoints = NULL;
6111   PetscInt      *cpoints = NULL;
6112   PetscInt       foffsets[32], coffsets[32];
6113   const PetscInt *fclperm, *cclperm;
6114   CellRefiner    cellRefiner;
6115   PetscInt       numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
6116   PetscErrorCode ierr;
6117 
6118   PetscFunctionBegin;
6119   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
6120   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
6121   if (!fsection) {ierr = DMGetSection(dmf, &fsection);CHKERRQ(ierr);}
6122   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
6123   if (!csection) {ierr = DMGetSection(dmc, &csection);CHKERRQ(ierr);}
6124   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
6125   if (!globalFSection) {ierr = DMGetGlobalSection(dmf, &globalFSection);CHKERRQ(ierr);}
6126   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
6127   if (!globalCSection) {ierr = DMGetGlobalSection(dmc, &globalCSection);CHKERRQ(ierr);}
6128   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
6129   ierr = PetscSectionGetNumFields(fsection, &numFields);CHKERRQ(ierr);
6130   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6131   ierr = PetscArrayzero(foffsets, 32);CHKERRQ(ierr);
6132   ierr = PetscArrayzero(coffsets, 32);CHKERRQ(ierr);
6133   ierr = PetscSectionGetClosureInversePermutation_Internal(fsection, (PetscObject) dmf, NULL, &fclperm);CHKERRQ(ierr);
6134   ierr = PetscSectionGetClosureInversePermutation_Internal(csection, (PetscObject) dmc, NULL, &cclperm);CHKERRQ(ierr);
6135   /* Column indices */
6136   ierr = DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6137   maxFPoints = numCPoints;
6138   /* Compress out points not in the section */
6139   /*   TODO: Squeeze out points with 0 dof as well */
6140   ierr = PetscSectionGetChart(csection, &pStart, &pEnd);CHKERRQ(ierr);
6141   for (p = 0, q = 0; p < numCPoints*2; p += 2) {
6142     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
6143       cpoints[q*2]   = cpoints[p];
6144       cpoints[q*2+1] = cpoints[p+1];
6145       ++q;
6146     }
6147   }
6148   numCPoints = q;
6149   for (p = 0, numCIndices = 0; p < numCPoints*2; p += 2) {
6150     PetscInt fdof;
6151 
6152     ierr = PetscSectionGetDof(csection, cpoints[p], &dof);CHKERRQ(ierr);
6153     if (!dof) continue;
6154     for (f = 0; f < numFields; ++f) {
6155       ierr           = PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof);CHKERRQ(ierr);
6156       coffsets[f+1] += fdof;
6157     }
6158     numCIndices += dof;
6159   }
6160   for (f = 1; f < numFields; ++f) coffsets[f+1] += coffsets[f];
6161   /* Row indices */
6162   ierr = DMPlexGetCellRefiner_Internal(dmc, &cellRefiner);CHKERRQ(ierr);
6163   ierr = CellRefinerGetAffineTransforms_Internal(cellRefiner, &numSubcells, NULL, NULL, NULL);CHKERRQ(ierr);
6164   ierr = DMGetWorkArray(dmf, maxFPoints*2*numSubcells, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6165   for (r = 0, q = 0; r < numSubcells; ++r) {
6166     /* TODO Map from coarse to fine cells */
6167     ierr = DMPlexGetTransitiveClosure(dmf, point*numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6168     /* Compress out points not in the section */
6169     ierr = PetscSectionGetChart(fsection, &pStart, &pEnd);CHKERRQ(ierr);
6170     for (p = 0; p < numFPoints*2; p += 2) {
6171       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
6172         ierr = PetscSectionGetDof(fsection, fpoints[p], &dof);CHKERRQ(ierr);
6173         if (!dof) continue;
6174         for (s = 0; s < q; ++s) if (fpoints[p] == ftotpoints[s*2]) break;
6175         if (s < q) continue;
6176         ftotpoints[q*2]   = fpoints[p];
6177         ftotpoints[q*2+1] = fpoints[p+1];
6178         ++q;
6179       }
6180     }
6181     ierr = DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints);CHKERRQ(ierr);
6182   }
6183   numFPoints = q;
6184   for (p = 0, numFIndices = 0; p < numFPoints*2; p += 2) {
6185     PetscInt fdof;
6186 
6187     ierr = PetscSectionGetDof(fsection, ftotpoints[p], &dof);CHKERRQ(ierr);
6188     if (!dof) continue;
6189     for (f = 0; f < numFields; ++f) {
6190       ierr           = PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof);CHKERRQ(ierr);
6191       foffsets[f+1] += fdof;
6192     }
6193     numFIndices += dof;
6194   }
6195   for (f = 1; f < numFields; ++f) foffsets[f+1] += foffsets[f];
6196 
6197   if (numFields && foffsets[numFields] != numFIndices) SETERRQ2(PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", foffsets[numFields], numFIndices);
6198   if (numFields && coffsets[numFields] != numCIndices) SETERRQ2(PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %D should be %D", coffsets[numFields], numCIndices);
6199   if (numFields) {
6200     const PetscInt **permsF[32] = {NULL};
6201     const PetscInt **permsC[32] = {NULL};
6202 
6203     for (f = 0; f < numFields; f++) {
6204       ierr = PetscSectionGetFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6205       ierr = PetscSectionGetFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6206     }
6207     for (p = 0; p < numFPoints; p++) {
6208       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6209       DMPlexGetIndicesPointFields_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices);
6210     }
6211     for (p = 0; p < numCPoints; p++) {
6212       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6213       DMPlexGetIndicesPointFields_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices);
6214     }
6215     for (f = 0; f < numFields; f++) {
6216       ierr = PetscSectionRestoreFieldPointSyms(fsection,f,numFPoints,ftotpoints,&permsF[f],NULL);CHKERRQ(ierr);
6217       ierr = PetscSectionRestoreFieldPointSyms(csection,f,numCPoints,cpoints,&permsC[f],NULL);CHKERRQ(ierr);
6218     }
6219   } else {
6220     const PetscInt **permsF = NULL;
6221     const PetscInt **permsC = NULL;
6222 
6223     ierr = PetscSectionGetPointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6224     ierr = PetscSectionGetPointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6225     for (p = 0, off = 0; p < numFPoints; p++) {
6226       const PetscInt *perm = permsF ? permsF[p] : NULL;
6227 
6228       ierr = PetscSectionGetOffset(globalFSection, ftotpoints[2*p], &globalOff);CHKERRQ(ierr);
6229       DMPlexGetIndicesPoint_Internal(fsection, ftotpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices);
6230     }
6231     for (p = 0, off = 0; p < numCPoints; p++) {
6232       const PetscInt *perm = permsC ? permsC[p] : NULL;
6233 
6234       ierr = PetscSectionGetOffset(globalCSection, cpoints[2*p], &globalOff);CHKERRQ(ierr);
6235       DMPlexGetIndicesPoint_Internal(csection, cpoints[2*p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices);
6236     }
6237     ierr = PetscSectionRestorePointSyms(fsection,numFPoints,ftotpoints,&permsF,NULL);CHKERRQ(ierr);
6238     ierr = PetscSectionRestorePointSyms(csection,numCPoints,cpoints,&permsC,NULL);CHKERRQ(ierr);
6239   }
6240   ierr = DMRestoreWorkArray(dmf, numCPoints*2*4, MPIU_INT, &ftotpoints);CHKERRQ(ierr);
6241   ierr = DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints);CHKERRQ(ierr);
6242   PetscFunctionReturn(0);
6243 }
6244 
6245 /*@
6246   DMPlexGetHybridBounds - Get the first mesh point of each dimension which is a hybrid
6247 
6248   Input Parameter:
6249 . dm - The DMPlex object
6250 
6251   Output Parameters:
6252 + cMax - The first hybrid cell
6253 . fMax - The first hybrid face
6254 . eMax - The first hybrid edge
6255 - vMax - The first hybrid vertex
6256 
6257   Level: developer
6258 
6259 .seealso DMPlexCreateHybridMesh(), DMPlexSetHybridBounds()
6260 @*/
6261 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
6262 {
6263   DM_Plex       *mesh = (DM_Plex*) dm->data;
6264   PetscInt       dim;
6265   PetscErrorCode ierr;
6266 
6267   PetscFunctionBegin;
6268   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6269   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6270   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6271   if (cMax) *cMax = mesh->hybridPointMax[dim];
6272   if (fMax) *fMax = mesh->hybridPointMax[PetscMax(dim-1,0)];
6273   if (eMax) *eMax = mesh->hybridPointMax[1];
6274   if (vMax) *vMax = mesh->hybridPointMax[0];
6275   PetscFunctionReturn(0);
6276 }
6277 
6278 static PetscErrorCode DMPlexCreateDimStratum(DM dm, DMLabel depthLabel, DMLabel dimLabel, PetscInt d, PetscInt dMax)
6279 {
6280   IS             is, his;
6281   PetscInt       first = 0, stride;
6282   PetscBool      isStride;
6283   PetscErrorCode ierr;
6284 
6285   PetscFunctionBegin;
6286   ierr = DMLabelGetStratumIS(depthLabel, d, &is);CHKERRQ(ierr);
6287   ierr = PetscObjectTypeCompare((PetscObject) is, ISSTRIDE, &isStride);CHKERRQ(ierr);
6288   if (isStride) {
6289     ierr = ISStrideGetInfo(is, &first, &stride);CHKERRQ(ierr);
6290   }
6291   if (is && (!isStride || stride != 1)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM is not stratified: depth %D IS is not contiguous", d);
6292   ierr = ISCreateStride(PETSC_COMM_SELF, (dMax - first), first, 1, &his);CHKERRQ(ierr);
6293   ierr = DMLabelSetStratumIS(dimLabel, d, his);CHKERRQ(ierr);
6294   ierr = ISDestroy(&his);CHKERRQ(ierr);
6295   ierr = ISDestroy(&is);CHKERRQ(ierr);
6296   PetscFunctionReturn(0);
6297 }
6298 
6299 /*@
6300   DMPlexSetHybridBounds - Set the first mesh point of each dimension which is a hybrid
6301 
6302   Input Parameters:
6303 + dm   - The DMPlex object
6304 . cMax - The first hybrid cell
6305 . fMax - The first hybrid face
6306 . eMax - The first hybrid edge
6307 - vMax - The first hybrid vertex
6308 
6309   Level: developer
6310 
6311 .seealso DMPlexCreateHybridMesh(), DMPlexGetHybridBounds()
6312 @*/
6313 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
6314 {
6315   DM_Plex       *mesh = (DM_Plex*) dm->data;
6316   PetscInt       dim;
6317   PetscErrorCode ierr;
6318 
6319   PetscFunctionBegin;
6320   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6321   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6322   if (dim < 0) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "DM dimension not yet set");
6323   if (cMax >= 0) mesh->hybridPointMax[dim]               = cMax;
6324   if (fMax >= 0) mesh->hybridPointMax[PetscMax(dim-1,0)] = fMax;
6325   if (eMax >= 0) mesh->hybridPointMax[1]                 = eMax;
6326   if (vMax >= 0) mesh->hybridPointMax[0]                 = vMax;
6327   PetscFunctionReturn(0);
6328 }
6329 
6330 /*@C
6331   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
6332 
6333   Input Parameter:
6334 . dm   - The DMPlex object
6335 
6336   Output Parameter:
6337 . cellHeight - The height of a cell
6338 
6339   Level: developer
6340 
6341 .seealso DMPlexSetVTKCellHeight()
6342 @*/
6343 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
6344 {
6345   DM_Plex *mesh = (DM_Plex*) dm->data;
6346 
6347   PetscFunctionBegin;
6348   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6349   PetscValidPointer(cellHeight, 2);
6350   *cellHeight = mesh->vtkCellHeight;
6351   PetscFunctionReturn(0);
6352 }
6353 
6354 /*@C
6355   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
6356 
6357   Input Parameters:
6358 + dm   - The DMPlex object
6359 - cellHeight - The height of a cell
6360 
6361   Level: developer
6362 
6363 .seealso DMPlexGetVTKCellHeight()
6364 @*/
6365 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
6366 {
6367   DM_Plex *mesh = (DM_Plex*) dm->data;
6368 
6369   PetscFunctionBegin;
6370   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6371   mesh->vtkCellHeight = cellHeight;
6372   PetscFunctionReturn(0);
6373 }
6374 
6375 /* We can easily have a form that takes an IS instead */
6376 PetscErrorCode DMPlexCreateNumbering_Internal(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
6377 {
6378   PetscSection   section, globalSection;
6379   PetscInt      *numbers, p;
6380   PetscErrorCode ierr;
6381 
6382   PetscFunctionBegin;
6383   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6384   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
6385   for (p = pStart; p < pEnd; ++p) {
6386     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
6387   }
6388   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
6389   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
6390   ierr = PetscMalloc1(pEnd - pStart, &numbers);CHKERRQ(ierr);
6391   for (p = pStart; p < pEnd; ++p) {
6392     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
6393     if (numbers[p-pStart] < 0) numbers[p-pStart] -= shift;
6394     else                       numbers[p-pStart] += shift;
6395   }
6396   ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
6397   if (globalSize) {
6398     PetscLayout layout;
6399     ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject) dm), globalSection, &layout);CHKERRQ(ierr);
6400     ierr = PetscLayoutGetSize(layout, globalSize);CHKERRQ(ierr);
6401     ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
6402   }
6403   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6404   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
6405   PetscFunctionReturn(0);
6406 }
6407 
6408 PetscErrorCode DMPlexCreateCellNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalCellNumbers)
6409 {
6410   PetscInt       cellHeight, cStart, cEnd, cMax;
6411   PetscErrorCode ierr;
6412 
6413   PetscFunctionBegin;
6414   ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
6415   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6416   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6417   if (cMax >= 0 && !includeHybrid) cEnd = PetscMin(cEnd, cMax);
6418   ierr = DMPlexCreateNumbering_Internal(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers);CHKERRQ(ierr);
6419   PetscFunctionReturn(0);
6420 }
6421 
6422 /*@
6423   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
6424 
6425   Input Parameter:
6426 . dm   - The DMPlex object
6427 
6428   Output Parameter:
6429 . globalCellNumbers - Global cell numbers for all cells on this process
6430 
6431   Level: developer
6432 
6433 .seealso DMPlexGetVertexNumbering()
6434 @*/
6435 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
6436 {
6437   DM_Plex       *mesh = (DM_Plex*) dm->data;
6438   PetscErrorCode ierr;
6439 
6440   PetscFunctionBegin;
6441   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6442   if (!mesh->globalCellNumbers) {ierr = DMPlexCreateCellNumbering_Internal(dm, PETSC_FALSE, &mesh->globalCellNumbers);CHKERRQ(ierr);}
6443   *globalCellNumbers = mesh->globalCellNumbers;
6444   PetscFunctionReturn(0);
6445 }
6446 
6447 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
6448 {
6449   PetscInt       vStart, vEnd, vMax;
6450   PetscErrorCode ierr;
6451 
6452   PetscFunctionBegin;
6453   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6454   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6455   ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
6456   if (vMax >= 0 && !includeHybrid) vEnd = PetscMin(vEnd, vMax);
6457   ierr = DMPlexCreateNumbering_Internal(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers);CHKERRQ(ierr);
6458   PetscFunctionReturn(0);
6459 }
6460 
6461 /*@
6462   DMPlexGetVertexNumbering - Get a global certex numbering for all vertices on this process
6463 
6464   Input Parameter:
6465 . dm   - The DMPlex object
6466 
6467   Output Parameter:
6468 . globalVertexNumbers - Global vertex numbers for all vertices on this process
6469 
6470   Level: developer
6471 
6472 .seealso DMPlexGetCellNumbering()
6473 @*/
6474 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
6475 {
6476   DM_Plex       *mesh = (DM_Plex*) dm->data;
6477   PetscErrorCode ierr;
6478 
6479   PetscFunctionBegin;
6480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6481   if (!mesh->globalVertexNumbers) {ierr = DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers);CHKERRQ(ierr);}
6482   *globalVertexNumbers = mesh->globalVertexNumbers;
6483   PetscFunctionReturn(0);
6484 }
6485 
6486 /*@
6487   DMPlexCreatePointNumbering - Create a global numbering for all points on this process
6488 
6489   Input Parameter:
6490 . dm   - The DMPlex object
6491 
6492   Output Parameter:
6493 . globalPointNumbers - Global numbers for all points on this process
6494 
6495   Level: developer
6496 
6497 .seealso DMPlexGetCellNumbering()
6498 @*/
6499 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
6500 {
6501   IS             nums[4];
6502   PetscInt       depths[4], gdepths[4], starts[4];
6503   PetscInt       depth, d, shift = 0;
6504   PetscErrorCode ierr;
6505 
6506   PetscFunctionBegin;
6507   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6508   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6509   /* For unstratified meshes use dim instead of depth */
6510   if (depth < 0) {ierr = DMGetDimension(dm, &depth);CHKERRQ(ierr);}
6511   for (d = 0; d <= depth; ++d) {
6512     PetscInt end;
6513 
6514     depths[d] = depth-d;
6515     ierr = DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end);CHKERRQ(ierr);
6516     if (!(starts[d]-end)) { starts[d] = depths[d] = -1; }
6517   }
6518   ierr = PetscSortIntWithArray(depth+1, starts, depths);CHKERRQ(ierr);
6519   ierr = MPIU_Allreduce(depths, gdepths, depth+1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));CHKERRQ(ierr);
6520   for (d = 0; d <= depth; ++d) {
6521     if (starts[d] >= 0 && depths[d] != gdepths[d]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected depth %D, found %D",depths[d],gdepths[d]);
6522   }
6523   for (d = 0; d <= depth; ++d) {
6524     PetscInt pStart, pEnd, gsize;
6525 
6526     ierr = DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd);CHKERRQ(ierr);
6527     ierr = DMPlexCreateNumbering_Internal(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]);CHKERRQ(ierr);
6528     shift += gsize;
6529   }
6530   ierr = ISConcatenate(PetscObjectComm((PetscObject) dm), depth+1, nums, globalPointNumbers);CHKERRQ(ierr);
6531   for (d = 0; d <= depth; ++d) {ierr = ISDestroy(&nums[d]);CHKERRQ(ierr);}
6532   PetscFunctionReturn(0);
6533 }
6534 
6535 
6536 /*@
6537   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
6538 
6539   Input Parameter:
6540 . dm - The DMPlex object
6541 
6542   Output Parameter:
6543 . ranks - The rank field
6544 
6545   Options Database Keys:
6546 . -dm_partition_view - Adds the rank field into the DM output from -dm_view using the same viewer
6547 
6548   Level: intermediate
6549 
6550 .seealso: DMView()
6551 @*/
6552 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
6553 {
6554   DM             rdm;
6555   PetscFE        fe;
6556   PetscScalar   *r;
6557   PetscMPIInt    rank;
6558   PetscInt       dim, cStart, cEnd, c;
6559   PetscErrorCode ierr;
6560 
6561   PetscFunctionBeginUser;
6562   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6563   PetscValidPointer(ranks, 2);
6564   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
6565   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6566   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6567   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___rank_", -1, &fe);CHKERRQ(ierr);
6568   ierr = PetscObjectSetName((PetscObject) fe, "rank");CHKERRQ(ierr);
6569   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6570   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6571   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6572   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6573   ierr = DMCreateGlobalVector(rdm, ranks);CHKERRQ(ierr);
6574   ierr = PetscObjectSetName((PetscObject) *ranks, "partition");CHKERRQ(ierr);
6575   ierr = VecGetArray(*ranks, &r);CHKERRQ(ierr);
6576   for (c = cStart; c < cEnd; ++c) {
6577     PetscScalar *lr;
6578 
6579     ierr = DMPlexPointGlobalRef(rdm, c, r, &lr);CHKERRQ(ierr);
6580     *lr = rank;
6581   }
6582   ierr = VecRestoreArray(*ranks, &r);CHKERRQ(ierr);
6583   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6584   PetscFunctionReturn(0);
6585 }
6586 
6587 /*@
6588   DMPlexCreateLabelField - Create a cell field whose value is the label value for that cell
6589 
6590   Input Parameters:
6591 + dm    - The DMPlex
6592 - label - The DMLabel
6593 
6594   Output Parameter:
6595 . val - The label value field
6596 
6597   Options Database Keys:
6598 . -dm_label_view - Adds the label value field into the DM output from -dm_view using the same viewer
6599 
6600   Level: intermediate
6601 
6602 .seealso: DMView()
6603 @*/
6604 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
6605 {
6606   DM             rdm;
6607   PetscFE        fe;
6608   PetscScalar   *v;
6609   PetscInt       dim, cStart, cEnd, c;
6610   PetscErrorCode ierr;
6611 
6612   PetscFunctionBeginUser;
6613   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6614   PetscValidPointer(label, 2);
6615   PetscValidPointer(val, 3);
6616   ierr = DMClone(dm, &rdm);CHKERRQ(ierr);
6617   ierr = DMGetDimension(rdm, &dim);CHKERRQ(ierr);
6618   ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) rdm), dim, 1, PETSC_TRUE, "PETSc___label_value_", -1, &fe);CHKERRQ(ierr);
6619   ierr = PetscObjectSetName((PetscObject) fe, "label_value");CHKERRQ(ierr);
6620   ierr = DMSetField(rdm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
6621   ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
6622   ierr = DMCreateDS(rdm);CHKERRQ(ierr);
6623   ierr = DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6624   ierr = DMCreateGlobalVector(rdm, val);CHKERRQ(ierr);
6625   ierr = PetscObjectSetName((PetscObject) *val, "label_value");CHKERRQ(ierr);
6626   ierr = VecGetArray(*val, &v);CHKERRQ(ierr);
6627   for (c = cStart; c < cEnd; ++c) {
6628     PetscScalar *lv;
6629     PetscInt     cval;
6630 
6631     ierr = DMPlexPointGlobalRef(rdm, c, v, &lv);CHKERRQ(ierr);
6632     ierr = DMLabelGetValue(label, c, &cval);CHKERRQ(ierr);
6633     *lv = cval;
6634   }
6635   ierr = VecRestoreArray(*val, &v);CHKERRQ(ierr);
6636   ierr = DMDestroy(&rdm);CHKERRQ(ierr);
6637   PetscFunctionReturn(0);
6638 }
6639 
6640 /*@
6641   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
6642 
6643   Input Parameter:
6644 . dm - The DMPlex object
6645 
6646   Note: This is a useful diagnostic when creating meshes programmatically.
6647 
6648   Level: developer
6649 
6650 .seealso: DMCreate(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6651 @*/
6652 PetscErrorCode DMPlexCheckSymmetry(DM dm)
6653 {
6654   PetscSection    coneSection, supportSection;
6655   const PetscInt *cone, *support;
6656   PetscInt        coneSize, c, supportSize, s;
6657   PetscInt        pStart, pEnd, p, pp, csize, ssize;
6658   PetscBool       storagecheck = PETSC_TRUE;
6659   PetscErrorCode  ierr;
6660 
6661   PetscFunctionBegin;
6662   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6663   ierr = DMPlexGetConeSection(dm, &coneSection);CHKERRQ(ierr);
6664   ierr = DMPlexGetSupportSection(dm, &supportSection);CHKERRQ(ierr);
6665   /* Check that point p is found in the support of its cone points, and vice versa */
6666   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6667   for (p = pStart; p < pEnd; ++p) {
6668     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
6669     ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6670     for (c = 0; c < coneSize; ++c) {
6671       PetscBool dup = PETSC_FALSE;
6672       PetscInt  d;
6673       for (d = c-1; d >= 0; --d) {
6674         if (cone[c] == cone[d]) {dup = PETSC_TRUE; break;}
6675       }
6676       ierr = DMPlexGetSupportSize(dm, cone[c], &supportSize);CHKERRQ(ierr);
6677       ierr = DMPlexGetSupport(dm, cone[c], &support);CHKERRQ(ierr);
6678       for (s = 0; s < supportSize; ++s) {
6679         if (support[s] == p) break;
6680       }
6681       if ((s >= supportSize) || (dup && (support[s+1] != p))) {
6682         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", p);CHKERRQ(ierr);
6683         for (s = 0; s < coneSize; ++s) {
6684           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[s]);CHKERRQ(ierr);
6685         }
6686         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6687         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", cone[c]);CHKERRQ(ierr);
6688         for (s = 0; s < supportSize; ++s) {
6689           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[s]);CHKERRQ(ierr);
6690         }
6691         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6692         if (dup) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not repeatedly found in support of repeated cone point %D", p, cone[c]);
6693         else SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in support of cone point %D", p, cone[c]);
6694       }
6695     }
6696     ierr = DMPlexGetTreeParent(dm, p, &pp, NULL);CHKERRQ(ierr);
6697     if (p != pp) { storagecheck = PETSC_FALSE; continue; }
6698     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
6699     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
6700     for (s = 0; s < supportSize; ++s) {
6701       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
6702       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
6703       for (c = 0; c < coneSize; ++c) {
6704         ierr = DMPlexGetTreeParent(dm, cone[c], &pp, NULL);CHKERRQ(ierr);
6705         if (cone[c] != pp) { c = 0; break; }
6706         if (cone[c] == p) break;
6707       }
6708       if (c >= coneSize) {
6709         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D support: ", p);CHKERRQ(ierr);
6710         for (c = 0; c < supportSize; ++c) {
6711           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", support[c]);CHKERRQ(ierr);
6712         }
6713         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6714         ierr = PetscPrintf(PETSC_COMM_SELF, "p: %D cone: ", support[s]);CHKERRQ(ierr);
6715         for (c = 0; c < coneSize; ++c) {
6716           ierr = PetscPrintf(PETSC_COMM_SELF, "%D, ", cone[c]);CHKERRQ(ierr);
6717         }
6718         ierr = PetscPrintf(PETSC_COMM_SELF, "\n");CHKERRQ(ierr);
6719         SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %D not found in cone of support point %D", p, support[s]);
6720       }
6721     }
6722   }
6723   if (storagecheck) {
6724     ierr = PetscSectionGetStorageSize(coneSection, &csize);CHKERRQ(ierr);
6725     ierr = PetscSectionGetStorageSize(supportSection, &ssize);CHKERRQ(ierr);
6726     if (csize != ssize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %D != Total support size %D", csize, ssize);
6727   }
6728   PetscFunctionReturn(0);
6729 }
6730 
6731 /*@
6732   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
6733 
6734   Input Parameters:
6735 + dm - The DMPlex object
6736 - cellHeight - Normally 0
6737 
6738   Note: This is a useful diagnostic when creating meshes programmatically.
6739   Currently applicable only to homogeneous simplex or tensor meshes.
6740 
6741   Level: developer
6742 
6743 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckFaces()
6744 @*/
6745 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
6746 {
6747   PetscInt       dim, numCorners, numHybridCorners, vStart, vEnd, cStart, cEnd, cMax, c;
6748   PetscBool      isSimplex = PETSC_FALSE;
6749   PetscErrorCode ierr;
6750 
6751   PetscFunctionBegin;
6752   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6753   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6754   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
6755   if (cStart < cEnd) {
6756     ierr = DMPlexGetConeSize(dm, cStart, &c);CHKERRQ(ierr);
6757     isSimplex = c == dim+1 ? PETSC_TRUE : PETSC_FALSE;
6758   }
6759   switch (dim) {
6760   case 1: numCorners = isSimplex ? 2 : 2; numHybridCorners = isSimplex ? 2 : 2; break;
6761   case 2: numCorners = isSimplex ? 3 : 4; numHybridCorners = isSimplex ? 4 : 4; break;
6762   case 3: numCorners = isSimplex ? 4 : 8; numHybridCorners = isSimplex ? 6 : 8; break;
6763   default:
6764     SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot handle meshes of dimension %D", dim);
6765   }
6766   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6767   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
6768   cMax = cMax >= 0 ? cMax : cEnd;
6769   for (c = cStart; c < cMax; ++c) {
6770     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6771 
6772     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6773     for (cl = 0; cl < closureSize*2; cl += 2) {
6774       const PetscInt p = closure[cl];
6775       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6776     }
6777     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6778     if (coneSize != numCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has  %D vertices != %D", c, coneSize, numCorners);
6779   }
6780   for (c = cMax; c < cEnd; ++c) {
6781     PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
6782 
6783     ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6784     for (cl = 0; cl < closureSize*2; cl += 2) {
6785       const PetscInt p = closure[cl];
6786       if ((p >= vStart) && (p < vEnd)) ++coneSize;
6787     }
6788     ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6789     if (coneSize > numHybridCorners) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Hybrid cell %D has  %D vertices > %D", c, coneSize, numHybridCorners);
6790   }
6791   PetscFunctionReturn(0);
6792 }
6793 
6794 /*@
6795   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
6796 
6797   Input Parameters:
6798 + dm - The DMPlex object
6799 - cellHeight - Normally 0
6800 
6801   Note: This is a useful diagnostic when creating meshes programmatically.
6802 
6803   Level: developer
6804 
6805 .seealso: DMCreate(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton()
6806 @*/
6807 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
6808 {
6809   PetscInt       pMax[4];
6810   PetscInt       dim, depth, vStart, vEnd, cStart, cEnd, c, h;
6811   PetscErrorCode ierr;
6812 
6813   PetscFunctionBegin;
6814   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6815   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6816   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6817   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
6818   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], &pMax[dim-1], &pMax[1], &pMax[0]);CHKERRQ(ierr);
6819   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
6820     ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
6821     for (c = cStart; c < cEnd; ++c) {
6822       const PetscInt *cone, *ornt, *faces;
6823       PetscInt        numFaces, faceSize, coneSize,f;
6824       PetscInt       *closure = NULL, closureSize, cl, numCorners = 0;
6825 
6826       if (pMax[dim-h] >= 0 && c >= pMax[dim-h]) continue;
6827       ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
6828       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
6829       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
6830       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6831       for (cl = 0; cl < closureSize*2; cl += 2) {
6832         const PetscInt p = closure[cl];
6833         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
6834       }
6835       ierr = DMPlexGetRawFaces_Internal(dm, dim-h, numCorners, closure, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6836       if (coneSize != numFaces) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %D has %D faces but should have %D", c, coneSize, numFaces);
6837       for (f = 0; f < numFaces; ++f) {
6838         PetscInt *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
6839 
6840         ierr = DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6841         for (cl = 0; cl < fclosureSize*2; cl += 2) {
6842           const PetscInt p = fclosure[cl];
6843           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
6844         }
6845         if (fnumCorners != faceSize) SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%D) of cell %D has %D vertices but should have %D", cone[f], f, c, fnumCorners, faceSize);
6846         for (v = 0; v < fnumCorners; ++v) {
6847           if (fclosure[v] != faces[f*faceSize+v]) SETERRQ6(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %D (%d) of cell %D vertex %D, %D != %D", cone[f], f, c, v, fclosure[v], faces[f*faceSize+v]);
6848         }
6849         ierr = DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure);CHKERRQ(ierr);
6850       }
6851       ierr = DMPlexRestoreFaces_Internal(dm, dim, c, &numFaces, &faceSize, &faces);CHKERRQ(ierr);
6852       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
6853     }
6854   }
6855   PetscFunctionReturn(0);
6856 }
6857 
6858 /*@
6859   DMPlexCheckGeometry - Check the geometry of mesh cells
6860 
6861   Input Parameter:
6862 . dm - The DMPlex object
6863 
6864   Note: This is a useful diagnostic when creating meshes programmatically.
6865 
6866   Level: developer
6867 
6868 .seealso: DMCreate(), DMCheckSymmetry(), DMCheckSkeleton(), DMCheckFaces()
6869 @*/
6870 PetscErrorCode DMPlexCheckGeometry(DM dm)
6871 {
6872   PetscReal      detJ, J[9], refVol = 1.0;
6873   PetscReal      vol;
6874   PetscInt       dim, depth, d, cStart, cEnd, c;
6875   PetscErrorCode ierr;
6876 
6877   PetscFunctionBegin;
6878   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
6879   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6880   for (d = 0; d < dim; ++d) refVol *= 2.0;
6881   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
6882   for (c = cStart; c < cEnd; ++c) {
6883     ierr = DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ);CHKERRQ(ierr);
6884     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted, |J| = %g", c, (double) detJ);
6885     ierr = PetscInfo2(dm, "Cell %D FEM Volume %g\n", c, (double) detJ*refVol);CHKERRQ(ierr);
6886     if (depth > 1) {
6887       ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
6888       if (vol <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %d is inverted, vol = %g", c, (double) vol);
6889       ierr = PetscInfo2(dm, "Cell %D FVM Volume %g\n", c, (double) vol);CHKERRQ(ierr);
6890     }
6891   }
6892   PetscFunctionReturn(0);
6893 }
6894 
6895 static PetscErrorCode DMPlexAreAllConePointsInArray_Private(DM dm, PetscInt p, PetscInt npoints, const PetscInt *points, PetscInt *missingPoint)
6896 {
6897   PetscInt i,l,n;
6898   const PetscInt *cone;
6899   PetscErrorCode ierr;
6900 
6901   PetscFunctionBegin;
6902   *missingPoint = -1;
6903   ierr = DMPlexGetConeSize(dm, p, &n);CHKERRQ(ierr);
6904   ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
6905   for (i=0; i<n; i++) {
6906     ierr = PetscFindInt(cone[i], npoints, points, &l);CHKERRQ(ierr);
6907     if (l < 0) {
6908       *missingPoint = cone[i];
6909       break;
6910     }
6911   }
6912   PetscFunctionReturn(0);
6913 }
6914 
6915 /*@
6916   DMPlexCheckPointSF - Check that several sufficient conditions are met for the point SF of this plex.
6917 
6918   Input Parameters:
6919 . dm - The DMPlex object
6920 
6921   Note: This is mainly intended for debugging/testing purposes.
6922 
6923   Level: developer
6924 
6925 .seealso: DMGetPointSF(), DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6926 @*/
6927 PetscErrorCode DMPlexCheckPointSF(DM dm)
6928 {
6929   PetscSF sf;
6930   PetscInt d,depth,i,nleaves,p,plo,phi,missingPoint;
6931   const PetscInt *locals;
6932   PetscErrorCode ierr;
6933 
6934   PetscFunctionBegin;
6935   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6936   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6937   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
6938   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &locals, NULL);CHKERRQ(ierr);
6939 
6940   /* 1) check there are no faces in 2D, cells in 3D, in interface */
6941   ierr = DMPlexGetVTKCellHeight(dm, &d);CHKERRQ(ierr);
6942   ierr = DMPlexGetHeightStratum(dm, d, &plo, &phi);CHKERRQ(ierr);
6943   for (i=0; i<nleaves; i++) {
6944     p = locals[i];
6945     if (p >= plo && p < phi) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d which is a cell",p);
6946   }
6947 
6948   /* 2) if some point is in interface, then all its cone points must be also in interface  */
6949   for (i=0; i<nleaves; i++) {
6950     p = locals[i];
6951     ierr = DMPlexAreAllConePointsInArray_Private(dm, p, nleaves, locals, &missingPoint);CHKERRQ(ierr);
6952     if (missingPoint >= 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "point SF contains %d but not %d from its cone",p,missingPoint);
6953   }
6954   PetscFunctionReturn(0);
6955 }
6956 
6957 typedef struct cell_stats
6958 {
6959   PetscReal min, max, sum, squaresum;
6960   PetscInt  count;
6961 } cell_stats_t;
6962 
6963 static void cell_stats_reduce(void *a, void *b, int * len, MPI_Datatype *datatype)
6964 {
6965   PetscInt i, N = *len;
6966 
6967   for (i = 0; i < N; i++) {
6968     cell_stats_t *A = (cell_stats_t *) a;
6969     cell_stats_t *B = (cell_stats_t *) b;
6970 
6971     B->min = PetscMin(A->min,B->min);
6972     B->max = PetscMax(A->max,B->max);
6973     B->sum += A->sum;
6974     B->squaresum += A->squaresum;
6975     B->count += A->count;
6976   }
6977 }
6978 
6979 /*@
6980   DMPlexCheckCellShape - Checks the Jacobian of the mapping and computes some minimal statistics.
6981 
6982   Input Parameters:
6983 + dm - The DMPlex object
6984 - output - If true, statistics will be displayed on stdout
6985 
6986   Note: This is mainly intended for debugging/testing purposes.
6987 
6988   Level: developer
6989 
6990 .seealso: DMPlexCheckSymmetry(), DMPlexCheckSkeleton(), DMPlexCheckFaces()
6991 @*/
6992 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output)
6993 {
6994   PetscMPIInt    rank,size;
6995   PetscInt       dim, c, cStart, cEnd, cMax, count = 0;
6996   cell_stats_t   stats, globalStats;
6997   PetscReal      *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
6998   MPI_Comm       comm = PetscObjectComm((PetscObject)dm);
6999   DM             dmCoarse;
7000   PetscErrorCode ierr;
7001 
7002   PetscFunctionBegin;
7003   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7004   stats.min   = PETSC_MAX_REAL;
7005   stats.max   = PETSC_MIN_REAL;
7006   stats.sum   = stats.squaresum = 0.;
7007   stats.count = 0;
7008 
7009   ierr = DMGetCoordinateDim(dm,&dim);CHKERRQ(ierr);
7010   ierr = PetscMalloc2(dim * dim, &J, dim * dim, &invJ);CHKERRQ(ierr);
7011   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
7012   ierr = DMPlexGetHybridBounds(dm,&cMax,NULL,NULL,NULL);CHKERRQ(ierr);
7013   cMax = cMax < 0 ? cEnd : cMax;
7014   for (c = cStart; c < cMax; c++) {
7015     PetscInt  i;
7016     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
7017 
7018     ierr = DMPlexComputeCellGeometryAffineFEM(dm,c,NULL,J,invJ,&detJ);CHKERRQ(ierr);
7019     if (detJ < 0.0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %D is inverted", c);
7020     for (i = 0; i < dim * dim; i++) {
7021       frobJ    += J[i] * J[i];
7022       frobInvJ += invJ[i] * invJ[i];
7023     }
7024     cond2 = frobJ * frobInvJ;
7025     cond  = PetscSqrtReal(cond2);
7026 
7027     stats.min        = PetscMin(stats.min,cond);
7028     stats.max        = PetscMax(stats.max,cond);
7029     stats.sum       += cond;
7030     stats.squaresum += cond2;
7031     stats.count++;
7032   }
7033 
7034   ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
7035   if (size > 1) {
7036     PetscMPIInt   blockLengths[2] = {4,1};
7037     MPI_Aint      blockOffsets[2] = {offsetof(cell_stats_t,min),offsetof(cell_stats_t,count)};
7038     MPI_Datatype  blockTypes[2]   = {MPIU_REAL,MPIU_INT}, statType;
7039     MPI_Op        statReduce;
7040 
7041     ierr = MPI_Type_create_struct(2,blockLengths,blockOffsets,blockTypes,&statType);CHKERRQ(ierr);
7042     ierr = MPI_Type_commit(&statType);CHKERRQ(ierr);
7043     ierr = MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce);CHKERRQ(ierr);
7044     ierr = MPI_Reduce(&stats,&globalStats,1,statType,statReduce,0,comm);CHKERRQ(ierr);
7045     ierr = MPI_Op_free(&statReduce);CHKERRQ(ierr);
7046     ierr = MPI_Type_free(&statType);CHKERRQ(ierr);
7047   } else {
7048     ierr = PetscArraycpy(&globalStats,&stats,1);CHKERRQ(ierr);
7049   }
7050 
7051   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
7052   if (!rank) {
7053     count = globalStats.count;
7054     min   = globalStats.min;
7055     max   = globalStats.max;
7056     mean  = globalStats.sum / globalStats.count;
7057     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1),0)) : 0.0;
7058   }
7059 
7060   if (output) {
7061     ierr = PetscPrintf(comm,"Mesh with %D cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double) min, (double) max, (double) mean, (double) stdev);CHKERRQ(ierr);
7062   }
7063   ierr = PetscFree2(J,invJ);CHKERRQ(ierr);
7064 
7065   ierr = DMGetCoarseDM(dm,&dmCoarse);CHKERRQ(ierr);
7066   if (dmCoarse) {
7067     PetscBool isplex;
7068 
7069     ierr = PetscObjectTypeCompare((PetscObject)dmCoarse,DMPLEX,&isplex);CHKERRQ(ierr);
7070     if (isplex) {
7071       ierr = DMPlexCheckCellShape(dmCoarse,output);CHKERRQ(ierr);
7072     }
7073   }
7074   PetscFunctionReturn(0);
7075 }
7076 
7077 /* Pointwise interpolation
7078      Just code FEM for now
7079      u^f = I u^c
7080      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
7081      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
7082      I_{ij} = psi^f_i phi^c_j
7083 */
7084 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
7085 {
7086   PetscSection   gsc, gsf;
7087   PetscInt       m, n;
7088   void          *ctx;
7089   DM             cdm;
7090   PetscBool      regular, ismatis;
7091   PetscErrorCode ierr;
7092 
7093   PetscFunctionBegin;
7094   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7095   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7096   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7097   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7098 
7099   ierr = PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis);CHKERRQ(ierr);
7100   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), interpolation);CHKERRQ(ierr);
7101   ierr = MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7102   ierr = MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype);CHKERRQ(ierr);
7103   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7104 
7105   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7106   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7107   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeInterpolatorNested(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7108   else                            {ierr = DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx);CHKERRQ(ierr);}
7109   ierr = MatViewFromOptions(*interpolation, NULL, "-interp_mat_view");CHKERRQ(ierr);
7110   if (scaling) {
7111     /* Use naive scaling */
7112     ierr = DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling);CHKERRQ(ierr);
7113   }
7114   PetscFunctionReturn(0);
7115 }
7116 
7117 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
7118 {
7119   PetscErrorCode ierr;
7120   VecScatter     ctx;
7121 
7122   PetscFunctionBegin;
7123   ierr = DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL);CHKERRQ(ierr);
7124   ierr = MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat);CHKERRQ(ierr);
7125   ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
7126   PetscFunctionReturn(0);
7127 }
7128 
7129 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
7130 {
7131   PetscSection   gsc, gsf;
7132   PetscInt       m, n;
7133   void          *ctx;
7134   DM             cdm;
7135   PetscBool      regular;
7136   PetscErrorCode ierr;
7137 
7138   PetscFunctionBegin;
7139   ierr = DMGetGlobalSection(dmFine, &gsf);CHKERRQ(ierr);
7140   ierr = PetscSectionGetConstrainedStorageSize(gsf, &m);CHKERRQ(ierr);
7141   ierr = DMGetGlobalSection(dmCoarse, &gsc);CHKERRQ(ierr);
7142   ierr = PetscSectionGetConstrainedStorageSize(gsc, &n);CHKERRQ(ierr);
7143 
7144   ierr = MatCreate(PetscObjectComm((PetscObject) dmCoarse), mass);CHKERRQ(ierr);
7145   ierr = MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
7146   ierr = MatSetType(*mass, dmCoarse->mattype);CHKERRQ(ierr);
7147   ierr = DMGetApplicationContext(dmFine, &ctx);CHKERRQ(ierr);
7148 
7149   ierr = DMGetCoarseDM(dmFine, &cdm);CHKERRQ(ierr);
7150   ierr = DMPlexGetRegularRefinement(dmFine, &regular);CHKERRQ(ierr);
7151   if (regular && cdm == dmCoarse) {ierr = DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7152   else                            {ierr = DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx);CHKERRQ(ierr);}
7153   ierr = MatViewFromOptions(*mass, NULL, "-mass_mat_view");CHKERRQ(ierr);
7154   PetscFunctionReturn(0);
7155 }
7156 
7157 /*@
7158   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7159 
7160   Input Parameter:
7161 . dm - The DMPlex object
7162 
7163   Output Parameter:
7164 . regular - The flag
7165 
7166   Level: intermediate
7167 
7168 .seealso: DMPlexSetRegularRefinement()
7169 @*/
7170 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
7171 {
7172   PetscFunctionBegin;
7173   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7174   PetscValidPointer(regular, 2);
7175   *regular = ((DM_Plex *) dm->data)->regularRefinement;
7176   PetscFunctionReturn(0);
7177 }
7178 
7179 /*@
7180   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
7181 
7182   Input Parameters:
7183 + dm - The DMPlex object
7184 - regular - The flag
7185 
7186   Level: intermediate
7187 
7188 .seealso: DMPlexGetRegularRefinement()
7189 @*/
7190 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
7191 {
7192   PetscFunctionBegin;
7193   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7194   ((DM_Plex *) dm->data)->regularRefinement = regular;
7195   PetscFunctionReturn(0);
7196 }
7197 
7198 /* anchors */
7199 /*@
7200   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
7201   call DMPlexGetAnchors() directly: if there are anchors, then DMPlexGetAnchors() is called during DMGetConstraints().
7202 
7203   not collective
7204 
7205   Input Parameters:
7206 . dm - The DMPlex object
7207 
7208   Output Parameters:
7209 + anchorSection - If not NULL, set to the section describing which points anchor the constrained points.
7210 - anchorIS - If not NULL, set to the list of anchors indexed by anchorSection
7211 
7212 
7213   Level: intermediate
7214 
7215 .seealso: DMPlexSetAnchors(), DMGetConstraints(), DMSetConstraints()
7216 @*/
7217 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
7218 {
7219   DM_Plex *plex = (DM_Plex *)dm->data;
7220   PetscErrorCode ierr;
7221 
7222   PetscFunctionBegin;
7223   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7224   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) {ierr = (*plex->createanchors)(dm);CHKERRQ(ierr);}
7225   if (anchorSection) *anchorSection = plex->anchorSection;
7226   if (anchorIS) *anchorIS = plex->anchorIS;
7227   PetscFunctionReturn(0);
7228 }
7229 
7230 /*@
7231   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.  Unlike boundary conditions,
7232   when a point's degrees of freedom in a section are constrained to an outside value, the anchor constraints set a
7233   point's degrees of freedom to be a linear combination of other points' degrees of freedom.
7234 
7235   After specifying the layout of constraints with DMPlexSetAnchors(), one specifies the constraints by calling
7236   DMGetConstraints() and filling in the entries in the constraint matrix.
7237 
7238   collective on dm
7239 
7240   Input Parameters:
7241 + dm - The DMPlex object
7242 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7243 - anchorIS - The list of all anchor points.  Must have a local communicator (PETSC_COMM_SELF or derivative).
7244 
7245   The reference counts of anchorSection and anchorIS are incremented.
7246 
7247   Level: intermediate
7248 
7249 .seealso: DMPlexGetAnchors(), DMGetConstraints(), DMSetConstraints()
7250 @*/
7251 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
7252 {
7253   DM_Plex        *plex = (DM_Plex *)dm->data;
7254   PetscMPIInt    result;
7255   PetscErrorCode ierr;
7256 
7257   PetscFunctionBegin;
7258   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7259   if (anchorSection) {
7260     PetscValidHeaderSpecific(anchorSection,PETSC_SECTION_CLASSID,2);
7261     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorSection),&result);CHKERRQ(ierr);
7262     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor section must have local communicator");
7263   }
7264   if (anchorIS) {
7265     PetscValidHeaderSpecific(anchorIS,IS_CLASSID,3);
7266     ierr = MPI_Comm_compare(PETSC_COMM_SELF,PetscObjectComm((PetscObject)anchorIS),&result);CHKERRQ(ierr);
7267     if (result != MPI_CONGRUENT && result != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"anchor IS must have local communicator");
7268   }
7269 
7270   ierr = PetscObjectReference((PetscObject)anchorSection);CHKERRQ(ierr);
7271   ierr = PetscSectionDestroy(&plex->anchorSection);CHKERRQ(ierr);
7272   plex->anchorSection = anchorSection;
7273 
7274   ierr = PetscObjectReference((PetscObject)anchorIS);CHKERRQ(ierr);
7275   ierr = ISDestroy(&plex->anchorIS);CHKERRQ(ierr);
7276   plex->anchorIS = anchorIS;
7277 
7278 #if defined(PETSC_USE_DEBUG)
7279   if (anchorIS && anchorSection) {
7280     PetscInt size, a, pStart, pEnd;
7281     const PetscInt *anchors;
7282 
7283     ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7284     ierr = ISGetLocalSize(anchorIS,&size);CHKERRQ(ierr);
7285     ierr = ISGetIndices(anchorIS,&anchors);CHKERRQ(ierr);
7286     for (a = 0; a < size; a++) {
7287       PetscInt p;
7288 
7289       p = anchors[a];
7290       if (p >= pStart && p < pEnd) {
7291         PetscInt dof;
7292 
7293         ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7294         if (dof) {
7295           PetscErrorCode ierr2;
7296 
7297           ierr2 = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr2);
7298           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Point %D cannot be constrained and an anchor",p);
7299         }
7300       }
7301     }
7302     ierr = ISRestoreIndices(anchorIS,&anchors);CHKERRQ(ierr);
7303   }
7304 #endif
7305   /* reset the generic constraints */
7306   ierr = DMSetDefaultConstraints(dm,NULL,NULL);CHKERRQ(ierr);
7307   PetscFunctionReturn(0);
7308 }
7309 
7310 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
7311 {
7312   PetscSection anchorSection;
7313   PetscInt pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
7314   PetscErrorCode ierr;
7315 
7316   PetscFunctionBegin;
7317   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7318   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7319   ierr = PetscSectionCreate(PETSC_COMM_SELF,cSec);CHKERRQ(ierr);
7320   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7321   if (numFields) {
7322     PetscInt f;
7323     ierr = PetscSectionSetNumFields(*cSec,numFields);CHKERRQ(ierr);
7324 
7325     for (f = 0; f < numFields; f++) {
7326       PetscInt numComp;
7327 
7328       ierr = PetscSectionGetFieldComponents(section,f,&numComp);CHKERRQ(ierr);
7329       ierr = PetscSectionSetFieldComponents(*cSec,f,numComp);CHKERRQ(ierr);
7330     }
7331   }
7332   ierr = PetscSectionGetChart(anchorSection,&pStart,&pEnd);CHKERRQ(ierr);
7333   ierr = PetscSectionGetChart(section,&sStart,&sEnd);CHKERRQ(ierr);
7334   pStart = PetscMax(pStart,sStart);
7335   pEnd   = PetscMin(pEnd,sEnd);
7336   pEnd   = PetscMax(pStart,pEnd);
7337   ierr = PetscSectionSetChart(*cSec,pStart,pEnd);CHKERRQ(ierr);
7338   for (p = pStart; p < pEnd; p++) {
7339     ierr = PetscSectionGetDof(anchorSection,p,&dof);CHKERRQ(ierr);
7340     if (dof) {
7341       ierr = PetscSectionGetDof(section,p,&dof);CHKERRQ(ierr);
7342       ierr = PetscSectionSetDof(*cSec,p,dof);CHKERRQ(ierr);
7343       for (f = 0; f < numFields; f++) {
7344         ierr = PetscSectionGetFieldDof(section,p,f,&dof);CHKERRQ(ierr);
7345         ierr = PetscSectionSetFieldDof(*cSec,p,f,dof);CHKERRQ(ierr);
7346       }
7347     }
7348   }
7349   ierr = PetscSectionSetUp(*cSec);CHKERRQ(ierr);
7350   PetscFunctionReturn(0);
7351 }
7352 
7353 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
7354 {
7355   PetscSection aSec;
7356   PetscInt pStart, pEnd, p, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
7357   const PetscInt *anchors;
7358   PetscInt numFields, f;
7359   IS aIS;
7360   PetscErrorCode ierr;
7361 
7362   PetscFunctionBegin;
7363   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7364   ierr = PetscSectionGetStorageSize(cSec, &m);CHKERRQ(ierr);
7365   ierr = PetscSectionGetStorageSize(section, &n);CHKERRQ(ierr);
7366   ierr = MatCreate(PETSC_COMM_SELF,cMat);CHKERRQ(ierr);
7367   ierr = MatSetSizes(*cMat,m,n,m,n);CHKERRQ(ierr);
7368   ierr = MatSetType(*cMat,MATSEQAIJ);CHKERRQ(ierr);
7369   ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
7370   ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
7371   /* cSec will be a subset of aSec and section */
7372   ierr = PetscSectionGetChart(cSec,&pStart,&pEnd);CHKERRQ(ierr);
7373   ierr = PetscMalloc1(m+1,&i);CHKERRQ(ierr);
7374   i[0] = 0;
7375   ierr = PetscSectionGetNumFields(section,&numFields);CHKERRQ(ierr);
7376   for (p = pStart; p < pEnd; p++) {
7377     PetscInt rDof, rOff, r;
7378 
7379     ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7380     if (!rDof) continue;
7381     ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7382     if (numFields) {
7383       for (f = 0; f < numFields; f++) {
7384         annz = 0;
7385         for (r = 0; r < rDof; r++) {
7386           a = anchors[rOff + r];
7387           ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7388           annz += aDof;
7389         }
7390         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7391         ierr = PetscSectionGetFieldOffset(cSec,p,f,&off);CHKERRQ(ierr);
7392         for (q = 0; q < dof; q++) {
7393           i[off + q + 1] = i[off + q] + annz;
7394         }
7395       }
7396     }
7397     else {
7398       annz = 0;
7399       for (q = 0; q < dof; q++) {
7400         a = anchors[off + q];
7401         ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7402         annz += aDof;
7403       }
7404       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7405       ierr = PetscSectionGetOffset(cSec,p,&off);CHKERRQ(ierr);
7406       for (q = 0; q < dof; q++) {
7407         i[off + q + 1] = i[off + q] + annz;
7408       }
7409     }
7410   }
7411   nnz = i[m];
7412   ierr = PetscMalloc1(nnz,&j);CHKERRQ(ierr);
7413   offset = 0;
7414   for (p = pStart; p < pEnd; p++) {
7415     if (numFields) {
7416       for (f = 0; f < numFields; f++) {
7417         ierr = PetscSectionGetFieldDof(cSec,p,f,&dof);CHKERRQ(ierr);
7418         for (q = 0; q < dof; q++) {
7419           PetscInt rDof, rOff, r;
7420           ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7421           ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7422           for (r = 0; r < rDof; r++) {
7423             PetscInt s;
7424 
7425             a = anchors[rOff + r];
7426             ierr = PetscSectionGetFieldDof(section,a,f,&aDof);CHKERRQ(ierr);
7427             ierr = PetscSectionGetFieldOffset(section,a,f,&aOff);CHKERRQ(ierr);
7428             for (s = 0; s < aDof; s++) {
7429               j[offset++] = aOff + s;
7430             }
7431           }
7432         }
7433       }
7434     }
7435     else {
7436       ierr = PetscSectionGetDof(cSec,p,&dof);CHKERRQ(ierr);
7437       for (q = 0; q < dof; q++) {
7438         PetscInt rDof, rOff, r;
7439         ierr = PetscSectionGetDof(aSec,p,&rDof);CHKERRQ(ierr);
7440         ierr = PetscSectionGetOffset(aSec,p,&rOff);CHKERRQ(ierr);
7441         for (r = 0; r < rDof; r++) {
7442           PetscInt s;
7443 
7444           a = anchors[rOff + r];
7445           ierr = PetscSectionGetDof(section,a,&aDof);CHKERRQ(ierr);
7446           ierr = PetscSectionGetOffset(section,a,&aOff);CHKERRQ(ierr);
7447           for (s = 0; s < aDof; s++) {
7448             j[offset++] = aOff + s;
7449           }
7450         }
7451       }
7452     }
7453   }
7454   ierr = MatSeqAIJSetPreallocationCSR(*cMat,i,j,NULL);CHKERRQ(ierr);
7455   ierr = PetscFree(i);CHKERRQ(ierr);
7456   ierr = PetscFree(j);CHKERRQ(ierr);
7457   ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
7458   PetscFunctionReturn(0);
7459 }
7460 
7461 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
7462 {
7463   DM_Plex        *plex = (DM_Plex *)dm->data;
7464   PetscSection   anchorSection, section, cSec;
7465   Mat            cMat;
7466   PetscErrorCode ierr;
7467 
7468   PetscFunctionBegin;
7469   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7470   ierr = DMPlexGetAnchors(dm,&anchorSection,NULL);CHKERRQ(ierr);
7471   if (anchorSection) {
7472     PetscInt Nf;
7473 
7474     ierr = DMGetSection(dm,&section);CHKERRQ(ierr);
7475     ierr = DMPlexCreateConstraintSection_Anchors(dm,section,&cSec);CHKERRQ(ierr);
7476     ierr = DMPlexCreateConstraintMatrix_Anchors(dm,section,cSec,&cMat);CHKERRQ(ierr);
7477     ierr = DMGetNumFields(dm,&Nf);CHKERRQ(ierr);
7478     if (Nf && plex->computeanchormatrix) {ierr = (*plex->computeanchormatrix)(dm,section,cSec,cMat);CHKERRQ(ierr);}
7479     ierr = DMSetDefaultConstraints(dm,cSec,cMat);CHKERRQ(ierr);
7480     ierr = PetscSectionDestroy(&cSec);CHKERRQ(ierr);
7481     ierr = MatDestroy(&cMat);CHKERRQ(ierr);
7482   }
7483   PetscFunctionReturn(0);
7484 }
7485 
7486 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
7487 {
7488   IS             subis;
7489   PetscSection   section, subsection;
7490   PetscErrorCode ierr;
7491 
7492   PetscFunctionBegin;
7493   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
7494   if (!section) SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
7495   if (!subdm)   SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
7496   /* Create subdomain */
7497   ierr = DMPlexFilter(dm, label, value, subdm);CHKERRQ(ierr);
7498   /* Create submodel */
7499   ierr = DMPlexCreateSubpointIS(*subdm, &subis);CHKERRQ(ierr);
7500   ierr = PetscSectionCreateSubmeshSection(section, subis, &subsection);CHKERRQ(ierr);
7501   ierr = ISDestroy(&subis);CHKERRQ(ierr);
7502   ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
7503   ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
7504   ierr = DMCopyDisc(dm, *subdm);CHKERRQ(ierr);
7505   /* Create map from submodel to global model */
7506   if (is) {
7507     PetscSection    sectionGlobal, subsectionGlobal;
7508     IS              spIS;
7509     const PetscInt *spmap;
7510     PetscInt       *subIndices;
7511     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
7512     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
7513 
7514     ierr = DMPlexCreateSubpointIS(*subdm, &spIS);CHKERRQ(ierr);
7515     ierr = ISGetIndices(spIS, &spmap);CHKERRQ(ierr);
7516     ierr = PetscSectionGetNumFields(section, &Nf);CHKERRQ(ierr);
7517     ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
7518     ierr = DMGetDefaultGlobalSection(*subdm, &subsectionGlobal);CHKERRQ(ierr);
7519     ierr = PetscSectionGetChart(subsection, &pStart, &pEnd);CHKERRQ(ierr);
7520     for (p = pStart; p < pEnd; ++p) {
7521       PetscInt gdof, pSubSize  = 0;
7522 
7523       ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
7524       if (gdof > 0) {
7525         for (f = 0; f < Nf; ++f) {
7526           PetscInt fdof, fcdof;
7527 
7528           ierr     = PetscSectionGetFieldDof(subsection, p, f, &fdof);CHKERRQ(ierr);
7529           ierr     = PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof);CHKERRQ(ierr);
7530           pSubSize += fdof-fcdof;
7531         }
7532         subSize += pSubSize;
7533         if (pSubSize) {
7534           if (bs < 0) {
7535             bs = pSubSize;
7536           } else if (bs != pSubSize) {
7537             /* Layout does not admit a pointwise block size */
7538             bs = 1;
7539           }
7540         }
7541       }
7542     }
7543     /* Must have same blocksize on all procs (some might have no points) */
7544     bsLocal[0] = bs < 0 ? PETSC_MAX_INT : bs; bsLocal[1] = bs;
7545     ierr = PetscGlobalMinMaxInt(PetscObjectComm((PetscObject) dm), bsLocal, bsMinMax);CHKERRQ(ierr);
7546     if (bsMinMax[0] != bsMinMax[1]) {bs = 1;}
7547     else                            {bs = bsMinMax[0];}
7548     ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
7549     for (p = pStart; p < pEnd; ++p) {
7550       PetscInt gdof, goff;
7551 
7552       ierr = PetscSectionGetDof(subsectionGlobal, p, &gdof);CHKERRQ(ierr);
7553       if (gdof > 0) {
7554         const PetscInt point = spmap[p];
7555 
7556         ierr = PetscSectionGetOffset(sectionGlobal, point, &goff);CHKERRQ(ierr);
7557         for (f = 0; f < Nf; ++f) {
7558           PetscInt fdof, fcdof, fc, f2, poff = 0;
7559 
7560           /* Can get rid of this loop by storing field information in the global section */
7561           for (f2 = 0; f2 < f; ++f2) {
7562             ierr  = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
7563             ierr  = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
7564             poff += fdof-fcdof;
7565           }
7566           ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
7567           ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
7568           for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
7569             subIndices[subOff] = goff+poff+fc;
7570           }
7571         }
7572       }
7573     }
7574     ierr = ISRestoreIndices(spIS, &spmap);CHKERRQ(ierr);
7575     ierr = ISDestroy(&spIS);CHKERRQ(ierr);
7576     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
7577     if (bs > 1) {
7578       /* We need to check that the block size does not come from non-contiguous fields */
7579       PetscInt i, j, set = 1;
7580       for (i = 0; i < subSize; i += bs) {
7581         for (j = 0; j < bs; ++j) {
7582           if (subIndices[i+j] != subIndices[i]+j) {set = 0; break;}
7583         }
7584       }
7585       if (set) {ierr = ISSetBlockSize(*is, bs);CHKERRQ(ierr);}
7586     }
7587     /* Attach nullspace */
7588     for (f = 0; f < Nf; ++f) {
7589       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
7590       if ((*subdm)->nullspaceConstructors[f]) break;
7591     }
7592     if (f < Nf) {
7593       MatNullSpace nullSpace;
7594 
7595       ierr = (*(*subdm)->nullspaceConstructors[f])(*subdm, f, &nullSpace);CHKERRQ(ierr);
7596       ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
7597       ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
7598     }
7599   }
7600   PetscFunctionReturn(0);
7601 }
7602